AnyEvent-Feed-0.3/0000755000000000000000000000000011533114303012436 5ustar rootrootAnyEvent-Feed-0.3/MANIFEST0000644000000000000000000000042311533114303013566 0ustar rootrootChanges MANIFEST Makefile.PL README t/00-load.t lib/AnyEvent/Feed.pm samples/gmail_unread_feed samples/simple_reader META.yml Module meta-data (added by MakeMaker) META.json Module meta-data (added by MakeMaker) AnyEvent-Feed-0.3/samples/0000755000000000000000000000000011533114303014102 5ustar rootrootAnyEvent-Feed-0.3/samples/simple_reader0000755000000000000000000000113111444066621016651 0ustar rootroot#!/opt/perl/bin/perl use strict; use Encode; use AnyEvent::Feed; use AnyEvent; $XML::Atom::ForceUnicode = 1; my $cv = AnyEvent->condvar; my $f = AnyEvent::Feed->new ( url => $ARGV[0], interval => 300, on_fetch => sub { my ($fee, $ent, $feed, $er) = @_; if (defined $er) { warn "ERROR: $er\n"; $cv->send; return; } for (@$ent) { printf "New story found:\n %s\n => %s\n", encode ('utf-8', $_->[1]->title), $_->[1]->link; } }); $cv->recv; AnyEvent-Feed-0.3/samples/gmail_unread_feed0000755000000000000000000000127411444066621017460 0ustar rootroot#!/opt/perl/bin/perl use strict; use Encode; use AnyEvent::Feed; use AnyEvent; $XML::Atom::ForceUnicode = 1; my $cv = AnyEvent->condvar; my $f = AnyEvent::Feed->new ( url => 'https://mail.google.com/mail/feed/atom/unread/', username => $ARGV[0], password => $ARGV[1], interval => 60, on_fetch => sub { my ($fee, $ent, $feed, $er) = @_; if (defined $er) { warn "ERROR: $er\n"; $cv->send; return; } for (@$ent) { printf "New mail found:\n %s\n => %s\n", encode ('utf-8', $_->[1]->title), $_->[1]->link; } }); $cv->recv; AnyEvent-Feed-0.3/Changes0000644000000000000000000000134611533114010013730 0ustar rootrootRevision history for AnyEvent::Feed 0.3 Tue Mar 1 07:53:57 CET 2011 - fixed typo in the synopsis. - applied patch for encode_base64 issue, which trashes the HTTP headers. Thanks to Michael Stapelberg for spotting. 0.2 Fri Jul 24 14:40:48 CEST 2009 - added note about encoding problems with XML::Atom. - added 'headers' field to constructor for additional http headers. - added 'username' and 'password' field to constructor, for HTTP Basic Auth. - added googlemail unread messages example script. - added 'If-Modified-Since' header support for a great deal of bandwidth savings. 0.1 Fri Jul 3 00:57:01 CEST 2009 - initial release. AnyEvent-Feed-0.3/t/0000755000000000000000000000000011533114303012701 5ustar rootrootAnyEvent-Feed-0.3/t/00-load.t0000644000000000000000000000023011444066621014227 0ustar rootroot#!perl -T use Test::More tests => 1; BEGIN { use_ok( 'AnyEvent::Feed' ); } diag( "Testing AnyEvent::Feed $AnyEvent::Feed::VERSION, Perl $], $^X" ); AnyEvent-Feed-0.3/lib/0000755000000000000000000000000011533114303013204 5ustar rootrootAnyEvent-Feed-0.3/lib/AnyEvent/0000755000000000000000000000000011533114303014735 5ustar rootrootAnyEvent-Feed-0.3/lib/AnyEvent/Feed.pm0000644000000000000000000002275711533114022016151 0ustar rootrootpackage AnyEvent::Feed; use strict; no warnings; use Carp qw/croak/; use Encode; use XML::Feed; use MIME::Base64; use AnyEvent::HTTP; use Digest::SHA1 qw/sha1_base64/; use Scalar::Util qw/weaken/; our $VERSION = '0.3'; =head1 NAME AnyEvent::Feed - Receiving RSS/Atom Feed reader with XML::Feed =head1 VERSION Version 0.3 =head1 SYNOPSIS use AnyEvent; use AnyEvent::Feed; my $feed_reader = AnyEvent::Feed->new ( url => 'http://example.com/atom.xml', ); $feed_reader->fetch (sub { my ($feed_reader, $new_entries, $feed, $error) = @_; if (defined $error) { warn "ERROR: $error\n"; return; } # $feed is the XML::Feed object belonging to that fetch. for (@$new_entries) { my ($hash, $entry) = @$_; # $hash a unique hash describing the $entry # $entry is the XML::Feed::Entry object of the new entries # since the last fetch. } }); # Or: my $feed_reader = AnyEvent::Feed->new ( url => 'http://example.com/atom.xml', interval => $seconds, on_fetch => sub { my ($feed_reader, $new_entries, $feed, $error) = @_; if (defined $error) { warn "ERROR: $error\n"; return; } # see above } ); =head1 DESCRIPTION This module implements some glue between L and L. It can fetch a RSS/Atom feed on a regular interval as well as on customized times. It also keeps track of already fetched entries so that you will only get the new entries. =head1 METHODS =over 4 =item $feed_reader = AnyEvent::Feed->new (url => $url, %args) This is the constructor for a new feed reader for the RSS/Atom feed reachable by the URL C<$url>. C<%args> may contain additional key/value pairs: =over 4 =item interval => $seconds If this is set you also have to specify the C callback (see below). It will try to fetch the C<$url> every C<$seconds> seconds and call the callback given by C with the result. =item headers => $http_hdrs Additional HTTP headers for each GET request can be passed in the C<$http_hdrs> hash reference, just like you would pass it to the C argument of the C request of L. =item username => $http_user =item password => $http_pass These are the HTTP username and password that will be used for Basic HTTP Authentication with the HTTP server when fetching the feed. This is mostly sugar for you so you don't have to encode them yourself and pass them to the C argument above. =item on_fetch => $cb->($feed_reader, $new_entries, $feed_obj, $error) This callback is called if the C parameter is given (see above) with the same arguments as the callback given to the C method (see below). =item entry_ages => $hash This will set the hash which keeps track of seen and old entries. See also the documentation of the C method below. The default will be an empty hash reference. =item max_entry_age => $count This will set the maximum number of times an entry is kept in the C hash after it has not been seen in the feed anymore. The default value is 2 which means that an entry hash is removed from the C hash after it has not been seen in the feed for 2 fetches. =back =cut sub new { my $this = shift; my $class = ref($this) || $this; my $self = { @_ }; bless $self, $class; $self->{entry_ages} ||= {}; if (defined $self->{interval}) { unless (defined $self->{on_fetch}) { croak "no 'on_fetch' callback given!"; } my $wself = $self; weaken $wself; $self->{timer_cb} = sub { $wself->fetch (sub { my ($self, $e, $f, $err) = @_; $self->{on_fetch}->($self, $e, $f, $err); $self->{timer} = AnyEvent->timer ( after => $self->{interval}, cb => $self->{timer_cb}); }) }; $self->{timer_cb}->(); } return $self } sub _entry_to_hash { my ($entry) = @_; my $x = sha1_base64 encode 'utf-8', (my $a = join '/', $entry->title, ($entry->summary ? $entry->summary->body : ''), ($entry->content ? $entry->content->body : ''), $entry->id, $entry->link); $x } sub _new_entries { my ($self) = @_; $self->{entry_ages} ||= {}; my (@ents) = $self->{feed}->entries; my @new; # 'age' the old entries $self->{entry_ages}->{$_}++ for keys %{$self->{entry_ages}}; for my $ent (@ents) { my $hash = _entry_to_hash ($ent); unless (exists $self->{entry_ages}->{$hash}) { push @new, [$hash, $ent]; } $self->{entry_ages}->{$hash} = 0; # reset age of old entry. } for (keys %{$self->{entry_ages}}) { delete $self->{entry_ages}->{$_} if $self->{entry_ages}->{$_} > $self->{max_entry_ages}; } \@new } =item $feed_reader->url Just returns the url that this feed reader is fetching from. =cut sub url { $_[0]->{url} } =item $feed_reader->entry_ages ($new_entry_ages) =item my $entry_ages = $feed_reader->entry_ages This will set the age hash which will keep track of already seen entries. The keys of the hash will be the calculated hashes of the entries and the values will be a counter of how often they have NOT been seen anymore (kind of an age counter). After each fetch this hash is updated and seen entries get a value of 0. =cut sub entry_ages { defined $_[1] ? $_[0]->{entry_ages} = $_[1] : $_[0]->{entry_ages} } =item $feed_reader->fetch ($cb->($feed_reader, $new_entries, $feed_obj, $error)) This will initiate a HTTP GET on the URL passed to C and call C<$cb> when done. C<$feed_reader> is the feed reader object itself. C<$new_entries> is an array reference containing the new entries. A new entry in that array is another array containing a calculated hash over the contents of the new entry, and the L object of that entry. C<$feed_obj> is the L feed object used to parse the fetched feed and contains all entries (and not just the 'new' ones). What a 'new' entry is, is decided by a map of hashes as described in the C method's documentation above. =cut sub _get_headers { my ($self, %hdrs) = @_; my %hdrs = %{$self->{headers} || {}}; if (defined $self->{last_mod}) { $hdrs{'If-Modified-Since'} = $self->{last_mod}; } $hdrs{Authorization} = "Basic " . encode_base64 (join ':', $self->{username}, $self->{password}, '') if defined $self->{username}; \%hdrs } sub fetch { my ($self, $cb) = @_; unless (defined $cb) { croak "no callback given to fetch!"; } http_get $self->{url}, headers => $self->_get_headers, sub { my ($data, $hdr) = @_; #d# warn "HEADERS ($self->{last_mod}): " #d# . (join ",\n", map { "$_:\t$hdr->{$_}" } keys %$hdr) #d# . "\n"; if ($hdr->{Status} =~ /^2/) { my $feed; eval { $self->{feed} = XML::Feed->parse (\$data); }; if ($@) { $cb->($self, undef, undef, "exception: $@"); } elsif (not defined $self->{feed}) { $cb->($self, undef, undef, XML::Feed->errstr); } else { $cb->($self, $self->_new_entries, $self->{feed}); $self->{last_mod} = $hdr->{'last-modified'}; } } elsif (defined ($self->{last_mod}) && $hdr->{Status} eq '304') { # do nothing, everything was/is fine! $cb->($self, [], $self->{feed}); } else { $cb->($self, undef, undef, "$hdr->{Status} $hdr->{Reason}"); } }; } =back =head1 AUTHOR Robin Redeker, C<< >> =head1 SEE ALSO L L L =head1 BUGS =head2 Known Bugs There is actually a known bug with encodings of contents of Atom feeds. L by default gives you UTF-8 encoded data. You have to set this global variable to be able to use the L interface without knowledge of the underlying feed type: $XML::Atom::ForceUnicode = 1; I've re-reported this bug against L, as I think it should take care of this. L should probably just fix it's Unicode interface, but it seems to be a bit deserted w.r.t. fixing the bugs in the tracker. =head2 Contact Please report any bugs or feature requests to C, or through the web interface at L. I will be notified and then you'll automatically be notified of progress on your bug as I make changes. =head1 SUPPORT You can find documentation for this module with the perldoc command. perldoc AnyEvent::Feed You can also look for information at: =over 4 =item * IRC: AnyEvent::Feed IRC Channel See the same channel as the L module: IRC Network: http://freenode.net/ Server : chat.freenode.net Channel : #ae_xmpp Feel free to join and ask questions! =item * AnnoCPAN: Annotated CPAN documentation L =item * CPAN Ratings L =item * RT: CPAN's request tracker L =item * Search CPAN L =back =head1 COPYRIGHT & LICENSE Copyright 2009 Robin Redeker, all rights reserved. This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut 1; AnyEvent-Feed-0.3/README0000644000000000000000000001517511533114303013327 0ustar rootrootNAME AnyEvent::Feed - Receiving RSS/Atom Feed reader with XML::Feed VERSION Version 0.3 SYNOPSIS use AnyEvent; use AnyEvent::Feed; my $feed_reader = AnyEvent::Feed->new ( url => 'http://example.com/atom.xml', ); $feed_reader->fetch (sub { my ($feed_reader, $new_entries, $feed, $error) = @_; if (defined $error) { warn "ERROR: $error\n"; return; } # $feed is the XML::Feed object belonging to that fetch. for (@$new_entries) { my ($hash, $entry) = @$_; # $hash a unique hash describing the $entry # $entry is the XML::Feed::Entry object of the new entries # since the last fetch. } }); # Or: my $feed_reader = AnyEvent::Feed->new ( url => 'http://example.com/atom.xml', interval => $seconds, on_fetch => sub { my ($feed_reader, $new_entries, $feed, $error) = @_; if (defined $error) { warn "ERROR: $error\n"; return; } # see above } ); DESCRIPTION This module implements some glue between AnyEvent::HTTP and XML::Feed. It can fetch a RSS/Atom feed on a regular interval as well as on customized times. It also keeps track of already fetched entries so that you will only get the new entries. METHODS $feed_reader = AnyEvent::Feed->new (url => $url, %args) This is the constructor for a new feed reader for the RSS/Atom feed reachable by the URL $url. %args may contain additional key/value pairs: interval => $seconds If this is set you also have to specify the "on_fetch" callback (see below). It will try to fetch the $url every $seconds seconds and call the callback given by "on_fetch" with the result. headers => $http_hdrs Additional HTTP headers for each GET request can be passed in the $http_hdrs hash reference, just like you would pass it to the "headers" argument of the "http_get" request of AnyEvent::HTTP. username => $http_user password => $http_pass These are the HTTP username and password that will be used for Basic HTTP Authentication with the HTTP server when fetching the feed. This is mostly sugar for you so you don't have to encode them yourself and pass them to the "headers" argument above. on_fetch => $cb->($feed_reader, $new_entries, $feed_obj, $error) This callback is called if the "interval" parameter is given (see above) with the same arguments as the callback given to the "fetch" method (see below). entry_ages => $hash This will set the hash which keeps track of seen and old entries. See also the documentation of the "entry_ages" method below. The default will be an empty hash reference. max_entry_age => $count This will set the maximum number of times an entry is kept in the "entry_ages" hash after it has not been seen in the feed anymore. The default value is 2 which means that an entry hash is removed from the "entry_ages" hash after it has not been seen in the feed for 2 fetches. $feed_reader->url Just returns the url that this feed reader is fetching from. $feed_reader->entry_ages ($new_entry_ages) my $entry_ages = $feed_reader->entry_ages This will set the age hash which will keep track of already seen entries. The keys of the hash will be the calculated hashes of the entries and the values will be a counter of how often they have NOT been seen anymore (kind of an age counter). After each fetch this hash is updated and seen entries get a value of 0. $feed_reader->fetch ($cb->($feed_reader, $new_entries, $feed_obj, $error)) This will initiate a HTTP GET on the URL passed to "new" and call $cb when done. $feed_reader is the feed reader object itself. $new_entries is an array reference containing the new entries. A new entry in that array is another array containing a calculated hash over the contents of the new entry, and the XML::Feed::Entry object of that entry. $feed_obj is the XML::Feed feed object used to parse the fetched feed and contains all entries (and not just the 'new' ones). What a 'new' entry is, is decided by a map of hashes as described in the "entry_ages" method's documentation above. AUTHOR Robin Redeker, "" SEE ALSO XML::Feed AnyEvent::HTTP AnyEvent BUGS Known Bugs There is actually a known bug with encodings of contents of Atom feeds. XML::Atom by default gives you UTF-8 encoded data. You have to set this global variable to be able to use the XML::Feed::Entry interface without knowledge of the underlying feed type: $XML::Atom::ForceUnicode = 1; I've re-reported this bug against XML::Feed, as I think it should take care of this. XML::Atom should probably just fix it's Unicode interface, but it seems to be a bit deserted w.r.t. fixing the bugs in the tracker. Contact Please report any bugs or feature requests to "bug-anyevent-feed at rt.cpan.org", or through the web interface at . I will be notified and then you'll automatically be notified of progress on your bug as I make changes. SUPPORT You can find documentation for this module with the perldoc command. perldoc AnyEvent::Feed You can also look for information at: * IRC: AnyEvent::Feed IRC Channel See the same channel as the AnyEvent::XMPP module: IRC Network: http://freenode.net/ Server : chat.freenode.net Channel : #ae_xmpp Feel free to join and ask questions! * AnnoCPAN: Annotated CPAN documentation * CPAN Ratings * RT: CPAN's request tracker * Search CPAN COPYRIGHT & LICENSE Copyright 2009 Robin Redeker, all rights reserved. This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. AnyEvent-Feed-0.3/Makefile.PL0000644000000000000000000000165611444066621014432 0ustar rootrootuse strict; use warnings; use ExtUtils::MakeMaker; WriteMakefile( NAME => 'AnyEvent::Feed', AUTHOR => 'Robin Redeker ', LICENSE => 'perl', VERSION_FROM => 'lib/AnyEvent/Feed.pm', ABSTRACT_FROM => 'lib/AnyEvent/Feed.pm', PL_FILES => {}, test => { TESTS => "t/*.t t/methds/*.t" }, PREREQ_PM => { 'Test::More' => 0, 'AnyEvent' => 3.5, 'MIME::Base64' => 0, 'AnyEvent::HTTP' => 0, 'Digest::SHA1' => 0, 'Scalar::Util' => 0, 'XML::Feed' => 0, 'Encode' => 0, }, dist => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', PREOP => 'pod2text lib/AnyEvent/Feed.pm | tee README >$(DISTVNAME)/README; chmod -R u=rwX,go=rX . ;', }, clean => { FILES => 'AnyEvent-Feed' }, ); AnyEvent-Feed-0.3/META.yml0000644000000000000000000000151711533114303013713 0ustar rootroot{ "no_index" : { "directory" : [ "t", "inc" ] }, "meta-spec" : { "version" : 1.4, "url" : "http://module-build.sourceforge.net/META-spec-v1.4.html" }, "generated_by" : "ExtUtils::MakeMaker version 6.56", "distribution_type" : "module", "version" : "0.3", "name" : "AnyEvent-Feed", "author" : [ "Robin Redeker " ], "license" : "perl", "build_requires" : { "ExtUtils::MakeMaker" : 0 }, "requires" : { "Scalar::Util" : 0, "Test::More" : 0, "AnyEvent" : 3.5, "XML::Feed" : 0, "Digest::SHA1" : 0, "MIME::Base64" : 0, "AnyEvent::HTTP" : 0, "Encode" : 0 }, "abstract" : "Receiving RSS/Atom Feed reader with XML::Feed", "configure_requires" : { "ExtUtils::MakeMaker" : 0 } } AnyEvent-Feed-0.3/META.json0000644000000000000000000000112711533114303014060 0ustar rootroot{"no_index":{"directory":["t","inc"]},"meta-spec":{"version":1.4,"url":"http://module-build.sourceforge.net/META-spec-v1.4.html"},"generated_by":"ExtUtils::MakeMaker version 6.56","distribution_type":"module","version":"0.3","name":"AnyEvent-Feed","author":["Robin Redeker "],"license":"perl","build_requires":{"ExtUtils::MakeMaker":0},"requires":{"Scalar::Util":0,"Test::More":0,"AnyEvent":3.5,"XML::Feed":0,"Digest::SHA1":0,"MIME::Base64":0,"AnyEvent::HTTP":0,"Encode":0},"abstract":"Receiving RSS/Atom Feed reader with XML::Feed","configure_requires":{"ExtUtils::MakeMaker":0}}