LWP-UserAgent-Cached-0.08/0000755000175000017500000000000013570253100013575 5ustar olegolegLWP-UserAgent-Cached-0.08/Makefile.PL0000644000175000017500000000153313570245616015566 0ustar olegoleguse 5.005000; use ExtUtils::MakeMaker; use strict; # See lib/ExtUtils/MakeMaker.pm for details of how to influence # the contents of the Makefile that is written. WriteMakefile( NAME => 'LWP::UserAgent::Cached', LICENSE => 'perl', VERSION_FROM => 'lib/LWP/UserAgent/Cached.pm', # finds $VERSION PREREQ_PM => { 'LWP::UserAgent' => 0 }, # e.g., Module::Name => 1.1 BUILD_REQUIRES => { 'File::Temp' => 0, 'Test::More' => 0.88, }, CONFIGURE_REQUIRES => { 'ExtUtils::MakeMaker' => 6.52, }, META_MERGE => { resources => {repository => 'https://github.com/olegwtf/p5-LWP-UserAgent-Cached'}, }, ($] >= 5.005 ? ## Add these new keywords supported since 5.005 (ABSTRACT_FROM => 'lib/LWP/UserAgent/Cached.pm', # retrieve abstract from module AUTHOR => 'Oleg G ') : ()), ); LWP-UserAgent-Cached-0.08/t/0000755000175000017500000000000013570253100014040 5ustar olegolegLWP-UserAgent-Cached-0.08/t/2_request.t0000644000175000017500000001554213570252617016161 0ustar olegoleg#!/usr/bin/env perl use strict; use Test::More; use HTTP::Response; use HTTP::Request; use Digest::MD5; use File::Temp 'tempdir'; use LWP::UserAgent::Cached; my $cache_dir = eval { tempdir(CLEANUP => 1) }; unless ($cache_dir) { plan skip_all => "Сan't create temp dir"; } my %MAP; no warnings ('redefine', 'once'); *LWP::UserAgent::send_request = sub { my ($self, $req) = @_; if (my $resp = $MAP{$req->uri}) { $resp->request($req); return $resp; } return HTTP::Response->new(404); }; *LWP::UserAgent::map = sub { my ($self, $uri, $resp) = @_; $MAP{$uri} = $resp; return $uri; }; *LWP::UserAgent::unmap = sub { my ($self, $uri) = @_; delete $MAP{$uri}; }; use warnings ('redefine', 'once'); my $ua = LWP::UserAgent::Cached->new(cache_dir => $cache_dir, cookie_jar => {}); # simple request test my $mid = $ua->map('http://www.google.com/', HTTP::Response->new(200)); $ua->get('http://www.google.com/'); # cache 200 OK $ua->unmap($mid); $ua->map('http://www.google.com/', HTTP::Response->new(500)); is($ua->get('http://www.google.com/')->code, 200, 'Cached 200 ok response'); # more complex request test my $response = HTTP::Response->new(301, 'Moved Permanently', [Location => 'http://www.yahoo.com/']); $response->request(HTTP::Request->new(GET => 'http://yahoo.com')); $mid = $ua->map('http://yahoo.com', $response); my $y_mid = $ua->map('http://www.yahoo.com/', HTTP::Response->new(200, 'Ok', ['Set-Cookie' => 'lwp=true', 'Set-Cookie' => 'cached=yes'], 'This is a test')); $ua->get('http://yahoo.com'); # make cache is(scalar($ua->last_cached), 2, '@last_cached length = 2 on redirect'); is(scalar($ua->last_used_cache), 2, '@last_used_cache length = 2 on redirect'); $ua->unmap($mid); $ua->cookie_jar->clear(); my $resp = $ua->get('http://yahoo.com'); is($resp->code, 200, 'Cached response with redirect'); ok(index($resp->content, 'This is a test')!=-1, 'Cached response content') or diag "Content: ", $resp->content; my @cookies; $ua->cookie_jar->scan(sub { my ($ver, $key, $val) = @_; push @cookies, "$key=$val" }); is(@cookies, 2, 'Got correct cookies count from the cache'); @cookies = sort @cookies; is($cookies[0], 'cached=yes', 'Got correct first cookie'); is($cookies[1], 'lwp=true', 'Got correct second cookie'); is(scalar($ua->last_cached), 0, '@last_cached length = 0 when get from cache'); is(scalar($ua->last_used_cache), 2, '@last_used_cache length = 2 when get from cache'); # cookies bug (#gh5) $resp = $ua->get('http://www.yahoo.com/'); is(() = $resp->request->header('Cookie') =~ /lwp=true/g, 1, 'correct cookie 1 sent in the request'); is(() = $resp->request->header('Cookie') =~ /cached=yes/g, 1, 'correct cookie 2 sent in the request'); # binary content test my $content = "\0\1\2\3\4\5\6"; $mid = $ua->map('http://mail.com/bin', HTTP::Response->new(200, 'OK', [], $content)); $ua->get('http://mail.com/bin'); $ua->unmap($mid); is($ua->get('http://mail.com/bin')->decoded_content, $content, "got right binary content"); $content .= "\n"; $mid = $ua->map('http://mail.com/bin2', HTTP::Response->new(200, 'OK', [], $content)); $ua->get('http://mail.com/bin2'); $ua->unmap($mid); is($ua->get('http://mail.com/bin2')->decoded_content, $content, "got right binary content with trailing new line"); # nocache_if test $ua->nocache_if(sub { $_[0]->code > 399 }); $mid = $ua->map('http://perl.org', HTTP::Response->new(403, 'Forbbidden')); $ua->get('http://perl.org'); $ua->unmap($mid); $ua->map('http://perl.org', HTTP::Response->new(200, 'OK', [], 'Perl there')); $resp = $ua->get('http://perl.org'); is($resp->code, 200, 'Nocache code'); ok(index($resp->content, 'Perl there')!=-1, 'Nocache content') or diag 'Content: ', $resp->content; $ua->nocache_if(undef); # recache_if test $ua->recache_if(sub { my ($resp, $path, $req) = @_; isa_ok($resp, 'HTTP::Response'); isa_ok($req, 'HTTP::Request'); ok(-e $path, 'Cached file exists') or diag "Path: $path"; 1; }); $mid = $ua->map('http://perlmonks.org', HTTP::Response->new(407)); $ua->get('http://perlmonks.org'); $ua->unmap($mid); $ua->map('http://perlmonks.org', HTTP::Response->new(200)); is($ua->get('http://perlmonks.org')->code, 200, 'Recached'); $ua->recache_if(undef); # on_uncached test $mid = $ua->map('http://www.modernperlbooks.com/', HTTP::Response->new(200)); my $on_uncached; $ua->on_uncached(sub { $on_uncached = 1 }); $ua->get('http://www.modernperlbooks.com/'); is($on_uncached, 1, 'on_uncached called'); $on_uncached = undef; $ua->get('http://www.modernperlbooks.com/'); is($on_uncached, undef, 'on_uncached not called'); $ua->on_uncached(undef); # uncache test $mid = $ua->map('http://metacpan.org', HTTP::Response->new(200)); $ua->get('http://metacpan.org'); $ua->uncache(); $ua->unmap($mid); $ua->map('http://metacpan.org', HTTP::Response->new(503)); is($ua->get('http://metacpan.org')->code, 503, 'Uncache last response'); # collision test $ua->cookie_jar->clear(); $resp = $ua->get('http://yahoo.com'); $ua->cookie_jar->clear(); my $cache_name = $ua->_get_cache_name($resp->request); open FH, '>:raw', $cache_name; print FH "http://google.com\nHTTP/1.1 200 OK\n"; close FH; $ua->get('http://yahoo.com'); ok(-e "$cache_name-001", "Collision detected"); open FH, '>:raw', "$cache_name-001"; print FH "http://google.com\nHTTP/1.1 200 OK\n"; close FH; $ua->cookie_jar->clear(); $ua->get('http://yahoo.com'); ok(-e "$cache_name-002", "Double collision detected"); $ua->unmap($y_mid); $ua->map('http://www.yahoo.com/', HTTP::Response->new(404)); is($ua->get('http://yahoo.com')->code, 200, 'Cached response (collision list)'); # cache name specification test $mid = $ua->map('http://perl.com', HTTP::Response->new(200)); $ua->agent('Internet-Explorer'); $ua->get('http://perl.com'); $ua->post('http://perl.com', ['q' => 'perl6', 'w' => 'now']); $ua->unmap($mid); $mid = $ua->map('http://perl.com', HTTP::Response->new(500)); $ua->cachename_spec({ 'User-Agent' => 'Internet-Explorer', 'Accept' => undef }); $ua->agent('Mozilla/5.0'); is($ua->get('http://perl.com', Accept => 'text/html')->code, 200, 'Cache name specification'); $ua->cachename_spec({ _headers => ['User-Agent'], 'User-Agent' => 'Internet-Explorer' }); is($ua->get('http://perl.com', Accept => 'text/html')->code, 200, 'Cache name specification with _headers'); $ua->cachename_spec({ _body => 'q=perl6&w=now', 'User-Agent' => 'Internet-Explorer', 'Content-Length' => 13, }); is($ua->post('http://perl.com', ['q' => 'perl5', 'w' => 'yesterday'])->code, 200, 'Cache name specification with _body'); $mid = $ua->map('http://pause.perl.org', HTTP::Response->new(200)); $ua->cachename_spec({ _body => '', _headers => [] }); $ua->post('http://pause.perl.org', [u => 'OLEG', act => 'login'], 'Accept' => 'text/html', 'Accept-Charset' => 'iso-8859'); $ua->unmap($mid); $mid = $ua->map('http://pause.perl.org' => HTTP::Response->new(500)); $ua->agent('Google/Chrome'); is($ua->post('http://pause.perl.org', [u => 'UDGIN', act => 'logout'])->code, 200, 'Cache name based on url and http method'); done_testing; LWP-UserAgent-Cached-0.08/t/1_load.t0000644000175000017500000000154613570236335015405 0ustar olegoleg#!/usr/bin/env perl use Test::More; use strict; use_ok('LWP::UserAgent::Cached'); my $ua = LWP::UserAgent::Cached->new(); ok(defined $ua, 'new ua'); isa_ok($ua, 'LWP::UserAgent::Cached'); isa_ok($ua, 'LWP::UserAgent'); $ua = LWP::UserAgent::Cached->new(cache_dir => '/tmp', nocache_if => sub{1}, recache_if => sub{1}); is($ua->cache_dir, '/tmp', 'cache_dir param'); is(ref($ua->nocache_if), 'CODE', 'nocache_if is code'); is(ref($ua->recache_if), 'CODE', 'recache_if is code'); $ua->cache_dir('/var/tmp'); is($ua->cache_dir, '/var/tmp', 'runtime change cache_dir param'); my $old_nocache_if = $ua->nocache_if; $ua->nocache_if(sub{0}); isnt($old_nocache_if, $ua->nocache_if, 'runtime change nocache_if param'); my $old_recache_if = $ua->recache_if; $ua->recache_if(sub{0}); isnt($old_recache_if, $ua->recache_if, 'runtime change recache_if param'); done_testing; LWP-UserAgent-Cached-0.08/META.yml0000644000175000017500000000123313570253100015045 0ustar olegoleg--- abstract: 'LWP::UserAgent with simple caching mechanism' author: - 'Oleg G ' build_requires: File::Temp: '0' Test::More: '0.88' configure_requires: ExtUtils::MakeMaker: '6.52' dynamic_config: 1 generated_by: 'ExtUtils::MakeMaker version 7.1002, CPAN::Meta::Converter version 2.150005' license: perl meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html version: '1.4' name: LWP-UserAgent-Cached no_index: directory: - t - inc requires: LWP::UserAgent: '0' resources: repository: https://github.com/olegwtf/p5-LWP-UserAgent-Cached version: '0.08' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' LWP-UserAgent-Cached-0.08/lib/0000755000175000017500000000000013570253100014343 5ustar olegolegLWP-UserAgent-Cached-0.08/lib/LWP/0000755000175000017500000000000013570253100015005 5ustar olegolegLWP-UserAgent-Cached-0.08/lib/LWP/UserAgent/0000755000175000017500000000000013570253100016702 5ustar olegolegLWP-UserAgent-Cached-0.08/lib/LWP/UserAgent/Cached.pm0000644000175000017500000003313513570252632020425 0ustar olegolegpackage LWP::UserAgent::Cached; use strict; use Carp; use Digest::MD5; use HTTP::Response; use base 'LWP::UserAgent'; our $VERSION = '0.08'; sub new { my ($class, %opts) = @_; my $cache_dir = delete $opts{cache_dir}; my $nocache_if = delete $opts{nocache_if}; my $recache_if = delete $opts{recache_if}; my $on_uncached = delete $opts{on_uncached}; my $cachename_spec = delete $opts{cachename_spec}; my $self = $class->SUPER::new(%opts, parse_head => 0); $self->{cache_dir} = $cache_dir; $self->{nocache_if} = $nocache_if; $self->{recache_if} = $recache_if; $self->{on_uncached} = $on_uncached; $self->{cachename_spec} = $cachename_spec; return $self; } # generate getters and setters foreach my $opt_name (qw(cache_dir nocache_if recache_if on_uncached cachename_spec)) { no strict 'refs'; *$opt_name = sub { my $self = shift; if (@_) { my $opt_val = $self->{$opt_name}; $self->{$opt_name} = shift; return $opt_val; } return $self->{$opt_name}; } } sub parse_head { my ($self, $bool) = @_; if ($bool) { die "parse_head() is disabled, because it may cause encoding troubles while saving cache"; } $self->SUPER::parse_head($bool); } sub simple_request { my $self = shift; unless (defined $self->{cache_dir}) { return $self->SUPER::simple_request(@_); } my $request = $_[0]; $request = $self->prepare_request($request); my $fpath = $self->_get_cache_name($request); my $response; my $no_collision_suffix; unless ($self->{was_redirect}) { @{$self->{last_cached}} = (); @{$self->{last_used_cache}} = (); } if (-e $fpath) { unless ($response = $self->_parse_cached_response($fpath, $request)) { # collision if (my @cache_list = <$fpath-*>) { foreach my $cache_file (@cache_list) { if ($response = $self->_parse_cached_response($cache_file, $request)) { $fpath = $cache_file; last; } } unless ($response) { $no_collision_suffix = sprintf('-%03d', substr($cache_list[-1], -3) + 1); } } else { $no_collision_suffix = '-001'; } } if ($response && defined($self->{recache_if}) && $self->{recache_if}->($response, $fpath, $request)) { $response = undef; } } unless ($response) { if (defined $self->{on_uncached}) { $self->{on_uncached}->($request); } $response = $self->send_request(@_); if (!defined($self->{nocache_if}) || !$self->{nocache_if}->($response)) { if (defined $no_collision_suffix) { $fpath .= $no_collision_suffix; } if (open my $fh, '>:raw', $fpath) { print $fh $request->url, "\n"; print $fh $response->as_string("\n"); close $fh; push @{$self->{last_cached}}, $fpath; push @{$self->{last_used_cache}}, $fpath; } else { carp "open('$fpath', 'w'): $!"; } } } else { push @{$self->{last_used_cache}}, $fpath; } $self->{was_redirect} = $response->is_redirect && _in($request->method, $self->requests_redirectable); return $response; } sub last_cached { my $self = shift; return exists $self->{last_cached} ? @{$self->{last_cached}} : (); } sub last_used_cache { my $self = shift; return exists $self->{last_used_cache} ? @{$self->{last_used_cache}} : (); } sub uncache { my $self = shift; unlink $_ for $self->last_cached; } sub _get_cache_name { my ($self, $request) = @_; if (defined($self->{cachename_spec}) && %{$self->{cachename_spec}}) { my $tmp_request = $request->clone(); my $leave_only_specified; if (exists $self->{cachename_spec}{_headers}) { ref $self->{cachename_spec}{_headers} eq 'ARRAY' or croak 'cachename_spec->{_headers} should be array ref'; $leave_only_specified = 1; } foreach my $hname ($tmp_request->headers->header_field_names) { if (exists $self->{cachename_spec}{$hname}) { if (defined $self->{cachename_spec}{$hname}) { $tmp_request->headers->header($hname, $self->{cachename_spec}{$hname}); } else { $tmp_request->headers->remove_header($hname); } } elsif ($leave_only_specified && !_in($hname, $self->{cachename_spec}{_headers})) { $tmp_request->headers->remove_header($hname); } } if (exists $self->{cachename_spec}{_body}) { $tmp_request->content($self->{cachename_spec}{_body}); } return $self->{cache_dir} . '/' . Digest::MD5::md5_hex($tmp_request->as_string("\n")); } return $self->{cache_dir} . '/' . Digest::MD5::md5_hex($request->as_string("\n")); } sub _parse_cached_response { my ($self, $cache_file, $request) = @_; my $fh; unless (open $fh, '<:raw', $cache_file) { carp "open('$cache_file', 'r'): $!"; return; } my $url = <$fh>; $url =~ s/\s+$//; if ($url ne $request->url) { close $fh; return; } local $/ = undef; my $response_str = <$fh>; close $fh; my $response = HTTP::Response->parse($response_str); $response->request($request); if ($self->cookie_jar) { $self->cookie_jar->extract_cookies($response); } return $response; } sub _in($$) { my ($what, $where) = @_; foreach my $item (@$where) { return 1 if ($what eq $item); } return 0; } 1; =pod =head1 NAME LWP::UserAgent::Cached - LWP::UserAgent with simple caching mechanism =head1 SYNOPSIS use LWP::UserAgent::Cached; my $ua = LWP::UserAgent::Cached->new(cache_dir => '/tmp/lwp-cache'); my $resp = $ua->get('http://google.com/'); # makes http request ... $resp = $ua->get('http://google.com/'); # no http request - will get it from the cache =head1 DESCRIPTION When you process content from some website, you will get page one by one and extract some data from this page with regexp, DOM parser or smth else. Sometimes we makes errors in our data extractors and realize this only when all 1_000_000 pages were processed. We should fix our extraction logic and start all process from the beginning. Please STOP! How about cache? Yes, you can cache all responses and second, third and other attempts will be very fast. LWP::UserAgent::Cached is yet another LWP::UserAgent subclass with cache support. It stores cache in the files on local filesystem and if response already available in the cache returns it instead of making HTTP request. This module was writed because other available alternatives didn't meet my needs: =over =item L caches responses on local filesystem and gets it from the cache only if online document was not modified =item L same as above but stores cache in memory =item L can record responses in the cache or get responses from the cache, but not both for one useragent =item L seems it may cache responses and get responses from the cache, but has too much dependencies and unclear `delay' parameter =back =head1 METHODS All LWP::UserAgent methods and several new. =head2 new(...) Creates new LWP::UserAgent::Cached object. Since LWP::UserAgent::Cached is LWP::UserAgent subclass it has all same parameters, but in additional it has some new optional pararmeters: L L L L L LWP::UserAgent::Cached creation example: my $ua = LWP::UserAgent::Cached->new(cache_dir => 'cache/lwp', nocache_if => sub { my $response = shift; return $response->code >= 500; # do not cache any bad response }, recache_if => sub { my ($response, $path, $request) = @_; return $response->code == 404 && -M $path > 1; # recache any 404 response older than 1 day }, on_uncached => sub { my $request = shift; sleep 5 if $request->uri =~ '/category/\d+'; # delay before http requests inside "/category" }, cachename_spec => { 'User-Agent' => undef, # omit agent while calculating cache name }); =head2 cache_dir() or cache_dir($dir) Gets or sets path to the directory where cache will be stored. If not set useragent will behaves as LWP::UserAgent without cache support. =head2 nocache_if() or nocache_if($sub) Gets or sets reference to subroutine which will be called after receiving each non-cached response. First parameter of this subroutine will be HTTP::Response object. This subroutine should return true if this response should not be cached and false otherwise. If not set all responses will be cached. =head2 recache_if() or recache_if($sub) Gets or sets reference to subroutine which will be called for each response available in the cache. First parameter of this subroutine will be HTTP::Response object, second - path to file with cache, third - HTTP::Request object. This subroutine should return true if response needs to be recached (new HTTP request will be made) and false otherwise. This $sub will be called only if response already available in the cache. Here you can also modify request for your needs. This will not change name of the file with cache. =head2 on_uncached() or on_uncached($sub) Gets or sets reference to subroutine which will be called for each non-cached http request, before actually request. First parameter of this subroutine will be HTTP::Request object. Here you can also modify request for your needs. This will not change name of the file with cache. =head2 cachename_spec() or cachename_spec($spec) Gets or sets hash reference to cache naming specification. In fact cache naming for each request based on request content. Internally it is md5_hex($request->as_string("\n")). But what if some of request headers in your program changed dinamically, e.g. User-Agent or Cookie? In such case caching will not work properly for you. We need some way to omit this headers when calculating cache name. This option is what you need. Specification hash should contain header name and header value which will be used (instead of values in request) while calculating cache name. For example we already have cache where 'User-Agent' value in the headers was 'Mozilla/5.0', but in the current version of the program it will be changed for each request. So we force specified that for cache name calculation 'User-Agent' should be 'Mozilla/5.0'. Cached request had not 'Accept' header, but in the current version it has. So we force specified do not include this header for cache name calculation. cachename_spec => { 'User-Agent' => 'Mozilla/5.0', 'Accept' => undef } Specification hash may contain two special keys: '_body' and '_headers'. With '_body' key you can specify body content in the request for cache name calculation. For example to not include body content in cache name calculation set '_body' to undef or empty string. With '_headers' key you can specify which headers should be included in $request for cache name calculation. For example you can say to include only 'Host' and 'Referer'. '_headers' value should be array reference: cachename_spec => { _body => undef, # omit body _headers => ['Host'], # include only host with value from request # It will be smth like: # md5_hex("METHOD url\r\nHost: host\r\n\r\n") # method and url will be included in any case } Another example. Omit body, include only 'Host' and 'User-Agent' headers, use 'Host' value from request and specified 'User-Agent' value, in addition include referrer with specified value ('Referer' not specified in '_headers', but values from main specification hash has higher priority): cachename_spec => { _body => '', _headers => ['Host', 'User-Agent'], 'User-Agent' => 'Mozilla/5.0', 'Referer' => 'http://www.com' } One more example. Calculate cache name based only on method and url: cachename_spec => { _body =>'', _headers => [] } =head2 last_cached() Returns list with pathes to files with cache stored by last noncached response. List may contain more than one element if there was redirect. =head2 last_used_cache() Returns list with pathes to files with cache used in last response. This includes files just stored (last_cached) and files that may be already exists (cached earlier). List may contain more than one element if there was redirect. =head2 uncache() Removes last response from the cache. Use case example: my $page = $ua->get($url)->decoded_content; if ($page =~ /Access for this ip was blocked/) { $ua->uncache(); } =head1 Proxy and cache name Here you can see how changing of proxy for useragent will affect cache name =head2 HTTP proxy HTTP proxy support works out of the box and causes no problems. Changing of proxy server will not affect cache name =head2 HTTPS proxy Proper HTTPS proxy support added in LWP since 6.06 and causes no problems. Changing of proxy server will not affect cache name =head2 CONNECT proxy CONNECT proxy support may be added using L. The problem is that this module uses LWP's request() for creation of CONNECT tunnel, so this response will be cached. But in fact it shouldn't. To workaround this you need to install C hook $ua->nocache_if(sub { my $resp = shift; # do not cache creation of tunnel $resp->request->method eq 'CONNECT'; }); After that it works without problems. Changing of proxy server will not affect cache name =head2 SOCKS proxy SOCKS proxy support may be added using L and causes no problems. Changing of proxy server will not affect cache name =head1 SEE ALSO L =head1 COPYRIGHT Copyright Oleg G . This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut LWP-UserAgent-Cached-0.08/Changes0000644000175000017500000000234113570252751015103 0ustar olegolegRevision history for Perl extension LWP::UserAgent::Cached. 0.08 Sat Nov 30 00:30:37 +07 2019 - FIX: cookies duplication bug (#gh5) 0.07 Fri Feb 22 12:46:12 +07 2019 - FIX: extra new line could be added to the content after restoring it from the cache, which could corrupt binary content (#gh4) - WARN: fix above may change cache name for requests without trailing new line (most of POST requests), so such requests may be recached 0.06 Wed Sep 10 02:38:48 NOVT 2014 - additionally pass $request as third argument to `recache_if' callback - documentation improved a little bit 0.05 Sat Jun 28 12:16:14 NOVT 2014 - `last_used_cache' method added - lwp's `parse_head' now always false to prevent problems with encoding for `as_string' 0.04 Sun Feb 10 20:28:09 2013 - `on_uncached' option added - small documentation fixes 0.03 Sun Jan 22 22:27:45 2012 - `nocache' renamed to `nocache_if' and `recache' to `recache_if' - `last_cached' method added - `cachename_spec' option added 0.02 Wed Jan 4 15:09:00 2012 - Makefile.PL fix for tests dependencies - `recache' callback was added 0.01 Mon Jan 2 21:59:07 2012 - original version; LWP-UserAgent-Cached-0.08/README0000644000175000017500000000140313570236335014466 0ustar olegolegLWP::UserAgent::Cached - LWP::UserAgent with simple caching mechanism ====================== When you process content from some website, you will get page one by one and extract some data from this page with regexp, DOM parser or smth else. Sometimes we makes errors in our data extractors and realize this only when all 1_000_000 pages were processed. We should fix our extraction logic and start all process from the beginning. Please STOP! How about cache? Yes, you can cache all responses and second, third and other attempts will be very fast. LWP::UserAgent::Cached is yet another LWP::UserAgent subclass with cache support. It stores cache in the files on local filesystem and if response already available in the cache returns it instead of making HTTP request. LWP-UserAgent-Cached-0.08/META.json0000644000175000017500000000213113570253100015213 0ustar olegoleg{ "abstract" : "LWP::UserAgent with simple caching mechanism", "author" : [ "Oleg G " ], "dynamic_config" : 1, "generated_by" : "ExtUtils::MakeMaker version 7.1002, CPAN::Meta::Converter version 2.150005", "license" : [ "perl_5" ], "meta-spec" : { "url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec", "version" : "2" }, "name" : "LWP-UserAgent-Cached", "no_index" : { "directory" : [ "t", "inc" ] }, "prereqs" : { "build" : { "requires" : { "File::Temp" : "0", "Test::More" : "0.88" } }, "configure" : { "requires" : { "ExtUtils::MakeMaker" : "6.52" } }, "runtime" : { "requires" : { "LWP::UserAgent" : "0" } } }, "release_status" : "stable", "resources" : { "repository" : { "url" : "https://github.com/olegwtf/p5-LWP-UserAgent-Cached" } }, "version" : "0.08", "x_serialization_backend" : "JSON::PP version 2.27300_01" } LWP-UserAgent-Cached-0.08/MANIFEST0000644000175000017500000000042613570253100014730 0ustar olegolegChanges lib/LWP/UserAgent/Cached.pm Makefile.PL MANIFEST This list of files README t/1_load.t t/2_request.t META.yml Module YAML meta-data (added by MakeMaker) META.json Module JSON meta-data (added by MakeMaker)