WWW-Search-News-1.079/0000755000175600010010000000000011177117277013254 5ustar MartinNoneWWW-Search-News-1.079/Changes0000755000175600010010000000330111007744245014540 0ustar MartinNone2008-05-05 * lib/WWW/Search/WashPost.pm (_parse_tree): fixed for new HTML page format 2007-05-19 * lib/WWW/Search/WashPost.pm (parse_tree): fixed to parse new webpage format 2004-06-05 Kingpin * lib/WWW/Search/WashPost.pm (native_setup_search): fixed for new CGI option value for 'source' 2004-05-22 Kingpin * t/reuters.t: new file * lib/WWW/Search/Reuters.pm: new file * lib/WWW/Search/AP.pm (native_setup_search): all new CGI params * lib/WWW/Search/WashPost.pm (native_setup_search): all new URL and CGI params (parse_tree): all new HTML format 2004-02-20 Kingpin * t/ap.t: make 1-page queries TODO, because they're unreliable * t/washpost.t: make 1-page queries TODO, because they're unreliable 2004-02-07 Kingpin * lib/WWW/Search/WashPost.pm: updated the version number 2003-12-16 Kingpin * lib/WWW/Search/News.pm: new file 2003-12-13 Kingpin * t/washpost.t (my_test): convert to Test::More * lib/WWW/Search/AP.pm (native_setup_search): fix CGI arguments * lib/WWW/Search/WashPost.pm (native_setup_search): fix CGI arguments (parse_tree): fix 'Next' link pattern 2002-12-27 Kingpin * lib/WWW/Search/WashPost.pm (parse_tree): updated for new output format 2002-07-18 Kingpin * lib/WWW/Search/WashPost.pm (parse_tree): remove undef warnings (native_setup_search): use non-robot UserAgent * delete WashTech.pm because website no longer exists * delete t/washtech.t * delete Newsbytes.pm because website no longer exists * delete t/newsbytes.t WWW-Search-News-1.079/inc/0000755000175600010010000000000011177117276014024 5ustar MartinNoneWWW-Search-News-1.079/inc/Module/0000755000175600010010000000000011177117277015252 5ustar MartinNoneWWW-Search-News-1.079/inc/Module/Install/0000755000175600010010000000000011177117277016660 5ustar MartinNoneWWW-Search-News-1.079/inc/Module/Install/Base.pm0000755000175600010010000000212311177117270020062 0ustar MartinNone#line 1 package Module::Install::Base; $VERSION = '0.82'; # Suspend handler for "redefined" warnings BEGIN { my $w = $SIG{__WARN__}; $SIG{__WARN__} = sub { $w }; } ### This is the ONLY module that shouldn't have strict on # use strict; #line 41 sub new { my ($class, %args) = @_; foreach my $method ( qw(call load) ) { next if defined &{"$class\::$method"}; *{"$class\::$method"} = sub { shift()->_top->$method(@_); }; } bless( \%args, $class ); } #line 62 sub AUTOLOAD { my $self = shift; local $@; my $autoload = eval { $self->_top->autoload } or return; goto &$autoload; } #line 79 sub _top { $_[0]->{_top}; } #line 94 sub admin { $_[0]->_top->{admin} or Module::Install::Base::FakeAdmin->new; } #line 110 sub is_admin { $_[0]->admin->VERSION; } sub DESTROY {} package Module::Install::Base::FakeAdmin; my $fake; sub new { $fake ||= bless(\@_, $_[0]); } sub AUTOLOAD {} sub DESTROY {} # Restore warning handler BEGIN { $SIG{__WARN__} = $SIG{__WARN__}->(); } 1; #line 157 WWW-Search-News-1.079/inc/Module/Install/Can.pm0000755000175600010010000000344511177117270017721 0ustar MartinNone#line 1 package Module::Install::Can; use strict; use Module::Install::Base; use Config (); use File::Spec (); use ExtUtils::MakeMaker (); use vars qw{$VERSION $ISCORE @ISA}; BEGIN { $VERSION = '0.82'; $ISCORE = 1; @ISA = qw{Module::Install::Base}; } # check if we can load some module ### Upgrade this to not have to load the module if possible sub can_use { my ($self, $mod, $ver) = @_; $mod =~ s{::|\\}{/}g; $mod .= '.pm' unless $mod =~ /\.pm$/i; my $pkg = $mod; $pkg =~ s{/}{::}g; $pkg =~ s{\.pm$}{}i; local $@; eval { require $mod; $pkg->VERSION($ver || 0); 1 }; } # check if we can run some command sub can_run { my ($self, $cmd) = @_; my $_cmd = $cmd; return $_cmd if (-x $_cmd or $_cmd = MM->maybe_command($_cmd)); for my $dir ((split /$Config::Config{path_sep}/, $ENV{PATH}), '.') { next if $dir eq ''; my $abs = File::Spec->catfile($dir, $_[1]); return $abs if (-x $abs or $abs = MM->maybe_command($abs)); } return; } # can we locate a (the) C compiler sub can_cc { my $self = shift; my @chunks = split(/ /, $Config::Config{cc}) or return; # $Config{cc} may contain args; try to find out the program part while (@chunks) { return $self->can_run("@chunks") || (pop(@chunks), next); } return; } # Fix Cygwin bug on maybe_command(); if ( $^O eq 'cygwin' ) { require ExtUtils::MM_Cygwin; require ExtUtils::MM_Win32; if ( ! defined(&ExtUtils::MM_Cygwin::maybe_command) ) { *ExtUtils::MM_Cygwin::maybe_command = sub { my ($self, $file) = @_; if ($file =~ m{^/cygdrive/}i and ExtUtils::MM_Win32->can('maybe_command')) { ExtUtils::MM_Win32->maybe_command($file); } else { ExtUtils::MM_Unix->maybe_command($file); } } } } 1; __END__ #line 156 WWW-Search-News-1.079/inc/Module/Install/Fetch.pm0000755000175600010010000000476311177117270020255 0ustar MartinNone#line 1 package Module::Install::Fetch; use strict; use Module::Install::Base; use vars qw{$VERSION $ISCORE @ISA}; BEGIN { $VERSION = '0.82'; $ISCORE = 1; @ISA = qw{Module::Install::Base}; } sub get_file { my ($self, %args) = @_; my ($scheme, $host, $path, $file) = $args{url} =~ m|^(\w+)://([^/]+)(.+)/(.+)| or return; if ( $scheme eq 'http' and ! eval { require LWP::Simple; 1 } ) { $args{url} = $args{ftp_url} or (warn("LWP support unavailable!\n"), return); ($scheme, $host, $path, $file) = $args{url} =~ m|^(\w+)://([^/]+)(.+)/(.+)| or return; } $|++; print "Fetching '$file' from $host... "; unless (eval { require Socket; Socket::inet_aton($host) }) { warn "'$host' resolve failed!\n"; return; } return unless $scheme eq 'ftp' or $scheme eq 'http'; require Cwd; my $dir = Cwd::getcwd(); chdir $args{local_dir} or return if exists $args{local_dir}; if (eval { require LWP::Simple; 1 }) { LWP::Simple::mirror($args{url}, $file); } elsif (eval { require Net::FTP; 1 }) { eval { # use Net::FTP to get past firewall my $ftp = Net::FTP->new($host, Passive => 1, Timeout => 600); $ftp->login("anonymous", 'anonymous@example.com'); $ftp->cwd($path); $ftp->binary; $ftp->get($file) or (warn("$!\n"), return); $ftp->quit; } } elsif (my $ftp = $self->can_run('ftp')) { eval { # no Net::FTP, fallback to ftp.exe require FileHandle; my $fh = FileHandle->new; local $SIG{CHLD} = 'IGNORE'; unless ($fh->open("|$ftp -n")) { warn "Couldn't open ftp: $!\n"; chdir $dir; return; } my @dialog = split(/\n/, <<"END_FTP"); open $host user anonymous anonymous\@example.com cd $path binary get $file $file quit END_FTP foreach (@dialog) { $fh->print("$_\n") } $fh->close; } } else { warn "No working 'ftp' program available!\n"; chdir $dir; return; } unless (-f $file) { warn "Fetching failed: $@\n"; chdir $dir; return; } return if exists $args{size} and -s $file != $args{size}; system($args{run}) if exists $args{run}; unlink($file) if $args{remove}; print(((!exists $args{check_for} or -e $args{check_for}) ? "done!" : "failed! ($!)"), "\n"); chdir $dir; return !$?; } 1; WWW-Search-News-1.079/inc/Module/Install/Makefile.pm0000755000175600010010000001641411177117270020735 0ustar MartinNone#line 1 package Module::Install::Makefile; use strict 'vars'; use Module::Install::Base; use ExtUtils::MakeMaker (); use vars qw{$VERSION $ISCORE @ISA}; BEGIN { $VERSION = '0.82'; $ISCORE = 1; @ISA = qw{Module::Install::Base}; } sub Makefile { $_[0] } my %seen = (); sub prompt { shift; # Infinite loop protection my @c = caller(); if ( ++$seen{"$c[1]|$c[2]|$_[0]"} > 3 ) { die "Caught an potential prompt infinite loop ($c[1]|$c[2]|$_[0])"; } # In automated testing, always use defaults if ( $ENV{AUTOMATED_TESTING} and ! $ENV{PERL_MM_USE_DEFAULT} ) { local $ENV{PERL_MM_USE_DEFAULT} = 1; goto &ExtUtils::MakeMaker::prompt; } else { goto &ExtUtils::MakeMaker::prompt; } } sub makemaker_args { my $self = shift; my $args = ( $self->{makemaker_args} ||= {} ); %$args = ( %$args, @_ ); return $args; } # For mm args that take multiple space-seperated args, # append an argument to the current list. sub makemaker_append { my $self = sShift; my $name = shift; my $args = $self->makemaker_args; $args->{name} = defined $args->{$name} ? join( ' ', $args->{name}, @_ ) : join( ' ', @_ ); } sub build_subdirs { my $self = shift; my $subdirs = $self->makemaker_args->{DIR} ||= []; for my $subdir (@_) { push @$subdirs, $subdir; } } sub clean_files { my $self = shift; my $clean = $self->makemaker_args->{clean} ||= {}; %$clean = ( %$clean, FILES => join ' ', grep { length $_ } ($clean->{FILES} || (), @_), ); } sub realclean_files { my $self = shift; my $realclean = $self->makemaker_args->{realclean} ||= {}; %$realclean = ( %$realclean, FILES => join ' ', grep { length $_ } ($realclean->{FILES} || (), @_), ); } sub libs { my $self = shift; my $libs = ref $_[0] ? shift : [ shift ]; $self->makemaker_args( LIBS => $libs ); } sub inc { my $self = shift; $self->makemaker_args( INC => shift ); } my %test_dir = (); sub _wanted_t { /\.t$/ and -f $_ and $test_dir{$File::Find::dir} = 1; } sub tests_recursive { my $self = shift; if ( $self->tests ) { die "tests_recursive will not work if tests are already defined"; } my $dir = shift || 't'; unless ( -d $dir ) { die "tests_recursive dir '$dir' does not exist"; } %test_dir = (); require File::Find; File::Find::find( \&_wanted_t, $dir ); $self->tests( join ' ', map { "$_/*.t" } sort keys %test_dir ); } sub write { my $self = shift; die "&Makefile->write() takes no arguments\n" if @_; # Check the current Perl version my $perl_version = $self->perl_version; if ( $perl_version ) { eval "use $perl_version; 1" or die "ERROR: perl: Version $] is installed, " . "but we need version >= $perl_version"; } # Make sure we have a new enough MakeMaker require ExtUtils::MakeMaker; if ( $perl_version and $self->_cmp($perl_version, '5.006') >= 0 ) { # MakeMaker can complain about module versions that include # an underscore, even though its own version may contain one! # Hence the funny regexp to get rid of it. See RT #35800 # for details. $self->build_requires( 'ExtUtils::MakeMaker' => $ExtUtils::MakeMaker::VERSION =~ /^(\d+\.\d+)/ ); $self->configure_requires( 'ExtUtils::MakeMaker' => $ExtUtils::MakeMaker::VERSION =~ /^(\d+\.\d+)/ ); } else { # Allow legacy-compatibility with 5.005 by depending on the # most recent EU:MM that supported 5.005. $self->build_requires( 'ExtUtils::MakeMaker' => 6.42 ); $self->configure_requires( 'ExtUtils::MakeMaker' => 6.42 ); } # Generate the MakeMaker params my $args = $self->makemaker_args; $args->{DISTNAME} = $self->name; $args->{NAME} = $self->module_name || $self->name; $args->{VERSION} = $self->version; $args->{NAME} =~ s/-/::/g; if ( $self->tests ) { $args->{test} = { TESTS => $self->tests }; } if ( $] >= 5.005 ) { $args->{ABSTRACT} = $self->abstract; $args->{AUTHOR} = $self->author; } if ( eval($ExtUtils::MakeMaker::VERSION) >= 6.10 ) { $args->{NO_META} = 1; } if ( eval($ExtUtils::MakeMaker::VERSION) > 6.17 and $self->sign ) { $args->{SIGN} = 1; } unless ( $self->is_admin ) { delete $args->{SIGN}; } # Merge both kinds of requires into prereq_pm my $prereq = ($args->{PREREQ_PM} ||= {}); %$prereq = ( %$prereq, map { @$_ } map { @$_ } grep $_, ($self->configure_requires, $self->build_requires, $self->requires) ); # Remove any reference to perl, PREREQ_PM doesn't support it delete $args->{PREREQ_PM}->{perl}; # merge both kinds of requires into prereq_pm my $subdirs = ($args->{DIR} ||= []); if ($self->bundles) { foreach my $bundle (@{ $self->bundles }) { my ($file, $dir) = @$bundle; push @$subdirs, $dir if -d $dir; delete $prereq->{$file}; } } if ( my $perl_version = $self->perl_version ) { eval "use $perl_version; 1" or die "ERROR: perl: Version $] is installed, " . "but we need version >= $perl_version"; } $args->{INSTALLDIRS} = $self->installdirs; my %args = map { ( $_ => $args->{$_} ) } grep {defined($args->{$_})} keys %$args; my $user_preop = delete $args{dist}->{PREOP}; if (my $preop = $self->admin->preop($user_preop)) { foreach my $key ( keys %$preop ) { $args{dist}->{$key} = $preop->{$key}; } } my $mm = ExtUtils::MakeMaker::WriteMakefile(%args); $self->fix_up_makefile($mm->{FIRST_MAKEFILE} || 'Makefile'); } sub fix_up_makefile { my $self = shift; my $makefile_name = shift; my $top_class = ref($self->_top) || ''; my $top_version = $self->_top->VERSION || ''; my $preamble = $self->preamble ? "# Preamble by $top_class $top_version\n" . $self->preamble : ''; my $postamble = "# Postamble by $top_class $top_version\n" . ($self->postamble || ''); local *MAKEFILE; open MAKEFILE, "< $makefile_name" or die "fix_up_makefile: Couldn't open $makefile_name: $!"; my $makefile = do { local $/; }; close MAKEFILE or die $!; $makefile =~ s/\b(test_harness\(\$\(TEST_VERBOSE\), )/$1'inc', /; $makefile =~ s/( -I\$\(INST_ARCHLIB\))/ -Iinc$1/g; $makefile =~ s/( "-I\$\(INST_LIB\)")/ "-Iinc"$1/g; $makefile =~ s/^(FULLPERL = .*)/$1 "-Iinc"/m; $makefile =~ s/^(PERL = .*)/$1 "-Iinc"/m; # Module::Install will never be used to build the Core Perl # Sometimes PERL_LIB and PERL_ARCHLIB get written anyway, which breaks # PREFIX/PERL5LIB, and thus, install_share. Blank them if they exist $makefile =~ s/^PERL_LIB = .+/PERL_LIB =/m; #$makefile =~ s/^PERL_ARCHLIB = .+/PERL_ARCHLIB =/m; # Perl 5.005 mentions PERL_LIB explicitly, so we have to remove that as well. $makefile =~ s/(\"?)-I\$\(PERL_LIB\)\1//g; # XXX - This is currently unused; not sure if it breaks other MM-users # $makefile =~ s/^pm_to_blib\s+:\s+/pm_to_blib :: /mg; open MAKEFILE, "> $makefile_name" or die "fix_up_makefile: Couldn't open $makefile_name: $!"; print MAKEFILE "$preamble$makefile$postamble" or die $!; close MAKEFILE or die $!; 1; } sub preamble { my ($self, $text) = @_; $self->{preamble} = $text . $self->{preamble} if defined $text; $self->{preamble}; } sub postamble { my ($self, $text) = @_; $self->{postamble} ||= $self->admin->postamble; $self->{postamble} .= $text if defined $text; $self->{postamble} } 1; __END__ #line 394 WWW-Search-News-1.079/inc/Module/Install/Metadata.pm0000755000175600010010000003474311177117270020745 0ustar MartinNone#line 1 package Module::Install::Metadata; use strict 'vars'; use Module::Install::Base; use vars qw{$VERSION @ISA $ISCORE}; BEGIN { $VERSION = '0.82'; @ISA = qw{Module::Install::Base}; $ISCORE = 1; } my @boolean_keys = qw{ sign mymeta }; my @scalar_keys = qw{ name module_name abstract author version distribution_type tests installdirs }; my @tuple_keys = qw{ configure_requires build_requires requires recommends bundles resources }; my @resource_keys = qw{ homepage bugtracker repository }; my @array_keys = qw{ keywords }; sub Meta { shift } sub Meta_BooleanKeys { @boolean_keys } sub Meta_ScalarKeys { @scalar_keys } sub Meta_TupleKeys { @tuple_keys } sub Meta_ResourceKeys { @resource_keys } sub Meta_ArrayKeys { @array_keys } foreach my $key ( @boolean_keys ) { *$key = sub { my $self = shift; if ( defined wantarray and not @_ ) { return $self->{values}->{$key}; } $self->{values}->{$key} = ( @_ ? $_[0] : 1 ); return $self; }; } foreach my $key ( @scalar_keys ) { *$key = sub { my $self = shift; return $self->{values}->{$key} if defined wantarray and !@_; $self->{values}->{$key} = shift; return $self; }; } foreach my $key ( @array_keys ) { *$key = sub { my $self = shift; return $self->{values}->{$key} if defined wantarray and !@_; $self->{values}->{$key} ||= []; push @{$self->{values}->{$key}}, @_; return $self; }; } foreach my $key ( @resource_keys ) { *$key = sub { my $self = shift; unless ( @_ ) { return () unless $self->{values}->{resources}; return map { $_->[1] } grep { $_->[0] eq $key } @{ $self->{values}->{resources} }; } return $self->{values}->{resources}->{$key} unless @_; my $uri = shift or die( "Did not provide a value to $key()" ); $self->resources( $key => $uri ); return 1; }; } foreach my $key ( grep { $_ ne "resources" } @tuple_keys) { *$key = sub { my $self = shift; return $self->{values}->{$key} unless @_; my @added; while ( @_ ) { my $module = shift or last; my $version = shift || 0; push @added, [ $module, $version ]; } push @{ $self->{values}->{$key} }, @added; return map {@$_} @added; }; } # Resource handling my %lc_resource = map { $_ => 1 } qw{ homepage license bugtracker repository }; sub resources { my $self = shift; while ( @_ ) { my $name = shift or last; my $value = shift or next; if ( $name eq lc $name and ! $lc_resource{$name} ) { die("Unsupported reserved lowercase resource '$name'"); } $self->{values}->{resources} ||= []; push @{ $self->{values}->{resources} }, [ $name, $value ]; } $self->{values}->{resources}; } # Aliases for build_requires that will have alternative # meanings in some future version of META.yml. sub test_requires { shift->build_requires(@_) } sub install_requires { shift->build_requires(@_) } # Aliases for installdirs options sub install_as_core { $_[0]->installdirs('perl') } sub install_as_cpan { $_[0]->installdirs('site') } sub install_as_site { $_[0]->installdirs('site') } sub install_as_vendor { $_[0]->installdirs('vendor') } sub dynamic_config { my $self = shift; unless ( @_ ) { warn "You MUST provide an explicit true/false value to dynamic_config\n"; return $self; } $self->{values}->{dynamic_config} = $_[0] ? 1 : 0; return 1; } sub perl_version { my $self = shift; return $self->{values}->{perl_version} unless @_; my $version = shift or die( "Did not provide a value to perl_version()" ); # Normalize the version $version = $self->_perl_version($version); # We don't support the reall old versions unless ( $version >= 5.005 ) { die "Module::Install only supports 5.005 or newer (use ExtUtils::MakeMaker)\n"; } $self->{values}->{perl_version} = $version; } #Stolen from M::B my %license_urls = ( perl => 'http://dev.perl.org/licenses/', apache => 'http://apache.org/licenses/LICENSE-2.0', artistic => 'http://opensource.org/licenses/artistic-license.php', artistic_2 => 'http://opensource.org/licenses/artistic-license-2.0.php', lgpl => 'http://opensource.org/licenses/lgpl-license.php', lgpl2 => 'http://opensource.org/licenses/lgpl-2.1.php', lgpl3 => 'http://opensource.org/licenses/lgpl-3.0.html', bsd => 'http://opensource.org/licenses/bsd-license.php', gpl => 'http://opensource.org/licenses/gpl-license.php', gpl2 => 'http://opensource.org/licenses/gpl-2.0.php', gpl3 => 'http://opensource.org/licenses/gpl-3.0.html', mit => 'http://opensource.org/licenses/mit-license.php', mozilla => 'http://opensource.org/licenses/mozilla1.1.php', open_source => undef, unrestricted => undef, restrictive => undef, unknown => undef, ); sub license { my $self = shift; return $self->{values}->{license} unless @_; my $license = shift or die( 'Did not provide a value to license()' ); $self->{values}->{license} = $license; # Automatically fill in license URLs if ( $license_urls{$license} ) { $self->resources( license => $license_urls{$license} ); } return 1; } sub all_from { my ( $self, $file ) = @_; unless ( defined($file) ) { my $name = $self->name or die( "all_from called with no args without setting name() first" ); $file = join('/', 'lib', split(/-/, $name)) . '.pm'; $file =~ s{.*/}{} unless -e $file; unless ( -e $file ) { die("all_from cannot find $file from $name"); } } unless ( -f $file ) { die("The path '$file' does not exist, or is not a file"); } # Some methods pull from POD instead of code. # If there is a matching .pod, use that instead my $pod = $file; $pod =~ s/\.pm$/.pod/i; $pod = $file unless -e $pod; # Pull the different values $self->name_from($file) unless $self->name; $self->version_from($file) unless $self->version; $self->perl_version_from($file) unless $self->perl_version; $self->author_from($pod) unless $self->author; $self->license_from($pod) unless $self->license; $self->abstract_from($pod) unless $self->abstract; return 1; } sub provides { my $self = shift; my $provides = ( $self->{values}->{provides} ||= {} ); %$provides = (%$provides, @_) if @_; return $provides; } sub auto_provides { my $self = shift; return $self unless $self->is_admin; unless (-e 'MANIFEST') { warn "Cannot deduce auto_provides without a MANIFEST, skipping\n"; return $self; } # Avoid spurious warnings as we are not checking manifest here. local $SIG{__WARN__} = sub {1}; require ExtUtils::Manifest; local *ExtUtils::Manifest::manicheck = sub { return }; require Module::Build; my $build = Module::Build->new( dist_name => $self->name, dist_version => $self->version, license => $self->license, ); $self->provides( %{ $build->find_dist_packages || {} } ); } sub feature { my $self = shift; my $name = shift; my $features = ( $self->{values}->{features} ||= [] ); my $mods; if ( @_ == 1 and ref( $_[0] ) ) { # The user used ->feature like ->features by passing in the second # argument as a reference. Accomodate for that. $mods = $_[0]; } else { $mods = \@_; } my $count = 0; push @$features, ( $name => [ map { ref($_) ? ( ref($_) eq 'HASH' ) ? %$_ : @$_ : $_ } @$mods ] ); return @$features; } sub features { my $self = shift; while ( my ( $name, $mods ) = splice( @_, 0, 2 ) ) { $self->feature( $name, @$mods ); } return $self->{values}->{features} ? @{ $self->{values}->{features} } : (); } sub no_index { my $self = shift; my $type = shift; push @{ $self->{values}->{no_index}->{$type} }, @_ if $type; return $self->{values}->{no_index}; } sub read { my $self = shift; $self->include_deps( 'YAML::Tiny', 0 ); require YAML::Tiny; my $data = YAML::Tiny::LoadFile('META.yml'); # Call methods explicitly in case user has already set some values. while ( my ( $key, $value ) = each %$data ) { next unless $self->can($key); if ( ref $value eq 'HASH' ) { while ( my ( $module, $version ) = each %$value ) { $self->can($key)->($self, $module => $version ); } } else { $self->can($key)->($self, $value); } } return $self; } sub write { my $self = shift; return $self unless $self->is_admin; $self->admin->write_meta; return $self; } sub version_from { require ExtUtils::MM_Unix; my ( $self, $file ) = @_; $self->version( ExtUtils::MM_Unix->parse_version($file) ); } sub abstract_from { require ExtUtils::MM_Unix; my ( $self, $file ) = @_; $self->abstract( bless( { DISTNAME => $self->name }, 'ExtUtils::MM_Unix' )->parse_abstract($file) ); } # Add both distribution and module name sub name_from { my ($self, $file) = @_; if ( Module::Install::_read($file) =~ m/ ^ \s* package \s* ([\w:]+) \s* ; /ixms ) { my ($name, $module_name) = ($1, $1); $name =~ s{::}{-}g; $self->name($name); unless ( $self->module_name ) { $self->module_name($module_name); } } else { die("Cannot determine name from $file\n"); } } sub perl_version_from { my $self = shift; if ( Module::Install::_read($_[0]) =~ m/ ^ (?:use|require) \s* v? ([\d_\.]+) \s* ; /ixms ) { my $perl_version = $1; $perl_version =~ s{_}{}g; $self->perl_version($perl_version); } else { warn "Cannot determine perl version info from $_[0]\n"; return; } } sub author_from { my $self = shift; my $content = Module::Install::_read($_[0]); if ($content =~ m/ =head \d \s+ (?:authors?)\b \s* ([^\n]*) | =head \d \s+ (?:licen[cs]e|licensing|copyright|legal)\b \s* .*? copyright .*? \d\d\d[\d.]+ \s* (?:\bby\b)? \s* ([^\n]*) /ixms) { my $author = $1 || $2; $author =~ s{E}{<}g; $author =~ s{E}{>}g; $self->author($author); } else { warn "Cannot determine author info from $_[0]\n"; } } sub license_from { my $self = shift; if ( Module::Install::_read($_[0]) =~ m/ ( =head \d \s+ (?:licen[cs]e|licensing|copyright|legal)\b .*? ) (=head\\d.*|=cut.*|) \z /ixms ) { my $license_text = $1; my @phrases = ( 'under the same (?:terms|license) as perl itself' => 'perl', 1, 'GNU general public license' => 'gpl', 1, 'GNU public license' => 'gpl', 1, 'GNU lesser general public license' => 'lgpl', 1, 'GNU lesser public license' => 'lgpl', 1, 'GNU library general public license' => 'lgpl', 1, 'GNU library public license' => 'lgpl', 1, 'BSD license' => 'bsd', 1, 'Artistic license' => 'artistic', 1, 'GPL' => 'gpl', 1, 'LGPL' => 'lgpl', 1, 'BSD' => 'bsd', 1, 'Artistic' => 'artistic', 1, 'MIT' => 'mit', 1, 'proprietary' => 'proprietary', 0, ); while ( my ($pattern, $license, $osi) = splice(@phrases, 0, 3) ) { $pattern =~ s{\s+}{\\s+}g; if ( $license_text =~ /\b$pattern\b/i ) { $self->license($license); return 1; } } } warn "Cannot determine license info from $_[0]\n"; return 'unknown'; } sub _extract_bugtracker { my @links = $_[0] =~ m#L<(\Qhttp://rt.cpan.org/\E[^>]+)>#g; my %links; @links{@links}=(); @links=keys %links; return @links; } sub bugtracker_from { my $self = shift; my $content = Module::Install::_read($_[0]); my @links = _extract_bugtracker($content); unless ( @links ) { warn "Cannot determine bugtracker info from $_[0]\n"; return 0; } if ( @links > 1 ) { warn "Found more than on rt.cpan.org link in $_[0]\n"; return 0; } # Set the bugtracker bugtracker( $links[0] ); return 1; } sub requires_from { my $self = shift; my $content = Module::Install::_readperl($_[0]); my @requires = $content =~ m/^use\s+([^\W\d]\w*(?:::\w+)*)\s+([\d\.]+)/mg; while ( @requires ) { my $module = shift @requires; my $version = shift @requires; $self->requires( $module => $version ); } } # Convert triple-part versions (eg, 5.6.1 or 5.8.9) to # numbers (eg, 5.006001 or 5.008009). # Also, convert double-part versions (eg, 5.8) sub _perl_version { my $v = $_[-1]; $v =~ s/^([1-9])\.([1-9]\d?\d?)$/sprintf("%d.%03d",$1,$2)/e; $v =~ s/^([1-9])\.([1-9]\d?\d?)\.(0|[1-9]\d?\d?)$/sprintf("%d.%03d%03d",$1,$2,$3 || 0)/e; $v =~ s/(\.\d\d\d)000$/$1/; $v =~ s/_.+$//; if ( ref($v) ) { $v = $v + 0; # Numify } return $v; } ###################################################################### # MYMETA.yml Support sub WriteMyMeta { die "WriteMyMeta has been deprecated"; } sub write_mymeta { my $self = shift; # If there's no existing META.yml there is nothing we can do return unless -f 'META.yml'; # We need YAML::Tiny to write the MYMETA.yml file unless ( eval { require YAML::Tiny; 1; } ) { return 1; } # Merge the perl version into the dependencies my $val = $self->Meta->{values}; my $perl = delete $val->{perl_version}; if ( $perl ) { $val->{requires} ||= []; my $requires = $val->{requires}; # Canonize to three-dot version after Perl 5.6 if ( $perl >= 5.006 ) { $perl =~ s{^(\d+)\.(\d\d\d)(\d*)}{join('.', $1, int($2||0), int($3||0))}e } unshift @$requires, [ perl => $perl ]; } # Load the advisory META.yml file my @yaml = YAML::Tiny::LoadFile('META.yml'); my $meta = $yaml[0]; # Overwrite the non-configure dependency hashs delete $meta->{requires}; delete $meta->{build_requires}; delete $meta->{recommends}; if ( exists $val->{requires} ) { $meta->{requires} = { map { @$_ } @{ $val->{requires} } }; } if ( exists $val->{build_requires} ) { $meta->{build_requires} = { map { @$_ } @{ $val->{build_requires} } }; } # Save as the MYMETA.yml file print "Writing MYMETA.yml\n"; YAML::Tiny::DumpFile('MYMETA.yml', $meta); } 1; WWW-Search-News-1.079/inc/Module/Install/Win32.pm0000755000175600010010000000350211177117270020114 0ustar MartinNone#line 1 package Module::Install::Win32; use strict; use Module::Install::Base; use vars qw{$VERSION @ISA $ISCORE}; BEGIN { $VERSION = '0.82'; @ISA = qw{Module::Install::Base}; $ISCORE = 1; } # determine if the user needs nmake, and download it if needed sub check_nmake { my $self = shift; $self->load('can_run'); $self->load('get_file'); require Config; return unless ( $^O eq 'MSWin32' and $Config::Config{make} and $Config::Config{make} =~ /^nmake\b/i and ! $self->can_run('nmake') ); print "The required 'nmake' executable not found, fetching it...\n"; require File::Basename; my $rv = $self->get_file( url => 'http://download.microsoft.com/download/vc15/Patch/1.52/W95/EN-US/Nmake15.exe', ftp_url => 'ftp://ftp.microsoft.com/Softlib/MSLFILES/Nmake15.exe', local_dir => File::Basename::dirname($^X), size => 51928, run => 'Nmake15.exe /o > nul', check_for => 'Nmake.exe', remove => 1, ); die <<'END_MESSAGE' unless $rv; ------------------------------------------------------------------------------- Since you are using Microsoft Windows, you will need the 'nmake' utility before installation. It's available at: http://download.microsoft.com/download/vc15/Patch/1.52/W95/EN-US/Nmake15.exe or ftp://ftp.microsoft.com/Softlib/MSLFILES/Nmake15.exe Please download the file manually, save it to a directory in %PATH% (e.g. C:\WINDOWS\COMMAND\), then launch the MS-DOS command line shell, "cd" to that directory, and run "Nmake15.exe" from there; that will create the 'nmake.exe' file needed by this module. You may then resume the installation process described in README. ------------------------------------------------------------------------------- END_MESSAGE } 1; WWW-Search-News-1.079/inc/Module/Install/WriteAll.pm0000755000175600010010000000206011177117270020733 0ustar MartinNone#line 1 package Module::Install::WriteAll; use strict; use Module::Install::Base; use vars qw{$VERSION @ISA $ISCORE}; BEGIN { $VERSION = '0.82'; @ISA = qw{Module::Install::Base}; $ISCORE = 1; } sub WriteAll { my $self = shift; my %args = ( meta => 1, sign => 0, inline => 0, check_nmake => 1, @_, ); $self->sign(1) if $args{sign}; $self->admin->WriteAll(%args) if $self->is_admin; $self->check_nmake if $args{check_nmake}; unless ( $self->makemaker_args->{PL_FILES} ) { $self->makemaker_args( PL_FILES => {} ); } # Until ExtUtils::MakeMaker support MYMETA.yml, make sure # we clean it up properly ourself. $self->realclean_files('MYMETA.yml'); if ( $args{inline} ) { $self->Inline->write; } else { $self->Makefile->write; } # The Makefile write process adds a couple of dependencies, # so write the META.yml files after the Makefile. $self->Meta->write if $args{meta}; $self->Meta->write_mymeta if $self->mymeta; return 1; } 1; WWW-Search-News-1.079/inc/Module/Install.pm0000755000175600010010000002477411177117270017230 0ustar MartinNone#line 1 package Module::Install; # For any maintainers: # The load order for Module::Install is a bit magic. # It goes something like this... # # IF ( host has Module::Install installed, creating author mode ) { # 1. Makefile.PL calls "use inc::Module::Install" # 2. $INC{inc/Module/Install.pm} set to installed version of inc::Module::Install # 3. The installed version of inc::Module::Install loads # 4. inc::Module::Install calls "require Module::Install" # 5. The ./inc/ version of Module::Install loads # } ELSE { # 1. Makefile.PL calls "use inc::Module::Install" # 2. $INC{inc/Module/Install.pm} set to ./inc/ version of Module::Install # 3. The ./inc/ version of Module::Install loads # } use 5.005; use strict 'vars'; use vars qw{$VERSION $MAIN}; BEGIN { # All Module::Install core packages now require synchronised versions. # This will be used to ensure we don't accidentally load old or # different versions of modules. # This is not enforced yet, but will be some time in the next few # releases once we can make sure it won't clash with custom # Module::Install extensions. $VERSION = '0.82'; # Storage for the pseudo-singleton $MAIN = undef; *inc::Module::Install::VERSION = *VERSION; @inc::Module::Install::ISA = __PACKAGE__; } # Whether or not inc::Module::Install is actually loaded, the # $INC{inc/Module/Install.pm} is what will still get set as long as # the caller loaded module this in the documented manner. # If not set, the caller may NOT have loaded the bundled version, and thus # they may not have a MI version that works with the Makefile.PL. This would # result in false errors or unexpected behaviour. And we don't want that. my $file = join( '/', 'inc', split /::/, __PACKAGE__ ) . '.pm'; unless ( $INC{$file} ) { die <<"END_DIE" } Please invoke ${\__PACKAGE__} with: use inc::${\__PACKAGE__}; not: use ${\__PACKAGE__}; END_DIE # If the script that is loading Module::Install is from the future, # then make will detect this and cause it to re-run over and over # again. This is bad. Rather than taking action to touch it (which # is unreliable on some platforms and requires write permissions) # for now we should catch this and refuse to run. if ( -f $0 ) { my $s = (stat($0))[9]; # If the modification time is only slightly in the future, # sleep briefly to remove the problem. my $a = $s - time; if ( $a > 0 and $a < 5 ) { sleep 5 } # Too far in the future, throw an error. my $t = time; if ( $s > $t ) { die <<"END_DIE" } Your installer $0 has a modification time in the future ($s > $t). This is known to create infinite loops in make. Please correct this, then run $0 again. END_DIE } # Build.PL was formerly supported, but no longer is due to excessive # difficulty in implementing every single feature twice. if ( $0 =~ /Build.PL$/i ) { die <<"END_DIE" } Module::Install no longer supports Build.PL. It was impossible to maintain duel backends, and has been deprecated. Please remove all Build.PL files and only use the Makefile.PL installer. END_DIE # To save some more typing in Module::Install installers, every... # use inc::Module::Install # ...also acts as an implicit use strict. $^H |= strict::bits(qw(refs subs vars)); use Cwd (); use File::Find (); use File::Path (); use FindBin; sub autoload { my $self = shift; my $who = $self->_caller; my $cwd = Cwd::cwd(); my $sym = "${who}::AUTOLOAD"; $sym->{$cwd} = sub { my $pwd = Cwd::cwd(); if ( my $code = $sym->{$pwd} ) { # Delegate back to parent dirs goto &$code unless $cwd eq $pwd; } $$sym =~ /([^:]+)$/ or die "Cannot autoload $who - $sym"; my $method = $1; if ( uc($method) eq $method ) { # Do nothing return; } elsif ( $method =~ /^_/ and $self->can($method) ) { # Dispatch to the root M:I class return $self->$method(@_); } # Dispatch to the appropriate plugin unshift @_, ( $self, $1 ); goto &{$self->can('call')}; }; } sub import { my $class = shift; my $self = $class->new(@_); my $who = $self->_caller; unless ( -f $self->{file} ) { require "$self->{path}/$self->{dispatch}.pm"; File::Path::mkpath("$self->{prefix}/$self->{author}"); $self->{admin} = "$self->{name}::$self->{dispatch}"->new( _top => $self ); $self->{admin}->init; @_ = ($class, _self => $self); goto &{"$self->{name}::import"}; } *{"${who}::AUTOLOAD"} = $self->autoload; $self->preload; # Unregister loader and worker packages so subdirs can use them again delete $INC{"$self->{file}"}; delete $INC{"$self->{path}.pm"}; # Save to the singleton $MAIN = $self; return 1; } sub preload { my $self = shift; unless ( $self->{extensions} ) { $self->load_extensions( "$self->{prefix}/$self->{path}", $self ); } my @exts = @{$self->{extensions}}; unless ( @exts ) { @exts = $self->{admin}->load_all_extensions; } my %seen; foreach my $obj ( @exts ) { while (my ($method, $glob) = each %{ref($obj) . '::'}) { next unless $obj->can($method); next if $method =~ /^_/; next if $method eq uc($method); $seen{$method}++; } } my $who = $self->_caller; foreach my $name ( sort keys %seen ) { *{"${who}::$name"} = sub { ${"${who}::AUTOLOAD"} = "${who}::$name"; goto &{"${who}::AUTOLOAD"}; }; } } sub new { my ($class, %args) = @_; # ignore the prefix on extension modules built from top level. my $base_path = Cwd::abs_path($FindBin::Bin); unless ( Cwd::abs_path(Cwd::cwd()) eq $base_path ) { delete $args{prefix}; } return $args{_self} if $args{_self}; $args{dispatch} ||= 'Admin'; $args{prefix} ||= 'inc'; $args{author} ||= ($^O eq 'VMS' ? '_author' : '.author'); $args{bundle} ||= 'inc/BUNDLES'; $args{base} ||= $base_path; $class =~ s/^\Q$args{prefix}\E:://; $args{name} ||= $class; $args{version} ||= $class->VERSION; unless ( $args{path} ) { $args{path} = $args{name}; $args{path} =~ s!::!/!g; } $args{file} ||= "$args{base}/$args{prefix}/$args{path}.pm"; $args{wrote} = 0; bless( \%args, $class ); } sub call { my ($self, $method) = @_; my $obj = $self->load($method) or return; splice(@_, 0, 2, $obj); goto &{$obj->can($method)}; } sub load { my ($self, $method) = @_; $self->load_extensions( "$self->{prefix}/$self->{path}", $self ) unless $self->{extensions}; foreach my $obj (@{$self->{extensions}}) { return $obj if $obj->can($method); } my $admin = $self->{admin} or die <<"END_DIE"; The '$method' method does not exist in the '$self->{prefix}' path! Please remove the '$self->{prefix}' directory and run $0 again to load it. END_DIE my $obj = $admin->load($method, 1); push @{$self->{extensions}}, $obj; $obj; } sub load_extensions { my ($self, $path, $top) = @_; unless ( grep { ! ref $_ and lc $_ eq lc $self->{prefix} } @INC ) { unshift @INC, $self->{prefix}; } foreach my $rv ( $self->find_extensions($path) ) { my ($file, $pkg) = @{$rv}; next if $self->{pathnames}{$pkg}; local $@; my $new = eval { require $file; $pkg->can('new') }; unless ( $new ) { warn $@ if $@; next; } $self->{pathnames}{$pkg} = delete $INC{$file}; push @{$self->{extensions}}, &{$new}($pkg, _top => $top ); } $self->{extensions} ||= []; } sub find_extensions { my ($self, $path) = @_; my @found; File::Find::find( sub { my $file = $File::Find::name; return unless $file =~ m!^\Q$path\E/(.+)\.pm\Z!is; my $subpath = $1; return if lc($subpath) eq lc($self->{dispatch}); $file = "$self->{path}/$subpath.pm"; my $pkg = "$self->{name}::$subpath"; $pkg =~ s!/!::!g; # If we have a mixed-case package name, assume case has been preserved # correctly. Otherwise, root through the file to locate the case-preserved # version of the package name. if ( $subpath eq lc($subpath) || $subpath eq uc($subpath) ) { my $content = Module::Install::_read($subpath . '.pm'); my $in_pod = 0; foreach ( split //, $content ) { $in_pod = 1 if /^=\w/; $in_pod = 0 if /^=cut/; next if ($in_pod || /^=cut/); # skip pod text next if /^\s*#/; # and comments if ( m/^\s*package\s+($pkg)\s*;/i ) { $pkg = $1; last; } } } push @found, [ $file, $pkg ]; }, $path ) if -d $path; @found; } ##################################################################### # Common Utility Functions sub _caller { my $depth = 0; my $call = caller($depth); while ( $call eq __PACKAGE__ ) { $depth++; $call = caller($depth); } return $call; } sub _read { local *FH; if ( $] >= 5.006 ) { open( FH, '<', $_[0] ) or die "open($_[0]): $!"; } else { open( FH, "< $_[0]" ) or die "open($_[0]): $!"; } my $string = do { local $/; }; close FH or die "close($_[0]): $!"; return $string; } sub _readperl { my $string = Module::Install::_read($_[0]); $string =~ s/(?:\015{1,2}\012|\015|\012)/\n/sg; $string =~ s/(\n)\n*__(?:DATA|END)__\b.*\z/$1/s; $string =~ s/\n\n=\w+.+?\n\n=cut\b.+?\n+/\n\n/sg; return $string; } sub _readpod { my $string = Module::Install::_read($_[0]); $string =~ s/(?:\015{1,2}\012|\015|\012)/\n/sg; return $string if $_[0] =~ /\.pod\z/; $string =~ s/(^|\n=cut\b.+?\n+)[^=\s].+?\n(\n=\w+|\z)/$1$2/sg; $string =~ s/\n*=pod\b[^\n]*\n+/\n\n/sg; $string =~ s/\n*=cut\b[^\n]*\n+/\n\n/sg; $string =~ s/^\n+//s; return $string; } sub _write { local *FH; if ( $] >= 5.006 ) { open( FH, '>', $_[0] ) or die "open($_[0]): $!"; } else { open( FH, "> $_[0]" ) or die "open($_[0]): $!"; } foreach ( 1 .. $#_ ) { print FH $_[$_] or die "print($_[0]): $!"; } close FH or die "close($_[0]): $!"; } # _version is for processing module versions (eg, 1.03_05) not # Perl versions (eg, 5.8.1). sub _version ($) { my $s = shift || 0; my $d =()= $s =~ /(\.)/g; if ( $d >= 2 ) { # Normalise multipart versions $s =~ s/(\.)(\d{1,3})/sprintf("$1%03d",$2)/eg; } $s =~ s/^(\d+)\.?//; my $l = $1 || 0; my @v = map { $_ . '0' x (3 - length $_) } $s =~ /(\d{1,3})\D?/g; $l = $l . '.' . join '', @v if @v; return $l + 0; } sub _cmp ($$) { _version($_[0]) <=> _version($_[1]); } # Cloned from Params::Util::_CLASS sub _CLASS ($) { ( defined $_[0] and ! ref $_[0] and $_[0] =~ m/^[^\W\d]\w*(?:::\w+)*\z/s ) ? $_[0] : undef; } 1; # Copyright 2008 - 2009 Adam Kennedy. WWW-Search-News-1.079/lib/0000755000175600010010000000000011177117277014022 5ustar MartinNoneWWW-Search-News-1.079/lib/WWW/0000755000175600010010000000000011177117277014506 5ustar MartinNoneWWW-Search-News-1.079/lib/WWW/Search/0000755000175600010010000000000011177117277015713 5ustar MartinNoneWWW-Search-News-1.079/lib/WWW/Search/AP.pm0000755000175600010010000000364011177045170016547 0ustar MartinNone # $Id: AP.pm,v 2.108 2009/05/02 13:28:24 Martin Exp $ =head1 NAME WWW::Search::AP - backend for searching AP News at www.washingtonpost.com =head1 SYNOPSIS use WWW::Search; my $oSearch = new WWW::Search('AP'); my $sQuery = WWW::Search::escape_query("japan prime minister"); $oSearch->native_query($sQuery); while (my $oResult = $oSearch->next_result()) { print $oResult->url, "\n"; } =head1 DESCRIPTION This class is a specialization of WWW::Search. It handles making and interpreting searches on AP news stories, courtesy of The Washington Post F. This class exports no public interface; all interaction should be done through L objects. =head1 SEE ALSO To make new back-ends, see L. =head1 NOTES This backend only searches news stories from the last 60 days. =head1 BUGS Please tell the author if you find any! =head1 AUTHOR Martin Thurn =head1 LICENSE Copyright (C) 1998-2009 Martin 'Kingpin' Thurn This software is released under the same license as Perl itself. =cut package WWW::Search::AP; use strict; use warnings; use base 'WWW::Search::WashPost'; use vars qw( $VERSION $MAINTAINER ); $VERSION = do { my @r = (q$Revision: 2.108 $ =~ /\d+/g); sprintf "%d."."%03d" x $#r, @r }; $MAINTAINER = 'Martin Thurn '; # private sub _native_setup_search { my ($self, $sQuery, $rhOptions) = @_; $rhOptions->{'fa_1_sourcenavigator'} = 'AP'; # All further work is done by our superclass, WWW::Search::WashPost: $self->SUPER::_native_setup_search($sQuery, $rhOptions); } # _native_setup_search 1; __END__ http://www.washingtonpost.com/ac2/wp-dyn/NewsSearch?sa=as&sd=&ed=&sb=-1&st=turtle&blt=&fa_1_pagenavigator=&fa_1_sourcenavigator=AP&daterange=0&specificMonth=5&specificDay=18&specificYear=2007&FromRangeMonth=3&FromRangeDay=19&FromRangeYear=2007&ToRangeMonth=5&ToRangeDay=18&ToRangeYear=2007&sb2=1&x=18&y=12 WWW-Search-News-1.079/lib/WWW/Search/News.pm0000755000175600010010000000146011177117262017163 0ustar MartinNone # $rcs = ' $Id: News.pm,v 1.13 2009/05/02 19:27:46 Martin Exp $ ' ; =head1 NAME WWW::Search::News - WWW::Search backend for news searches =head1 DESCRIPTION This is an empty package. It only exists so that CPAN users can say "install WWW::Search::News" and they'll get what they expect. Well, actually I'm not sure what they expect, but they'll get something reasonable. I also use it to control the version number of the distribution. =head1 SEE ALSO WWW::Search::AP WWW::Search::Reuters WWW::Search::WashPost =head1 AUTHOR Martin Thurn =head1 LICENSE Copyright (C) 1998-2009 Martin 'Kingpin' Thurn This software is released under the same license as Perl itself. =cut package WWW::Search::News; use strict; use warnings; use 5.005; our $VERSION = '1.079'; 1; __END__ WWW-Search-News-1.079/lib/WWW/Search/Reuters.pm0000755000175600010010000000365111177045170017702 0ustar MartinNone # $Id: Reuters.pm,v 1.9 2009/05/02 13:28:24 Martin Exp $ =head1 NAME WWW::Search::Reuters - WWW::Search backend for searching Reuters News at www.washingtonpost.com =head1 SYNOPSIS use WWW::Search; my $oSearch = new WWW::Search('Reuters'); my $sQuery = WWW::Search::escape_query("japan prime minister"); $oSearch->native_query($sQuery); while (my $oResult = $oSearch->next_result()) { print $oResult->url, "\n"; } =head1 DESCRIPTION This class is a specialization of WWW::Search. It handles making and interpreting searches on Reuters news on The Washington Post website F. This class exports no public interface; all interaction should be done through L objects. =head1 SEE ALSO To make new back-ends, see L. =head1 NOTES This backend only searches news stories from the last 60 days. =head1 BUGS Please tell the author if you find any! =head1 AUTHOR Martin Thurn =head1 LICENSE Copyright (C) 1998-2009 Martin 'Kingpin' Thurn This software is released under the same license as Perl itself. =cut package WWW::Search::Reuters; use strict; use warnings; use base 'WWW::Search::WashPost'; our $VERSION = do { my @r = (q$Revision: 1.9 $ =~ /\d+/g); sprintf "%d."."%03d" x $#r, @r }; our $MAINTAINER = 'Martin Thurn '; # private sub _native_setup_search { my ($self, $sQuery, $rhOptions) = @_; # http://www.washingtonpost.com/ac2/wp-dyn/NewsSearch?sa=as&sd=&ed=&sb=-1&st=turtle&blt=&fa_1_pagenavigator=&fa_1_sourcenavigator=Reuters&daterange=0&specificMonth=8&specificDay=1&specificYear=2007&FromRangeMonth=6&FromRangeDay=3&FromRangeYear=2007&ToRangeMonth=8&ToRangeDay=1&ToRangeYear=2007&sb2=1&x=17&y=14 $rhOptions->{'fa_1_sourcenavigator'} = 'Reuters'; # All further work is done by our superclass, WWW::Search::WashPost: $self->SUPER::_native_setup_search($sQuery, $rhOptions); } # _native_setup_search 1; __END__ WWW-Search-News-1.079/lib/WWW/Search/WashPost.pm0000755000175600010010000002075311177117175020030 0ustar MartinNone # $Id: WashPost.pm,v 2.83 2009/05/02 19:26:53 Martin Exp $ =head1 NAME WWW::Search::WashPost - WWW::Search backend for searching www.washingtonpost.com =head1 SYNOPSIS use WWW::Search; my $oSearch = new WWW::Search('WashPost'); my $sQuery = WWW::Search::escape_query("japan prime minister"); $oSearch->native_query($sQuery); while (my $oResult = $oSearch->next_result()) { print $oResult->url, "\n"; } =head1 DESCRIPTION This class is a specialization of WWW::Search. It handles making and interpreting searches on news at The Washington Post F. This class exports no public interface; all interaction should be done through L objects. =head1 NOTES In the resulting WWW::SearchResult objects, the following elements will be set (in addition to the usual title, description, etc.): =over =item source Something like "Post" or a certain "Edition". =item location The page number of the print edition, e.g. "E05". =item category The type of item found, either "Article" or "Photo". =item seller The author or photographer. =back =head1 SEE ALSO To make new backends, see L. =head1 CAVEATS This backend (and all its subclasses) only searches news stories from the last 14 days. (This is a restriction on the Washington Post website that we can not avoid.) =head1 BUGS Please tell the author if you find any! =head1 AUTHOR Martin Thurn =head1 LICENSE Copyright (C) 1998-2009 Martin 'Kingpin' Thurn This software is released under the same license as Perl itself. =cut package WWW::Search::WashPost; use strict; use URI; use base 'WWW::Search'; use WWW::SearchResult; our $VERSION = do { my @r = (q$Revision: 2.83 $ =~ /\d+/g); sprintf "%d."."%03d" x $#r, @r }; our $MAINTAINER = 'Martin Thurn '; sub _native_setup_search { my ($self, $sQuery, $native_options_ref) = @_; $self->{_debug} = $native_options_ref->{'search_debug'}; $self->{_debug} = 2 if ($native_options_ref->{'search_parse_debug'}); $self->{_debug} ||= 0; # washingtonpost.com does not let us change this: my $DEFAULT_HITS_PER_PAGE = 10; $self->{'_hits_per_page'} = $DEFAULT_HITS_PER_PAGE; # $self->timeout(120); # use this if website is slow # Use this if website refuses robots: $self->user_agent('non-robot'); # Use this if website mucks up page format depending on browser: $self->{'agent_name'} = 'Mozilla/4.76'; $self->{_next_to_retrieve} = 0; $self->{'_num_hits'} = 0; # This is the result page number: $self->{_washpost_cp} = 1; if (!defined($self->{_options})) { # As of 2004-05-22, URL is http://www.washingtonpost.com/ac2/wp-dyn/Search?tab=article_tab&adv=a&keywords=japan&source=APOnline # As of 2007-05, full URL is http://www.washingtonpost.com/ac2/wp-dyn/NewsSearch?sa=as&sd=&ed=&sb=-1&x=0&y=0&st=treasure&blt=&fa_1_pagenavigator=&fa_1_sourcenavigator="The+Washington+Post"&daterange=0&specificMonth=5&specificDay=18&specificYear=2007&FromRangeMonth=3&FromRangeDay=19&FromRangeYear=2007&ToRangeMonth=5&ToRangeDay=18&ToRangeYear=2007&sb2=1 # As of 2007-05, simplest URL is http://www.washingtonpost.com/ac2/wp-dyn/NewsSearch?sa=as&st=treasure&fa_1_sourcenavigator="The+Washington+Post" $self->{_options} = { 'search_url' => 'http://www.washingtonpost.com/ac2/wp-dyn/NewsSearch', 'sa' => 'as', 'st' => $sQuery, 'fa_1_sourcenavigator' => q'"The+Washington+Post"', }; } # if my $options_ref = $self->{_options}; if (defined($native_options_ref)) { # Copy in new options. foreach (keys %$native_options_ref) { $options_ref->{$_} = $native_options_ref->{$_}; } # foreach } # if # Finally, figure out the url. $self->{_next_url} = $self->{_options}{'search_url'} .'?'. $self->hash_to_cgi_string($options_ref); } # _native_setup_search sub _preprocess_results_page_OFF { my $self = shift; my $sPage = shift; print STDERR $sPage if (2 < $self->{_debug}); return $sPage; } # _preprocess_results_page my $WS = q{[\t\r\n\240\ ]}; sub _parse_tree { my $self = shift; my $oTree = shift; my $hits_found = 0; # Look for the total hit count: my @aoSPANcount = $oTree->look_down( _tag => 'div', class => 'redBlock', ); COUNT_SPAN_TAG: foreach my $oSPAN (@aoSPANcount) { if (ref $oSPAN) { my $sSPAN = $oSPAN->as_text; print STDERR " DDD try SPANcount == $sSPAN\n" if 2 <= $self->{_debug}; if ($sSPAN =~ m!([0-9,]+)$WS+results!i) { my $sCount = $1; print STDERR " DDD raw count == $sCount\n" if 2 <= $self->{_debug}; $sCount =~ s!,!!g; # print STDERR " DDD cooked count == $sCount\n" if 2 <= $self->{_debug}; $self->approximate_result_count($sCount); last COUNT_SPAN_TAG; } # if } # if } # foreach COUNT_SPAN_TAG # $oTree->objectify_text; # Find all the results: my @aoSPAN = $oTree->look_down( _tag => 'div', class => 'resultBlock', ); SPAN_TAG: foreach my $oSPAN (@aoSPAN) { next SPAN_TAG unless ref $oSPAN; print STDERR " DDD try oSPAN ===", $oSPAN->as_HTML, "===\n" if (2 <= $self->{_debug}); my $oDIVheadline = $oSPAN->look_down('_tag' => 'h2', ); next SPAN_TAG unless ref $oDIVheadline; my $oA = $oDIVheadline->look_down('_tag' => 'a'); next SPAN_TAG unless ref $oA; my $sURL = $oA->attr('href'); my $sTitle = _strip($oA->as_text); print STDERR " DDD found , url=$sURL=\n" if (2 <= $self->{_debug}); print STDERR " DDD title=$sTitle=\n" if (2 <= $self->{_debug}); $oA->detach; $oA->delete; my $hit = new WWW::SearchResult; $hit->add_url($sURL); $hit->title($sTitle); my $oDIVdate = $oSPAN->look_down('_tag' => 'p', class => 'kicker'); if (ref($oDIVdate)) { my $s = $oDIVdate->as_text; print STDERR " DDD raw date (kicker) =$s=\n" if 2 <= $self->{_debug}; if ($s =~ m/(.+)\ \|\ (.+)/x) { my $sType = $1; $hit->category($sType); my $sDate = $2; $hit->change_date($sDate); } # if $oDIVdate->detach; $oDIVdate->delete; } # if my $oDIVdesc = $oSPAN->look_down('_tag' => 'p', class => 'teaser'); if (ref($oDIVdesc)) { $hit->description($oDIVdesc->as_text); $oDIVdesc->detach; $oDIVdesc->delete; } # if my $oDIVbyline = $oSPAN->look_down('_tag' => 'p', class => 'stamp', ); my $sByline = ''; if (ref($oDIVbyline)) { $sByline = $oDIVbyline->as_text; } # if else { $sByline = $oSPAN->as_text; } print STDERR " DDD try sByline ==$sByline==\n" if 2 <= $self->{_debug}; if ($sByline =~ m!\((.+)\)!) { $hit->source($1); } # if if ($sByline =~ m/\A(.+), (\S+)/) { $hit->seller($1); $hit->location($2); } if ($sByline =~ m/\b([A-Z]{1,2}[0-9]{1,3})\b/) { $hit->location($1); } push(@{$self->{cache}}, $hit); $self->{'_num_hits'}++; $hits_found++; } # foreach SPAN_TAG # The next-page link uses JavaScript to fill-in and submit the form. # In order to do it mechanically, we need to keep track of what page # we're on and put that number in cp, along with sa=np # Find the next link, if any: my $oDiv = $oTree->look_down('_tag', 'div', class => 'pagination', ); goto SKIP_NEXT_LINK if (! ref $oDiv); my @aoAnext = $oDiv->look_down(_tag => 'a'); my $oAnext = pop @aoAnext; goto SKIP_NEXT_LINK if (! ref($oAnext)); goto SKIP_NEXT_LINK if ($oAnext->as_text ne 'Next>'); $self->{_options}->{sa} = 'np'; $self->{_options}->{cp} = ++$self->{_washpost_cp}; $self->{_next_url} = $self->{_options}{'search_url'} .'?'. $self->hash_to_cgi_string($self->{_options}); SKIP_NEXT_LINK: return $hits_found; } # _parse_tree sub _strip { my $s = shift; $s =~ s!\A$WS+!!x; $s =~ s!$WS+\Z!!x; return $s; } # _strip sub _octalize { my $s = shift; return sprintf "\\%.3lo" x length($s), unpack("C*", $s); } # _octalize 1; __END__ WWW-Search-News-1.079/LICENSE0000755000175600010010000000010211130277772014251 0ustar MartinNone This software is released under the same license as Perl itself. WWW-Search-News-1.079/Makefile.PL0000755000175600010010000000131411177117156015224 0ustar MartinNone # $Id: Makefile.PL,v 1.18 2009/05/02 19:26:38 Martin Exp $ use inc::Module::Install; all_from('lib/WWW/Search/News.pm'); test_requires('Bit::Vector'); test_requires('Test::More'); requires('URI'); # We need the version of WWW::Search that has the agent_email() # method: build_requires('WWW::Search' => 2.534); # We need the version of WWW::Search::Test that exports its tm_ # functions: test_requires('WWW::Search::Test' => 2.265); WriteAll; print STDERR <<'ENDNOTE'; Note: it is very likely that some tests will fail. The news changes so rapidly, it is impossible to keep the test cases completely up-to-date. As long as you get more than zero results, it is safe to install the modules. ENDNOTE __END__ WWW-Search-News-1.079/MANIFEST0000755000175600010010000000073311177117212014400 0ustar MartinNoneChanges inc/Module/Install.pm inc/Module/Install/Base.pm inc/Module/Install/Can.pm inc/Module/Install/Fetch.pm inc/Module/Install/Makefile.pm inc/Module/Install/Metadata.pm inc/Module/Install/Win32.pm inc/Module/Install/WriteAll.pm lib/WWW/Search/AP.pm lib/WWW/Search/News.pm lib/WWW/Search/Reuters.pm lib/WWW/Search/WashPost.pm LICENSE Makefile.PL MANIFEST This list of files META.yml README t/ap.t t/pod-coverage.t t/pod.t t/reuters.t t/washpost.t WWW-Search-News-1.079/META.yml0000755000175600010010000000123611177117271014524 0ustar MartinNone--- abstract: 'WWW::Search backend for news searches' author: - 'Martin Thurn ' build_requires: Bit::Vector: 0 ExtUtils::MakeMaker: 6.42 Test::More: 0 WWW::Search: 2.534 WWW::Search::Test: 2.265 configure_requires: ExtUtils::MakeMaker: 6.42 distribution_type: module generated_by: 'Module::Install version 0.82' license: perl meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html version: 1.4 module_name: WWW::Search::News name: WWW-Search-News no_index: directory: - inc - t requires: URI: 0 perl: 5.005 resources: license: http://dev.perl.org/licenses/ version: 1.079 WWW-Search-News-1.079/README0000755000175600010010000000047310205775437014142 0ustar MartinNone This is a backend for use with the WWW::Search module. Please read the README for WWW::Search for general information. (One place to find it is http://www.perl.com/CPAN-local/modules/by-module/WWW/ ) Read the ChangeLog file in this distribution to see what is new with this backend since the previous release. WWW-Search-News-1.079/t/0000755000175600010010000000000011177117277013517 5ustar MartinNoneWWW-Search-News-1.079/t/ap.t0000755000175600010010000000303310725655074014306 0ustar MartinNone # $Id: ap.t,v 1.11 2007/12/06 02:11:08 Daddy Exp $ use ExtUtils::testlib; use Test::More no_plan; BEGIN { use_ok('WWW::Search') }; BEGIN { use_ok('WWW::Search::Test') }; BEGIN { use_ok('WWW::Search::AP') }; &tm_new_engine('AP'); my $iDebug = 0; my $iDump = 0; # This test returns no results (but we should not get an HTTP error): diag("Sending 0-page AP query to washingtonpost.com..."); &tm_run_test('normal', $WWW::Search::Test::bogus_query, 0, 0, $iDebug); # goto MULTI_RESULT; TODO: { $TODO = q{too hard to find a reliable one-page query}; # This query sometimes (rarely) returns 1 page of results: diag("Sending 1-page AP query to washingtonpost.com..."); $iDebug = 0; $iDump = 0; &tm_run_test('normal', 'turtle', 1, 9, $iDebug, $iDump); $TODO = ''; } # end of TODO block my @ao = $WWW::Search::Test::oSearch->results(); cmp_ok(0, '<', scalar(@ao), 'got any results'); foreach my $oResult (@ao) { like($oResult->url, qr{\Ahttp://}, 'result URL is http'); cmp_ok($oResult->title, 'ne', '', 'result Title is not empty'); cmp_ok($oResult->description, 'ne', '', 'result description is not empty'); cmp_ok($oResult->change_date, 'ne', '', 'result change_date is not empty'); like($oResult->source, qr'AP', 'result source is AP'); } # foreach # goto MULTI_RESULT; MULTI_RESULT: ; diag("Sending multi-page AP query to washingtonpost.com..."); $iDebug = 0; $iDump = 0; # This query returns MANY pages of results: &tm_run_test('normal', 'Japan', 11, undef, $iDebug, $iDump); exit 0; __END__ WWW-Search-News-1.079/t/pod-coverage.t0000755000175600010010000000131711007744615016256 0ustar MartinNone # $Id: pod-coverage.t,v 1.1 2008/05/06 02:59:25 Martin Exp $ use strict; use warnings; use Test::More; use blib; # BEGIN { sub Pod::Coverage::TRACE_ALL () { 1 } } # BEGIN { sub TRACE_ALL () { 1 } } # Ensure a recent version of Test::Pod::Coverage my $min_tpc = 1.08; eval "use Test::Pod::Coverage $min_tpc"; plan skip_all => "Test::Pod::Coverage $min_tpc required for testing POD coverage" if $@; # Test::Pod::Coverage doesn't require a minimum Pod::Coverage version, # but older versions don't recognize some common documentation styles my $min_pc = 0.18; eval "use Pod::Coverage $min_pc"; plan skip_all => "Pod::Coverage $min_pc required for testing POD coverage" if $@; all_pod_coverage_ok(); __END__ WWW-Search-News-1.079/t/pod.t0000755000175600010010000000027510623706440014464 0ustar MartinNone# $Id: pod.t,v 1.1 2007/05/19 23:49:52 Daddy Exp $ use Test::More; eval "use Test::Pod 1.00"; plan skip_all => "Test::Pod 1.00 required for testing POD" if $@; all_pod_files_ok(); __END__ WWW-Search-News-1.079/t/reuters.t0000755000175600010010000000335611130277612015374 0ustar MartinNone # $Id: reuters.t,v 1.9 2009/01/05 03:33:30 Martin Exp $ use blib; use Test::More no_plan; use WWW::Search::Test; BEGIN { use_ok('WWW::Search::Reuters'); } tm_new_engine('Reuters'); my $iDebug = 0; my $iDump = 0; # goto TEST_NOW; # This test returns no results (but we should not get an HTTP error): diag("Sending 0-page Reuters query to washingtonpost.com..."); tm_run_test('normal', $WWW::Search::Test::bogus_query, 0, 0, $iDebug); # goto MULTI_RESULT; TEST_NOW: pass; ONE_PAGE: pass; TODO: { $TODO = q{too hard to find a reliable one-page query}; # This query sometimes (if we're really lucky) returns 1 page of # results: diag("Sending 1-page Reuters query to washingtonpost.com..."); $iDebug = 0; $iDump = 0; tm_run_test('normal', 'frog', 1, 9, $iDebug, $iDump); my @ao = $WWW::Search::Test::oSearch->results(); cmp_ok(0, '<', scalar(@ao), 'got any results'); foreach my $oResult (@ao) { like($oResult->url, qr{\Ahttp://}, 'result URL is http'); cmp_ok($oResult->title, 'ne', '', 'result Title is not empty'); cmp_ok($oResult->description, 'ne', '', 'result description is not empty'); cmp_ok($oResult->change_date, 'ne', '', 'result change_date is not empty'); is($oResult->source, 'Reuters', 'source is Reuters'); } # foreach $TODO = ''; } # end of TODO block # goto SKIP_MULTI_RESULT; MULTI_RESULT: pass; TODO: { # local $TODO = q{www.washingtonpost.com's 'Next' button is broken}; diag("Sending multi-page Reuters query to washingtonpost.com..."); $iDebug = 0; $iDump = 0; # This query returns MANY pages of results: tm_run_test('normal', 'Japan', 11, undef, $iDebug, $iDump); } # end of TODO block SKIP_MULTI_RESULT: pass; exit 0; __END__ WWW-Search-News-1.079/t/washpost.t0000755000175600010010000000531511177117202015547 0ustar MartinNone # $Id: washpost.t,v 1.16 2009/05/02 19:26:58 Martin Exp $ use strict; use warnings; use blib; use Bit::Vector; use Data::Dumper; use Test::More 'no_plan'; use WWW::Search::Test; BEGIN { use_ok('WWW::Search::WashPost'); } tm_new_engine('WashPost'); my $iDebug = 0; my $iDump = 0; # goto DEBUG_NOW; # goto DETAIL_RESULTS; # for debugging # This test returns no results (but we should not get an HTTP error): diag("Sending 0-page query to washingtonpost.com..."); tm_run_test('normal', $WWW::Search::Test::bogus_query, 0, 0, $iDebug); # goto MULTI_RESULT; DEBUG_NOW: pass; DETAIL_RESULTS: pass; TODO: { $TODO = q{too hard to find a reliable one-page query}; diag("Sending 1-page query to washingtonpost.com..."); $iDump = 0; $iDebug = 0; # This query sometimes returns 1 page of results: tm_run_test('normal', 'belle', 1, 9, $iDebug, $iDump); $TODO = ''; } # end of TODO block my @ao = $WWW::Search::Test::oSearch->results(); cmp_ok(0, '<', scalar(@ao), 'got any results'); # We perform this many tests on each result object: my $iTests = 7; my $iAnyFailed = 0; my %hash; my $oV = new Bit::Vector($iTests); foreach my $oResult (@ao) { $oV->Fill; my $iVall = $oV->to_Dec; # print STDERR Dumper($oResult); $oV->Bit_Off(0) if ! like($oResult->url, qr{\Ahttp://}, 'result URL is http'); $oV->Bit_Off(1) if ! isnt($oResult->title, q'', 'result Title is not empty'); $oV->Bit_Off(2) if ! isnt($oResult->description, q'', 'result description is not empty'); $oV->Bit_Off(3) if ! isnt($oResult->change_date, q'', 'result change_date is not empty'); if (0) { # Some articles do not have the writer's name in the search results: $oV->Bit_Off(4) if ! isnt($oResult->seller, q'', 'result seller is not empty'); } # if $oV->Bit_Off(5) if ! isnt($oResult->location || q{}, q'', 'result location is not empty'); $oV->Bit_Off(6) if ! like($oResult->source, qr{(?i:POST|EDITION)}, 'source is Post'); my $iV = $oV->to_Dec; # diag(qq{ DDD iV=$iV, iVall=$iVall}); if ($iV < $iVall) { $hash{$iV} = $oResult; $iAnyFailed++; } # if } # foreach if ($iAnyFailed) { diag(" Here are results that exemplify the failures:"); while (my ($sKey, $sVal) = each %hash) { diag(Dumper($sVal)); } # while } # if # goto ALL_DONE; MULTI_RESULT: diag("Sending multi-page query to washingtonpost.com..."); $iDump = 0; $iDebug = 0; # This query usually returns many of pages of results: tm_run_test('normal', 'Bush', 21, undef, $iDebug, $iDump); SKIP_MULTI_RESULT: pass; ALL_DONE: pass('all done'); exit 0; __END__