XML-TreePuller-0.1.2/000755 000765 000024 00000000000 11370062076 014537 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/Changes000644 000765 000024 00000001305 11370062045 016025 0ustar00tylerstaff000000 000000 Revision history for XML-TreePuller 0.1.2 * Changed over to using constants for array offsets in ::Element * Added ::CookBook::Patterns 0.1.1 April 24, 2010 * Added in XPath support using Tree::XPathEngine * Improved documentation * Added Cookbook * Fleshed out existing documentation 0.1.0 April 22, 2010 * API change: config() becomes iterate_at() * Added more documentation 0.0.3 April 17, 2010 * Added output of example program to documentation * Commented code more 0.0.2 Apr 04, 2010 * Twiddled carp a bit to do better error reporting when constructing XML::LibXML::Reader fails 0.0.1 Feb 21, 2010 * First version, released on an unsuspecting world. XML-TreePuller-0.1.2/lib/000755 000765 000024 00000000000 11370062076 015305 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/Makefile.PL000644 000765 000024 00000001323 11364722424 016513 0ustar00tylerstaff000000 000000 use strict; use warnings; use ExtUtils::MakeMaker; WriteMakefile( NAME => 'XML::TreePuller', AUTHOR => q{Tyler Riddle }, VERSION_FROM => 'lib/XML/TreePuller.pm', ABSTRACT_FROM => 'lib/XML/TreePuller.pm', ($ExtUtils::MakeMaker::VERSION >= 6.3002 ? ('LICENSE'=> 'perl') : ()), PL_FILES => {}, PREREQ_PM => { 'Test::More' => 0, 'XML::LibXML' => 1.70, 'XML::CompactTree' => 0.03, 'Tree::XPathEngine' => 0.05, 'Scalar::Util' => 1.23, }, dist => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', }, clean => { FILES => 'XML-TreePuller-*' }, ); XML-TreePuller-0.1.2/MANIFEST000644 000765 000024 00000001102 11370062076 015662 0ustar00tylerstaff000000 000000 Changes MANIFEST Makefile.PL README TODO lib/XML/TreePuller.pm lib/XML/TreePuller/Element.pm lib/XML/TreePuller/Constants.pm lib/XML/TreePuller/CookBook/Intro.pm lib/XML/TreePuller/CookBook/Performance.pm lib/XML/TreePuller/CookBook/Patterns.pm t/00-load.t t/05-methods.t t/10-simple.t t/50-path.t t/50-short.t t/50-path.t t/60-xpath.t t/data/01-emptyelement.xml t/data/01-emptyelement2.xml t/data/02-singleelement.xml t/data/10-smallelement.xml t/data/50-wikiexample.xml t/data/60-complicated.xml META.yml Module meta-data (added by MakeMaker) XML-TreePuller-0.1.2/META.yml000644 000765 000024 00000001263 11370062076 016012 0ustar00tylerstaff000000 000000 --- #YAML:1.0 name: XML-TreePuller version: 0.1.2 abstract: Pull interface to work with XML document fragments author: - Tyler Riddle license: perl distribution_type: module configure_requires: ExtUtils::MakeMaker: 0 build_requires: ExtUtils::MakeMaker: 0 requires: Scalar::Util: 1.23 Test::More: 0 Tree::XPathEngine: 0.05 XML::CompactTree: 0.03 XML::LibXML: 1.7 no_index: directory: - t - inc generated_by: ExtUtils::MakeMaker version 6.56 meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html version: 1.4 XML-TreePuller-0.1.2/README000644 000765 000024 00000006341 11340000270 015404 0ustar00tylerstaff000000 000000 XML-TreePuller INSTALLATION To install this module, run the following commands: perl Makefile.PL make make test make install ABOUT This module implements a tree oriented XML pull processor using a combination of XML::LibXML::Reader and an object-oriented interface around the output of XML::CompactTree. It provides a fast and convenient way to access the content of extremely large XML documents serially. EXAMPLE #!/usr/bin/env perl use strict; use warnings; use XML::TreePuller; sub gen_xml { return < ExamplePedia http://example.pedia/ Special Talk A good article Some good content A bad article Some bad content EOF } sub element_example { my $xml = XML::TreePuller->new(string => gen_xml()); print "Printing namespace names using configuration style:\n"; $xml->config('/wiki/siteinfo/namespaces/namespace' => 'short'); while(defined(my $element = $xml->next)) { print $element->attribute('key'), ": ", $element->text, "\n"; } print "End of namespace names\n"; } sub subtree_example { my $xml = XML::TreePuller->new(string => gen_xml()); print "Printing titles using a subtree:\n"; $xml->config('/wiki/page' => 'subtree'); while(defined(my $element = $xml->next)) { print "Title: ", $element->get_elements('title')->text, "\n"; } print "End of titles\n"; } sub path_example { my $xml = XML::TreePuller->new(string => gen_xml()); print "Printing path example:\n"; $xml->config('/wiki/siteinfo', 'subtree'); $xml->config('/wiki/page/title', 'short'); while(my ($matched_path, $element) = $xml->next) { print "Path: $matched_path\n"; } print "End path example\n"; } element_example(); print "\n"; subtree_example(); print "\n"; path_example(); print "\n"; __END__ Printing namespace names using configuration style: -1: Special 0: 1: Talk End of namespace names Printing titles using a subtree: Title: A good article Title: A bad article End of titles Printing path example: Path: /wiki/siteinfo Path: /wiki/page/title Path: /wiki/page/title End path example SUPPORT AND DOCUMENTATION After installing, you can find documentation for this module with the perldoc command. perldoc XML::TreePuller You can also look for information at: RT, CPAN's request tracker http://rt.cpan.org/NoAuth/Bugs.html?Dist=XML-TreePuller AnnoCPAN, Annotated CPAN documentation http://annocpan.org/dist/XML-TreePuller CPAN Ratings http://cpanratings.perl.org/d/XML-TreePuller Search CPAN http://search.cpan.org/dist/XML-TreePuller/ COPYRIGHT AND LICENCE Copyright (C) 2010 "Tyler Riddle" This program is free software; you can redistribute it and/or modify it under the terms of either: the GNU General Public License as published by the Free Software Foundation; or the Artistic License. See http://dev.perl.org/licenses/ for more information. XML-TreePuller-0.1.2/t/000755 000765 000024 00000000000 11370062076 015002 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/TODO000644 000765 000024 00000000762 11366577776 015263 0ustar00tylerstaff000000 000000 * Add more recipes to the cook book * XPath tutorial and recipes * Add an 'attributes' instruction that reads only the next element and it's attributes * Improve speed of the module through XS * Validate the configuration prior to starting processing so errors in there don't show up in the middle of a long job * Support using a regex to match a path in iterate_at() * Make XPath optional supporting not having Tree::XPathEngine installed for the rest of the module to workXML-TreePuller-0.1.2/t/00-load.t000644 000765 000024 00000000234 11340237675 016330 0ustar00tylerstaff000000 000000 #!perl use Test::More tests => 1; BEGIN { use_ok( 'XML::TreePuller' ); } diag( "Testing XML::TreePuller $XML::TreePuller::VERSION, Perl $], $^X" ); XML-TreePuller-0.1.2/t/05-methods.t000644 000765 000024 00000002742 11363757167 017077 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl use Test::More tests => 26; use strict; use warnings; use XML::TreePuller; my $puller = new_puller(); ok(defined($puller->reader)); ok(ref($puller->reader) eq 'XML::LibXML::Reader'); my $element = $puller->next; ok(defined($element)); ok(ref($element) eq 'XML::TreePuller::Element'); ok($element->name eq 'element'); ok($element->text eq 'barbiddlemore biddle'); ok(ref($element->attribute) eq 'HASH'); ok($element->attribute->{one} eq '1'); ok(defined($element->attribute('one'))); ok($element->attribute('one') eq '1'); ok(! defined($element->attribute('bogus'))); my @results = $element->get_elements('baz'); ok(scalar(@results) == 2); foreach (@results) { ok(ref($_) eq 'XML::TreePuller::Element'); } ok($results[0]->text eq 'biddle'); ok($results[1]->text eq 'more biddle'); ok($element->get_elements() != $element); $element = $element->get_elements('baz'); ok(defined($element)); ok(ref($element) eq 'XML::TreePuller::Element'); ok($element->text eq 'biddle'); #double check the next() method for the array interface #since it did not get tested above $puller = new_puller(); undef($element); my $path; ($path, $element) = $puller->next; ok($path eq '/element'); ok(ref($element) eq 'XML::TreePuller::Element'); ok($element->name eq 'element'); ok(! defined($puller->next)); sub new_puller { my $puller = XML::TreePuller->new(location => 't/data/10-smallelement.xml'); ok(defined($puller)); $puller->iterate_at('/element' => 'subtree'); return $puller; }XML-TreePuller-0.1.2/t/10-simple.t000644 000765 000024 00000001163 11364075016 016700 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl use Test::More tests => 9; use strict; use warnings; use XML::TreePuller; my $puller = XML::TreePuller->new(location => 't/data/10-smallelement.xml'); ok(defined($puller)); $puller->iterate_at('/element' => 'subtree'); my $element = $puller->next; ok(defined($element) && ref($element) eq 'XML::TreePuller::Element'); ok($element->attribute('one') eq '1'); ok($element->attribute('two') eq '2'); my $foo = $element->get_elements('foo'); my $baz = $element->get_elements('baz'); ok(defined($foo)); ok(defined($baz)); ok($foo->text eq 'bar'); ok($baz->text eq 'biddle'); ok(! defined($puller->next));XML-TreePuller-0.1.2/t/50-path.t000644 000765 000024 00000001123 11363757222 016350 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl use Test::More tests => 9; use strict; use warnings; use XML::TreePuller; my $xml = XML::TreePuller->new(location => 't/data/50-wikiexample.xml'); $xml->iterate_at('/wiki', 'short'); $xml->iterate_at('/wiki/siteinfo', 'subtree'); $xml->iterate_at('/wiki/page/title', 'short'); my @results; while(my ($path, $e) = $xml->next) { push(@results, $path); ok(ref($e) eq 'XML::TreePuller::Element'); } ok($results[0] eq '/wiki'); ok($results[1] eq '/wiki/siteinfo'); ok($results[2] eq '/wiki/page/title'); ok($results[3] eq '/wiki/page/title'); ok(! defined($results[4]));XML-TreePuller-0.1.2/t/50-short.t000644 000765 000024 00000001123 11363757212 016552 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl use Test::More tests => 9; use strict; use warnings; use XML::TreePuller; my $xml = XML::TreePuller->new(location => 't/data/50-wikiexample.xml'); my @results; $xml->iterate_at('/wiki/siteinfo/namespaces/namespace' => 'short'); while(defined(my $element = $xml->next)) { ok(ref($element) eq 'XML::TreePuller::Element'); push(@results, $element); } ok($results[0]->text eq 'Special'); ok($results[1]->text eq ''); ok($results[2]->text eq 'Talk'); ok($results[0]->attribute('key') == -1); ok($results[1]->attribute('key') == 0); ok($results[2]->attribute('key') == 1);XML-TreePuller-0.1.2/t/60-xpath.t000644 000765 000024 00000000641 11364633433 016543 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl #there needs to be more tests, a whole #xpath validation suite would be nice use Test::More tests => 3; use strict; use warnings; use Data::Dumper; use XML::TreePuller; my $root = XML::TreePuller->parse(location => 't/data/60-complicated.xml'); my @a; ok(defined($root)); @a = $root->xpath('/complex'); ok(scalar(@a) == 1); ok(scalar(depth($a[0])) == 0); sub depth { return $_[0]->[7]; }XML-TreePuller-0.1.2/t/data/000755 000765 000024 00000000000 11370062076 015713 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/t/data/01-emptyelement.xml000644 000765 000024 00000000023 11331426557 021363 0ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/t/data/01-emptyelement2.xml000644 000765 000024 00000000013 11331426741 021437 0ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/t/data/02-singleelement.xml000644 000765 000024 00000000042 11331435226 021501 0ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/t/data/10-smallelement.xml000644 000765 000024 00000000206 11340230777 021335 0ustar00tylerstaff000000 000000 bar biddle more biddle XML-TreePuller-0.1.2/t/data/50-wikiexample.xml000644 000765 000024 00000001014 11340233554 021170 0ustar00tylerstaff000000 000000 ExamplePedia http://example.pedia/ Special Talk A good article Some good content A bad article Some bad content XML-TreePuller-0.1.2/t/data/60-complicated.xml000644 000765 000024 00000000320 11364620440 021135 0ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/lib/XML/000755 000765 000024 00000000000 11370062076 015745 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/lib/XML/TreePuller/000755 000765 000024 00000000000 11370062076 020030 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/lib/XML/TreePuller.pm000644 000765 000024 00000037447 11370061736 020407 0ustar00tylerstaff000000 000000 package XML::TreePuller; our $VERSION = '0.1.2'; use strict; use warnings; use Data::Dumper; use Carp qw(croak carp); use XML::LibXML::Reader; use XML::TreePuller::Element; use XML::TreePuller::Constants; our $NO_XS; BEGIN { if (! defined(eval { require XML::CompactTree::XS; })) { $NO_XS = 1; require XML::CompactTree; } } sub new { my ($class, @args) = @_; my $self = {}; my $reader; bless($self, $class); $self->{elements} = []; $self->{config} = {}; $self->{finished} = 0; $Carp::CarpLevel++; $reader = $self->{reader} = XML::LibXML::Reader->new(@args); $Carp::CarpLevel--; #arg how do you get error messages out of libxml reader? croak("could not construct libxml reader") unless defined $reader; return $self; } sub parse { my ($class, @args) = @_; return $class->new(@args)->next; } sub iterate_at { my ($self, $path, $todo) = @_; croak("must specify match and instruction") unless defined $path && defined $todo; $self->{config}->{$path} = $todo; return undef; } sub config { #turn this warning on later #carp "config() is depreciated, use iterate_at() instead"; return iterate_at(@_); } sub next { my ($self) = @_; my $reader = $self->{reader}; my $elements = $self->{elements}; my $config = $self->{config}; my $ret; return () if $self->{finished}; if ($reader->nodeType != XML_READER_TYPE_ELEMENT) { if (! $self->_find_next_element) { #no more elements available in the document return (); } } #the reader came in already sitting on an element so we have to #iterate at the end of the loop do { my $path; my $todo; my $ret; if(! $self->_sync) { #ran out of data in the document return (); } push(@$elements, $reader->name); $path = '/' . join('/', @$elements); #handle the default case where no config is specified if (scalar(keys(%$config)) == 0) { $self->{finished} = 1; if (wantarray()) { return($path, $self->_read_subtree); } return $self->_read_subtree; } #if this is converted over a dispatch hash then #the keys in the hash can be used to validate items #as they are passed to next() and allow this #method to scale to more instructions if (defined($todo = $config->{$path})) { if ($todo eq 'short') { $ret = $self->_read_element; } elsif ($todo eq 'subtree') { $ret = $self->_read_subtree; } else { die "invalid todo specified: $todo"; } if (wantarray()) { return($path, $ret); } return $ret; } } while ($self->_find_next_element); return (); } sub reader { return $_[0]->{reader}; } #private methods #get the reader to a point where it is in sync with #our internal element list sub _sync { my ($self) = @_; my $reader = $self->{reader}; my $depth = $self->{reader}->depth; my $elements = $self->{elements}; #if we are at a higher level than we have #tracked to we need to get back to the same #depth as our element list to properly process #data again while(scalar(@$elements) < $reader->depth) { my $ret = $reader->nextElement; if ($ret == -1) { die "libxml read error"; } elsif ($ret == 0) { $self->{finished} = 1; return 0; } } #handle the case where the reader is at a lower #depth than we have tracked to splice(@$elements, $reader->depth); return 1; } sub _find_next_element { my ($self) = @_; my $reader = $self->{reader}; my $ret; if (! ($ret = $reader->nextElement)) { $self->{finished} = 1; return 0; } elsif ($ret == -1) { die "libxml read error"; } return 1; } sub _read_subtree { my ($self) = @_; my $reader = $self->{reader}; my $elements = $self->{elements}; my $tree = XML::TreePuller::Element->new(_read_tree($reader)); if (! defined($tree)) { $self->{finished} = 1; return undef; } return $tree; } sub _read_element { my ($self) = @_; my $reader = $self->{reader}; my $is_empty = $reader->isEmptyElement; my $new; my %attr; my $node_type; my $ret; $new->[XML_TREEPULLER_ELEMENT_TYPE] = 1; $new->[XML_TREEPULLER_ELEMENT_NAME] = $reader->name; $new->[XML_TREEPULLER_ELEMENT_NAMESPACE] = 0; $new->[XML_TREEPULLER_ELEMENT_ATTRIBUTES] = \%attr; $new->[XML_TREEPULLER_ELEMENT_CHILDREN] = []; if ($reader->hasAttributes && $reader->moveToFirstAttribute == 1) { do { my $name = $reader->name; my $val = $reader->value; $attr{$name} = $val; } while($reader->moveToNextAttribute == 1); } $ret = $reader->read; if ($ret == -1) { die "libxml read error"; } elsif ($ret == 0) { return undef; } if ($is_empty) { return XML::TreePuller::Element->new($new); } $node_type = $reader->nodeType; while($node_type != XML_READER_TYPE_ELEMENT && $node_type != XML_READER_TYPE_END_ELEMENT) { $node_type = $reader->nodeType; if ($node_type == XML_READER_TYPE_TEXT || $node_type == XML_READER_TYPE_CDATA) { push(@{$new->[XML_TREEPULLER_ELEMENT_CHILDREN]}, [ $node_type, $reader->value ]); } $ret = $reader->read; if ($ret == -1) { die "libxml read error"; } elsif ($ret == 0) { return undef; } $node_type = $reader->nodeType; } return XML::TreePuller::Element->new($new); } sub _read_tree { my ($r) = @_; if ($NO_XS) { return XML::CompactTree::readSubtreeToPerl($r, 0); } return XML::CompactTree::XS::readSubtreeToPerl($r, 0); } 1; __END__ =head1 NAME XML::TreePuller - Pull interface to work with XML document fragments =head1 SYNOPSIS use XML::TreePuller; $pull = XML::TreePuller->new(location => '/what/ever/filename.xml'); $pull = XML::TreePuller->new(location => 'http://urls.too/data.xml'); $pull = XML::TreePuller->new(IO => \*FH); $pull = XML::TreePuller->new(string => ''); #parse the document and return the root element #takes same arguments as new() $element = XML::TreePuller->parse(%ARGS); $pull->reader; #return the XML::LibXML::Reader object $pull->iterate_at('/xml', 'short'); #read the first part of an element $pull->iterate_at('/xml', 'subtree'); #read the element and subtree while($element = $pull->next) { } $element->name; $element->text; #fetch text for the element and all children $element->attribute('attribute_name'); #get attribute value $element->attribute; #returns hashref of attributes $element->get_elements; #return all child elements $element->get_elements('element/path'); #elements from path $element->xpath('/xml'); #search using a XPath =head1 ABOUT This module implements a tree oriented XML pull processor providing fast and convenient unmarshalling of extremely large XML documents serially. Unmarshalling means the module is intended to turn the XML document into datastructures, not transform it. Tree oriented means the data is returned from the engine as a tree of data replicating the structure of the original XML document. Pull processor means you sequentially ask the engine for more data (the opposite of SAX). This engine also supports breaking the document into fragments so the trees are small enough to fit into RAM. =head2 Features =over 4 =item High speed This framework has been benchmarked to process XML between 1 meg/sec and 70 meg/sec in real world scenarios using the high level interface. =item Work with documents too big to fit into RAM The interface is nearly identical for large documents and small documents. =item High level The document is mapped to a high level XML element class that is easy to use. =item Low level If you need lower level access to the XML document you can treat the element class as a set of arrays representing the structure of your document or you can work with the XML::LibXML::Reader instance directly. =back =head2 Justification "Another XML processing scheme? Why don't you create a new template parsing framework to go with it!?" -- If I had a trillion dollars for every time I've heard this I could bail out the US Government (as of Apr 26, 2010 that is). When I set out to create the replacement for Parse::MediaWikiDump I started by benchmarking the performance of existing XML processing frameworks (XML::SAX (all of them), XML::Parser, and higher level frameworks such as XML::Twig). The results of my research was that there exists no very fast pull oriented high level framework for processing XML. I set about building MediaWiki::DumpFile using a base of XML::LibXML::Reader and XML::CompactTree; I wound up with a reconfigurable XML processing engine that I rather liked so I decided to publish it on CPAN. =head1 STATUS This software is currently ALPHA quality - the only known use is MediaWiki::DumpFile which is itself becoming tested in production. The API is not stable and there may be bugs: please report success and failure to the author below. =head1 XML::TreePuller =head2 METHODS =over 4 =item new The constructor for this class returns an instance of itself; all arguments are passed straight on to XML::LibXML::Reader when it is constructed. See the documentation for a full specification of what you can use but for quick reference: =over 4 =item new(location => '/what/ever/filename.xml'); =item new(location => 'http://urls.work.too/data.xml'); =item new(string => $xml_data); =item new(IO => \*FH); =back =item parse This method takes the same arguments as new() but parses the entire document into an element and returns it; you can use this if you don't need to break the document into chunks. =item iterate_at This method allows you to control the configuration of the processing engine; you specify two arguments: a path to an XML element and an instruction. The engine will move along node by node through the document and keep track of the full path to the current element. The combination of the current path of the XML document in the reader and the instruction to use will cause instances of XML::TreePuller::Element to be available from the "next" method. If iterate_at() is never called then the entire document will be read into a single element at the first invocation of next(). =over 4 =item iterate_at('/path/to/element' => 'short'); When the path of the current XML element matches the path specified the "next" method will return an instance of XML::TreePuller::Element that holds any attributes and will contain textual data up to the start of another element; there will be no child elements in this element. =item iterate_at('/ditto' => 'subtree'); When the path of the current XML element matches the path specified the "next" method will return an instance of XML::TreePuller::Element that holds the attributes for the element and all of the element textual data and child elements. =back =item next This method is the iterator for the processing system. Each time an instruction is matched it will return an instance of XML::TreePuller::Element. When called in scalar context returns a reference to the next available element or undef when no more data is available. When called in list context it returns a two item list with the first item being the path to the node that was matched and the second item being the next available element; returns an empty list when there is no more data to be processed. The returned path will always be a full path in the document starting at the root element and ending in the element that ultimately matched. =item reader Returns the instance of XML::LibXML::Reader that we are using to parse the XML document. You can move the cursor of the reader if you want but keep this in mind: if you move the cursor of the reader to an element in the document that is at a higher level than the reader was sitting at when you moved it then the reader must move the cursor to an element that was at the same depth in the document as it was at the start; this may cause some parts of the document to be thrown out that you are not expecting. =back =head1 XML::TreePuller::Element This class is how you access the data from XML::TreePuller. XML::TreePuller::Element is implemented as a set of methods that operate on arrays as returned by XML::CompactTree; you are free to work with XML::TreePuller::Element objects just as you would work with data returned from XML::CompactTree::readSubtreeToPerl() and such. =head2 METHODS =over 4 =item name Returns the name of the element as a string =item text Returns the text stored in the element and all subelements as a string; returns an empty string if there is no text =item attribute If called with out any arguments returns a hash reference containing the attribute names as keys and the attribute values as the data. If called with an argument returns the value for the attribute by that name or undef if there is no attribute by that name. =item get_elements Searches this element for any child elements as matched by the path supplied as an argument; the path is relative to the current element. The path is of the format 'element1/element2/element3' where each element name is seperated by a forward slash and there is no trailing or leading forwardslashes. If no path is specified it returns all of the child elements for the current element. If called in scalar context returns the first element that matches the path; if called in array context returns a list of all elements that matched. =item xpath Perform an XPath query on the element and return the results; if called in list context you'll get all of the elements that matched. If called in scalar context you'll get the first element that matched. XPath support is currently EXPERIMENTAL. The XPath query is rooted at the element so you must include the current element name as part of the path if you are specifying an absolute path to a subelement. =back =head1 IMPROVING PERFORMANCE First of all if you want to improve the throughput of this XML processing system be sure to install XML::CompactTree::XS - once installed this module is used automatically and drastically improves overall performance of unmarshalling the XML from the document (this does not involve XML::TreePuller::Element). Secondly there are a number of ways to solve problems with this module, see XML::TreePuller::CookBook::Performance for information. =head1 FURTHER READING =over 4 =item XML::TreePuller::CookBook::Intro Gentle introduction to parsing using Atom as an example. =item XML::TreePuller::CookBook::Performance High performance processing of Wikipedia dump files. =item XML::TreePuller::CookBook::Patterns =item XPath Tutorial =over 4 =item http://www.zvon.org/xxl/XPathTutorial/Output/example1.html =item http://www.w3schools.com/xpath/ =back =item MediaWiki::DumpFile::Pages Object oriented recursive descent parser that maps Mediawiki XML dump files into high level Perl objects for working with the data. =back =head1 LIMITATIONS =over 4 =item This module is not XML compliant though it is built from XML compliant components. There may be unexpected behavior compared to proper XML behavior and if this is encountered please open a bug report. =item XPath support is EXPERIMENTAL (even more so than the rest of this module) =item There is only support for elements, text in elements, and CDATA blocks - other features of XML are not part of the API and are not tested but may bleed through from the underlying modules used to build this system. If you have an idea on how to add support for these extra features the author is soliciting feedback and patches. =item Things are pretty arbitrary right now as this module started life as the heart of MediaWiki::DumpFile; it would be nice to bring in more formal XML processing concepts. =back =head1 ATTRIBUTION With out the following people this module would not be possible: =over 4 =item Andrew Rodland My Perl mentor and friend, his influence has helped me everywhere. =item Petr Pajas As the maintainer of XML::LibXML and creator of XML::CompactTree this module would not be possible with out building on his great work. =item Michel Rodriguez For creating Tree::XPathEngine which made adding XPath support a one day exercise. =back =head1 AUTHOR Tyler Riddle, C<< >>XML-TreePuller-0.1.2/lib/XML/TreePuller/Constants.pm000644 000765 000024 00000002001 11366601634 022337 0ustar00tylerstaff000000 000000 package XML::TreePuller::Constants; our $VERSION = '0.1.7'; use base qw(Exporter); our @EXPORT = qw( XML_TREEPULLER_ELEMENT_TYPE XML_TREEPULLER_ELEMENT_NAME XML_TREEPULLER_ELEMENT_NAMESPACE XML_TREEPULLER_ELEMENT_ATTRIBUTES XML_TREEPULLER_ELEMENT_CHILDREN XML_TREEPULLER_ELEMENT_PARENT XML_TREEPULLER_ELEMENT_ROOT XML_TREEPULLER_ELEMENT_DEPTH XML_TREEPULLER_ELEMENT_NEXT_SIBLING XML_TREEPULLER_ELEMENT_PREV_SIBLING XML_TREEPULLER_ELEMENT_XPATH_ENGINE ); use constant XML_TREEPULLER_ELEMENT_TYPE => 0; use constant XML_TREEPULLER_ELEMENT_NAME => 1; use constant XML_TREEPULLER_ELEMENT_NAMESPACE => 2; use constant XML_TREEPULLER_ELEMENT_ATTRIBUTES => 3; use constant XML_TREEPULLER_ELEMENT_CHILDREN => 4; use constant XML_TREEPULLER_ELEMENT_PARENT => 5; use constant XML_TREEPULLER_ELEMENT_ROOT => 6; use constant XML_TREEPULLER_ELEMENT_DEPTH => 7; use constant XML_TREEPULLER_ELEMENT_NEXT_SIBLING => 8; use constant XML_TREEPULLER_ELEMENT_PREV_SIBLING => 9; use constant XML_TREEPULLER_ELEMENT_XPATH_ENGINE => 10; 1;XML-TreePuller-0.1.2/lib/XML/TreePuller/CookBook/000755 000765 000024 00000000000 11370062076 021536 5ustar00tylerstaff000000 000000 XML-TreePuller-0.1.2/lib/XML/TreePuller/Element.pm000644 000765 000024 00000014102 11366602610 021754 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl #things got ugly in here when XPath #support was hammered in - should be cleaned up package XML::TreePuller::Element; our $VERSION = '0.1.0'; use strict; use warnings; use Carp qw(croak); use XML::LibXML::Reader; use Tree::XPathEngine::Number; use Data::Dumper; use Scalar::Util qw(weaken); use Tree::XPathEngine; use XML::TreePuller::Constants; sub new { my ($class, $tree) = @_; if ($tree->[XML_TREEPULLER_ELEMENT_TYPE] != XML_READER_TYPE_ELEMENT) { croak("must specify an element node"); } bless($tree, $class); $tree->[XML_TREEPULLER_ELEMENT_XPATH_ENGINE] = Tree::XPathEngine->new; $tree->_init($tree, 0); return $tree; } sub get_elements { my ($self, $path) = @_; my @results; if (! defined($path)) { @results = _extract_elements(@{$self->[XML_TREEPULLER_ELEMENT_CHILDREN]}); } else { @results = $self->_recursive_get_child_elements(split('/', $path)); } if (wantarray()) { return @results; } return shift(@results); } sub xpath { my @return = $_[0]->[XML_TREEPULLER_ELEMENT_XPATH_ENGINE]->findnodes($_[1], XML::TreePuller::Element::Document->new($_[0])); if (wantarray()) { return @return; } return shift(@return); } sub name { my ($tree) = @_; return $tree->[XML_TREEPULLER_ELEMENT_NAME]; } sub text { my ($self) = @_; my @content; foreach (@{$self->[XML_TREEPULLER_ELEMENT_CHILDREN]}) { if ($_->[XML_TREEPULLER_ELEMENT_TYPE] == XML_READER_TYPE_TEXT || $_->[0] == XML_READER_TYPE_CDATA) { push(@content, $_->[XML_TREEPULLER_ELEMENT_NAME]); } elsif ($_->[XML_TREEPULLER_ELEMENT_TYPE] == XML_READER_TYPE_ELEMENT) { push(@content, $_->text); } } return join('', @content); } sub attribute { my ($tree, $name) = @_; my $attr = $tree->[XML_TREEPULLER_ELEMENT_ATTRIBUTES]; $attr = {} unless defined $attr; if (! defined($name)) { return $attr; } return $attr->{$name}; } #private methods sub _extract_elements { return grep { $_->[XML_TREEPULLER_ELEMENT_TYPE] == XML_READER_TYPE_ELEMENT } @_; } #an easier to understand algorithm would be nice sub _recursive_get_child_elements { my ($tree, @path) = @_; my $child_nodes = $tree->[XML_TREEPULLER_ELEMENT_CHILDREN]; my @results; my $target; if (! scalar(@path)) { return $tree; } $target = shift(@path); return () unless defined $child_nodes; foreach (_extract_elements(@$child_nodes)) { next unless $_->[XML_TREEPULLER_ELEMENT_NAME] eq $target; push(@results, _recursive_get_child_elements($_, @path)); } return @results; } sub _init { my ($self, $root, $depth) = @_; my @elements = $self->get_elements; $self->[XML_TREEPULLER_ELEMENT_XPATH_ENGINE] = $root->[XML_TREEPULLER_ELEMENT_XPATH_ENGINE]; $self->[XML_TREEPULLER_ELEMENT_PARENT] = undef; $self->[XML_TREEPULLER_ELEMENT_ROOT] = $root; $self->[XML_TREEPULLER_ELEMENT_DEPTH] = $depth; weaken($self->[XML_TREEPULLER_ELEMENT_ROOT]); $depth++; for(my $i = 0; $i < @elements; $i++) { my $before = $elements[$i - 1]; my $after = $elements[$i + 1]; if ($i - 1 < 0) { $before = undef; } $elements[$i]->[XML_TREEPULLER_ELEMENT_NEXT_SIBLING] = $before; $elements[$i]->[XML_TREEPULLER_ELEMENT_PREV_SIBLING] = $after; weaken($elements[$i]->[XML_TREEPULLER_ELEMENT_NEXT_SIBLING]); weaken($elements[$i]->[XML_TREEPULLER_ELEMENT_PREV_SIBLING]); } foreach (@elements) { #set the parent and root of each element $_->[XML_TREEPULLER_ELEMENT_PARENT] = $self; $_->[XML_TREEPULLER_ELEMENT_ROOT] = $root; $_[XML_TREEPULLER_ELEMENT_DEPTH] = $depth; weaken($_->[XML_TREEPULLER_ELEMENT_PARENT]); weaken($_->[XML_TREEPULLER_ELEMENT_ROOT]); bless($_, 'XML::TreePuller::Element'); $_->_init($root, $depth); } } #methods for Tree::XPathEngine sub xpath_get_name { return name(@_); } sub xpath_string_value { return (text(@_)); } sub xpath_get_parent_node { return $_[0]->[XML_TREEPULLER_ELEMENT_ROOT] || XML::TreePuller::Element::Document->new($_[0]); } sub xpath_get_child_nodes { return $_[0]->get_elements; } sub xpath_is_element_node { return 1; } sub xpath_is_document_node { return 0; } sub xpath_is_attribute_node { return 0; } sub xpath_to_string { return $_[0]; } sub xpath_to_number { return Tree::XPathEngine::Number->new($_[0]->xpath_to_string); } sub xpath_cmp { return $_[0]->[XML_TREEPULLER_ELEMENT_DEPTH] cmp $_[1]->[XML_TREEPULLER_ELEMENT_DEPTH]; } sub xpath_get_attributes { my $elt= shift; my $atts= $elt->attribute; my $rank=-1; my @atts= map { bless( { name => $_, value => $atts->{$_}, elt => $elt, rank => $rank -- }, 'XML::TreePuller::Element::Attribute') } sort keys %$atts; return @atts; } sub xpath_get_next_sibling { return $_[0]->[XML_TREEPULLER_ELEMENT_NEXT_SIBLING]; } sub xpath_get_prev_sibling { return $_[0]->[XML_TREEPULLER_ELEMENT_PREV_SIBLING]; } sub xpath_get_root_node { my ($node) = @_; return $node->[XML_TREEPULLER_ELEMENT_ROOT]->xpath_get_parent_node; } package XML::TreePuller::Element::Document; use strict; use warnings; use XML::TreePuller::Constants; sub new { my ($class, $root) = @_; my $self = [ $root ]; $self->[XML_TREEPULLER_ELEMENT_DEPTH] = -1; return bless($self, $class); } sub xpath_get_child_nodes { return( $_[0]->[0] ); } sub xpath_get_attributes { return (); } sub xpath_is_document_node { return 1 } sub xpath_is_element_node { return 0 } sub xpath_is_attribute_node { return 0 } sub xpath_get_parent_node { return; } sub xpath_get_root_node { return $_[0] } sub xpath_get_name { return; } sub xpath_get_next_sibling { return; } sub xpath_get_previous_sibling { return; } package XML::TreePuller::Element::Attribute; use strict; use warnings; sub xpath_get_value { return $_[0]->{value}; } sub xpath_get_name { return $_[0]->{name} ; } sub xpath_string_value { return $_[0]->{value}; } sub xpath_to_number { return Tree::XPathEngine::Number->new( $_[0]->{value}); } sub xpath_is_document_node { 0 } sub xpath_is_element_node { 0 } sub xpath_is_attribute_node { 1 } sub to_string { return qq{$_[0]->{name}="$_[0]->{value}"}; } 1;XML-TreePuller-0.1.2/lib/XML/TreePuller/CookBook/Intro.pm000644 000765 000024 00000006264 11365066527 023210 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl 1; __END__ =head1 NAME XML::TreePuller::CookBook::Intro - Various ways to work with an Atom feed =head1 ABOUT Atom documents are simple and small - they fit into RAM and don't have many nested elements. Processing them is straight forward and a good place to start learning. =head2 Atom Format An Atom feed looks like this: Example Feed A subtitle. urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6 2003-12-13T18:30:02Z John Doe johndoe@example.com Atom-Powered Robots Run Amok urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a 2003-12-13T18:30:02Z Some text. =head1 PROGRAMS =head2 Feed summaries Lets say you have 10 Atom feeds you are interested in subscribing to but you want to see what they have to offer as a summary; Perl to the rescue! The following script generates a report of an arbitrary number of Atom feeds off the Internet fetching them directly from a URL or a file. The format of the report is like this: Feed: Example Feed * Atom-Powered Robots Run Amok (that sure does sound like an interesting feed) #!/usr/bin/env perl use strict; use warnings; use XML::TreePuller; foreach (@ARGV) { my $root = XML::TreePuller->parse(location => $_); my $title = $root->xpath('/feed/title')->text; print "Feed: $title\n"; foreach ($root->xpath('/feed/entry/title')) { print " * ", $_->text, "\n"; } print "\n\n\n"; } =head2 Linking to entries Given an Atom feed what is the easiest way to build an HTML list of hyperlinks to the entries that are specified in it? We need to get the title which is stored in a single element and the hyperlink to the entry; there are multiple link elements and we only want one - the one with "rel" attribute value of "alternate". XPath makes quick work of this. #!/usr/bin/env perl use strict; use warnings; use XML::TreePuller; my $root = XML::TreePuller->parse(location => shift(@ARGV)); print "
    \n"; foreach($root->xpath('/feed/entry')) { my $title = $_->xpath('//title')->text; #there are many link elements but we only want one of them my $to = $_->xpath("//link[\@rel='alternate']")->attribute('href'); print "
  • $title
    • \n"; } print "
\n"; =head1 COPYRIGHT The ATOM example XML document was taken from Wikipedia at the following URL: http://en.wikipedia.org/w/index.php?title=Atom_(standard)&oldid=353180236 and is available under the Creative Commons Attribution ShareAlike license All other content is copyright Tyler Riddle; see the README for licensing terms. XML-TreePuller-0.1.2/lib/XML/TreePuller/CookBook/Patterns.pm000644 000765 000024 00000003232 11370062026 023667 0ustar00tylerstaff000000 000000 #!/usr/bin/env perl 1; __END__ =head1 NAME XML::TreePuller::CookBook::Patterns - Recipes for dealing with XML patterns =head1 LISTS #!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use XML::TreePuller; my $xml = < zero one two EOF my $root = XML::TreePuller->parse(string => $xml); my @array; map({ push(@array, $_->text) } $root->get_elements('element')); print join(' ', @array), "\n"; =head1 HASHES =head2 With attributes #!/usr/bin/env perl use strict; use warnings; use XML::TreePuller; my $xml = < zero one two EOF my $root = XML::TreePuller->parse(string => $xml); my %hash; map({ $hash{$_->attribute('key')} = $_->text; } $root->get_elements('entry')); foreach (0..2) { print $hash{$_}, "\n"; } =head2 Without attributes #!/usr/bin/env perl use strict; use warnings; use Data::Dumper; use XML::TreePuller; my $xml = < 0 zero 1 one 2 two EOF my $root = XML::TreePuller->parse(string => $xml); my %hash; my @keys = $root->get_elements('key/'); my @values = $root->get_elements('value/'); map({ $hash{$_->text} = shift(@values)->text } @keys); foreach (0..2) { print $hash{$_}, "\n"; } =head1 COPYRIGHT All content is copyright Tyler Riddle; see the README for licensing terms. XML-TreePuller-0.1.2/lib/XML/TreePuller/CookBook/Performance.pm000644 000765 000024 00000011071 11365066567 024352 0ustar00tylerstaff000000 000000 1; __END__ =head1 NAME XML::TreePuller::CookBook::Performance - Increasing the rate of data through XML::TreePuller =head1 ABOUT Wikipedia (and MediaWiki) dump files present interesting parsing challenges - they are not of a high complexity but they do get to be very large; the English Wikipedia dump file is around 24 gigabytes and the dump file that has all of the revisions ever made is estimated to be around 1.5 terabytes (or larger). We'll cover parsing the Wikipedia dump files in faster and faster ways. =head2 Wikipedia dump format The dump file looks a little something like this: Wikipedia Talk Perl A Random Monger A nifty little language if I do say so myself!