Lingua-EN-Fathom-1.27000755000000000000 014465274402 14735 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/Build.PL000444000000000000 135213366411170 16361 0ustar00unknownunknown000000000000use Module::Build; Module::Build -> new ( module_name => 'Lingua::EN::Fathom', license => 'perl', dist_abstract => "Measure readability of English text", dist_author => 'Kim Ryan ', build_requires => { Test::More => 0.94, Test::Pod => 1.04, }, configure_requires => { Module::Build => 0.3800, }, requires => { locale => 1.00, strict => 1.04, warnings => 0, Lingua::EN::Syllable => 0.28, Lingua::EN::Sentence => 0.28 }, meta_merge => { resources => { repository => 'https://github.com/kimryan/Readability' } } ) -> create_build_script(); Lingua-EN-Fathom-1.27/Changes000444000000000000 646514465274332 16402 0ustar00unknownunknown000000000000Revision history for Perl CPAN module Lingua::En::Fathom 1.27 11 Aug 2023 Fixed typos 1.26 10 Aug 2023 Statistics now working correctly in accumulate mode Improved documentation 1.25 5 Jul 2023 Fixed documentation 1.24 3 Jul 2023 Measure blank and non text lines correctly. Thanks to Jeffrey Grace for the bug report 1.23 3 Jun 2023 Initialised all values prior to analyse. Thanks to Jeffrey Grace for the fix Improved documentation 1.22 1 Nov 2018 Added prerequisite module Lingua::EN:Sentence to build files Updated main.t 1.21 31 Oct 2018 Updated main.t 1.20 31 Oct 2018 Renamed github repo to 'Readability' Used Lingua::EN::Sentence instead of a regexp to get a more accurate sentence count 1.19 15 Sep 2016 Set github as repository 1.18 7 Apr 2015 removed indirect object notation for 'new' method Fix for #102898, no minimum version set for warnings in Build.PL, to support very old versions of Perl 1.17 20 Mar 2015 Added prerequiste moudule to Build.PL 1.16 20 Mar 2015 Removed tests that depended on Lingua::EN::Syllable results. Any changes to this required module will break test results. Added Build.PL 1.15 6 Aug 2011 Removed use of diagnostic pragma in main.t, seems to cause errors for CPAN testers 1.14 31 May 2011 Changed main.t to use 'like' instead of 'if' when testing for floating point numbers. This means tests are independent of machine precision 1.13 30 May 2011 Added more tests to main.t Upgraded all tests to use Test::More module Thanks to Kirk Kimmel for theses changes 1.12 13 Apr 2008 Minor fixes 1.11 08 Apr 2007 Fixed bug in generation of META.yml 1.10 07 Apr 2007 Improved documentation 1.09 11 Jul 2005 Improved documentation Updated distribution to current CPAN requirements 1.08 13 Feb 2004 Improved documentation 1.07 02 Apr 2002 Improved documentation Added more abbreviations to _analyse_words 1.06 04 Jan 2002 Fixed error in calculation of Fog index, thanks to Trevor Jenkins Added method to get num_complex_words Added demo_recurse.pl 1.05 19 May 2001 Renamed _analyse_line subroutine to _analyse_words Moved duplicated code from analyse_file & analyse_block to analyse_line subroutine Extended list of allowable abbreviations, such as "Pty. Ltd., etc." Added method to get syllables_per_word Added method to get words_per_sentence 1.04 23 Feb 2001 _initialize now resets readability indices and num_paragraphs Sentence count is no longer tricked by quotes before a full stop Optional argument on analyse_block and analyse_file to allow stats to accumulate 1.03 07 Jan 2001: Added documentation of 'report' method 1.02 15 Mar 2000: Added paragraph count Fixed error in test 2 of test script (word count was 38, not 39) 1.01 12 Mar 2000: Added stricter test for hyphenated words Added more statistics to report Accounted for name prefixes (Mr., Dr. etc ) in sentences Improved documentation Changed Fog index calculation to use percentage of complex words instead of total number of complex words. Thanks to Steve Grantz Allowed for sentences ending in '?' and '!' as well as '.' Thanks to Steve Grantz for spotting this limitation. 1.00 23 Jan 2000: First Release Lingua-EN-Fathom-1.27/MANIFEST000444000000000000 40712760447556 16214 0ustar00unknownunknown000000000000MANIFEST README Makefile.PL Build.PL examples/demo.pl examples/demo_recurse.pl lib/Lingua/EN/Fathom.pm Changes t/main.t xt/author/pod.t xt/author/pod-coverage.t META.yml Module meta-data (added by MakeMaker) META.json Lingua-EN-Fathom-1.27/META.json000444000000000000 251714465274402 16520 0ustar00unknownunknown000000000000{ "abstract" : "Measure readability of English text", "author" : [ "Kim Ryan " ], "dynamic_config" : 1, "generated_by" : "Module::Build version 0.4229", "license" : [ "perl_5" ], "meta-spec" : { "url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec", "version" : 2 }, "name" : "Lingua-EN-Fathom", "prereqs" : { "build" : { "requires" : { "Test::More" : "0.94", "Test::Pod" : "1.04" } }, "configure" : { "requires" : { "Module::Build" : "0.38" } }, "runtime" : { "requires" : { "Lingua::EN::Sentence" : "0.28", "Lingua::EN::Syllable" : "0.28", "locale" : "1", "strict" : "1.04", "warnings" : "0" } } }, "provides" : { "Lingua::EN::Fathom" : { "file" : "lib/Lingua/EN/Fathom.pm", "version" : "1.27" } }, "release_status" : "stable", "resources" : { "license" : [ "http://dev.perl.org/licenses/" ], "repository" : { "url" : "https://github.com/kimryan/Readability" } }, "version" : "1.27", "x_serialization_backend" : "JSON::PP version 4.04" } Lingua-EN-Fathom-1.27/META.yml000444000000000000 151414465274402 16344 0ustar00unknownunknown000000000000--- abstract: 'Measure readability of English text' author: - 'Kim Ryan ' build_requires: Test::More: '0.94' Test::Pod: '1.04' configure_requires: Module::Build: '0.38' dynamic_config: 1 generated_by: 'Module::Build version 0.4229, CPAN::Meta::Converter version 2.150010' license: perl meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html version: '1.4' name: Lingua-EN-Fathom provides: Lingua::EN::Fathom: file: lib/Lingua/EN/Fathom.pm version: '1.27' requires: Lingua::EN::Sentence: '0.28' Lingua::EN::Syllable: '0.28' locale: '1' strict: '1.04' warnings: '0' resources: license: http://dev.perl.org/licenses/ repository: https://github.com/kimryan/Readability version: '1.27' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' Lingua-EN-Fathom-1.27/Makefile.PL000444000000000000 175113366411134 17042 0ustar00unknownunknown000000000000use ExtUtils::MakeMaker; # See lib/ExtUtils/MakeMaker.pm for details of how to influence # the contents of the Makefile that is written. WriteMakefile ( 'NAME' => 'Lingua::EN::Fathom', 'VERSION_FROM' => 'lib/Lingua/EN/Fathom.pm', 'ABSTRACT' => 'Measure readability of English text', 'AUTHOR' => 'Kim Ryan ', 'LICENSE' => 'perl', 'PREREQ_PM' => { locale => 1.00, strict => 1.04, warnings => 0, 'Lingua::EN::Syllable' => 0.28, 'Lingua::EN::Sentence' => 0.28 }, TEST_REQUIRES => { Test::More => 0.94, Test::Pod => 1.04, }, META_MERGE => { 'meta-spec' => { version => 2 }, resources => { repository => { type => 'git', url => 'https://github.com/dwimperl/Task-DWIM.git', web => 'https://github.com/dwimperl/Task-DWIM', }, } } ); Lingua-EN-Fathom-1.27/README000444000000000000 332514450451006 15744 0ustar00unknownunknown000000000000Lingua::EN::Fathom - readability and general measurements of English text SYNOPSIS use Lingua::EN::Fathom; my $text = Lingua::EN::Fathom->new(); $text->analyse_file("sample.txt"); $accumulate = 1; $text->analyse_block($text_string,$accumulate); # Methods to return statistics on the analysed text $text->num_chars; $text->num_words; $text->percent_complex_words; $text->num_sentences; $text->num_text_lines; $text->num_non_text_lines; $text->num_blank_lines; $text->num_paragraphs; $text->syllables_per_word; $text->words_per_sentence; $text->unique_words; $text->fog; $text->flesch; $text->kincaid; %words = $text->unique_words; foreach $word ( sort keys %words ) { print("$words{$word} :$word\n"); } $fog = $text->fog; $flesch = $text->flesch; $kincaid = $text->kincaid; print($text->report); DESCRIPTION This module analyses English text in either a string or file. Totals are then calculated for the number of characters, words, sentences, blank and non blank (text) lines and paragraphs. Three common readability statistics are also derived, the Fog, Flesch and Kincaid indices. All of these properties can be accessed through individual methods, or by generating a text report. A hash of all unique words and the number of times they occur is generated. REQUIRES Lingua::EN::Syllable, Lingua::EN::Sentence HOW TO INSTALL perl Makefile.PL make make test make install or.. perl Build.PL build build test build install AUTHOR Lingua::EN::Fathom was written by Kim Ryan, kimryan at cpan dot org. COPYRIGHT AND LICENSE Copyright (c) 2023 Kim Ryan. All rights reserved. This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. Lingua-EN-Fathom-1.27/examples000755000000000000 014465274402 16553 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/examples/demo.pl000444000000000000 270113245375500 20165 0ustar00unknownunknown000000000000#! /usr/local/bin/perl # Demo script for Lingua::EN::Fathom.pm use lib "../lib"; use Lingua::EN::Fathom; my $sample1 = q{ In general, construction of pictograms follows the general procedure used in constructing bar charts. But two special rules should be followed! First, all of the picture units used must be of equal size. The comparisons must be made wholly on the basis of the number of illustrations used and never by varying the areas of the individual pictures used. The reason for this rule is obvious. The human eye is grossly inadequate in comparing areas of geometric designs. Second, the pictures or symbols used must appropriately depict the quantity to be illustrated. A comparison of the navies of the world, for example, might make use of miniature ship drawings. Cotton production might be shown by bales of cotton. Obviously, the drawings used must be immediately interpreted by the reader. End. }; my $sample2 = q{ The second paragraph to analyse. }; my $sample3 = q{ Mr. Brown goes to town. }; #------------------------------------------------------------------------------- my $text = Lingua::EN::Fathom->new(); # $text->analyse_block($sample1,1); # $text->analyse_block($sample2,1); $text->analyse_block($sample3,1); print($text->report,"\n"); %uniq_words = $text->unique_words; foreach $word ( sort keys %uniq_words ) { # print occurences of each unique word, followed by the word itself print("$uniq_words{$word}\t:$word\n"); } Lingua-EN-Fathom-1.27/examples/demo_recurse.pl000444000000000000 572 7317257104 21663 0ustar00unknownunknown000000000000#! /usr/local/bin/perl # demo_recurse.pl script for usew with Lingua::EN::Fathom.pm use Lingua::EN::Fathom; use File::Find; @ARGV = ('.') unless @ARGV; my $text = new Lingua::EN::Fathom; find(\&analyse, @ARGV); print($text->report); sub analyse { return unless ( -T and -s ); print("Analysing file: $File::Find::name\n"); $text->analyse_file($File::Find::name,1); } Lingua-EN-Fathom-1.27/lib000755000000000000 014465274402 15503 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/lib/Lingua000755000000000000 014465274402 16722 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/lib/Lingua/EN000755000000000000 014465274402 17224 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/lib/Lingua/EN/Fathom.pm000444000000000000 4542014465274362 21167 0ustar00unknownunknown000000000000=head1 NAME Lingua::EN::Fathom - Measure readability of English text =head1 SYNOPSIS use Lingua::EN::Fathom; my $text = Lingua::EN::Fathom->new(); $text->analyse_file("sample.txt"); # Analyse contents of a text file $accumulate = 1; my $text_string = q{ Returns the number of words in the analysed text file or block. A word must consist of letters a-z with at least one vowel sound, and optionally an apostrophe or hyphen. ########################################## Items such as "&, K108, NSW" are not counted as words. Common abbreviations such a U.S. or numbers like 1.23 will not denote the end of a sentence. }; $text->analyse_block($text_string,$accumulate); # Analyse contents of a text string print($text->report); # Create a formatted report Number of characters : 312 Number of words : 54 Percent of complex words : 7.41 Average syllables per word : 1.4259 Number of sentences : 4 Average words per sentence : 13.5000 Number of text lines : 6 Number of non-text lines : 1 Number of blank lines : 2 Number of paragraphs : 2 READABILITY INDICES Fog : 8.3630 Flesch : 72.4992 Flesch-Kincaid : 6.5009 # Methods to return statistics on the analysed text $text->num_chars; $text->num_words; $text->percent_complex_words; $text->num_sentences; $text->num_text_lines; $text->num_non_text_lines; $text->num_blank_lines; # trailing EOLs are ignored $text->num_paragraphs; $text->syllables_per_word; $text->words_per_sentence; $text->unique_words; $text->fog; $text->flesch; $text->kincaid; # get a hash of unique words, keyed by word and occurrence as the value $text->unique_words # Print a list of unique words %words = $text->unique_words; foreach $word ( sort keys %words ) { print("$words{$word} :$word\n"); } =head1 REQUIRES Lingua::EN::Syllable, Lingua::EN::Sentence =head1 DESCRIPTION This module analyses English text in either a string or file. Totals are then calculated for the number of characters, words, sentences, blank and non blank (text) lines and paragraphs. Three common readability statistics are also derived, the Fog, Flesch and Kincaid indices. All of these properties can be accessed through individual methods, or by generating a text report. A hash of all unique words and the number of times they occur is generated. =head1 METHODS =head2 new The C method creates an instance of an text object This must be called before any of the following methods are invoked. Note that the object only needs to be created once, and can be reused with new input data. my $text = Lingua::EN::Fathom->new(); =head2 analyse_file The C method takes as input the name of a text file. Various text based statistics are calculated for the file. This method and C are prerequisites for all the following methods. An optional argument may be supplied to control accumulation of statistics. If set to a non zero value, all statistics are accumulated with each successive call. $text->analyse_file("sample.txt"); =head2 analyse_block The C method takes as input a text string. Various text based statistics are calculated for the file. This method and C are prerequisites for all the following methods. An optional argument may be supplied to control accumulation of statistics. If set to a non zero value, all statistics are accumulated with each successive call. $text->analyse_block($text_str,$accumulate); =head2 num_chars Returns the number of characters in the analysed text file or block. This includes characters such as spaces, and punctuation marks. =head2 num_words Returns the number of words in the analysed text file or block. A word must consist of letters a-z with at least one vowel sound, and optionally an apostrophe or hyphen. Items such as "&, K108, NW" are not counted as words. =head2 percent_complex_words Returns the percentage of complex words in the analysed text file or block. A complex word must consist of three or more syllables. This statistic is used to calculate the fog index. =head2 num_sentences Returns the number of sentences in the analysed text file or block. A sentence is any group of words and non words terminated with a single full stop. Spaces may occur before and after the full stop. =head2 num_text_lines Returns the number of lines containing some text in the analysed text file or block. =head2 num_non_text_lines Returns the number of lines containing no text in the analysed text file or block. =head2 num_blank_lines Returns the number of empty lines in the analysed text file or block. =head2 num_paragraphs Returns the number of paragraphs in the analysed text file or block. =head2 syllables_per_word Returns the average number of syllables per word in the analysed text file or block. =head2 words_per_sentence Returns the average number of words per sentence in the analysed text file or block. =head2 READABILITY Three indices of text readability are calculated. They all measure complexity as a function of syllables per word and words per sentence. They assume the text is well formed and logical. You could analyse a passage of nonsensical English and find the readability is quite good, provided the words are not too complex and the sentences not too long. For more information see: L =head2 fog Returns the Fog index for the analysed text file or block. ( words_per_sentence + percent_complex_words ) * 0.4 The Fog index, developed by Robert Gunning, is a well known and simple formula for measuring readability. The index indicates the number of years of formal education a reader of average intelligence would need to read the text once and understand that piece of writing with its word sentence workload. 18 unreadable 14 difficult 12 ideal 10 acceptable 8 childish =head2 flesch Returns the Flesch reading ease score for the analysed text file or block. 206.835 - (1.015 * words_per_sentence) - (84.6 * syllables_per_word) This score rates text on a 100 point scale. The higher the score, the easier it is to understand the text. A score of 60 to 70 is considered to be optimal. =head2 kincaid Returns the Flesch-Kincaid grade level score for the analysed text file or block. (11.8 * syllables_per_word) + (0.39 * words_per_sentence) - 15.59; This score rates text on U.S. grade school level. So a score of 8.0 means that the document can be understood by an eighth grader. A score of 7.0 to 8.0 is considered to be optimal. =head2 unique_words Returns a hash of unique words. The words (in lower case) are held in the hash keys while the number of occurrences are held in the hash values. =head2 report print($text->report); Produces a text based report containing all Fathom statistics for the currently analysed text block or file. For example: Number of characters : 813 Number of words : 135 Percent of complex words : 20.00 Average syllables per word : 1.7704 Number of sentences : 12 Average words per sentence : 11.2500 Number of text lines : 13 Number of non text lines : 0 Number of blank lines : 8 Number of paragraphs : 4 READABILITY INDICES Fog : 12.5000 Flesch : 45.6429 Flesch-Kincaid : 9.6879 The return value is a string containing the report contents =head1 SEE ALSO L,L,L =head1 POSSIBLE EXTENSIONS Count white space and punctuation characters Allow user control over what strictly defines a word =head1 LIMITATIONS The syllable count provided in Lingua::EN::Syllable is about 90% accurate Acronyms that contain vowels, like GPO, will be counted as words. The fog index should exclude proper names =head1 AUTHOR Lingua::EN::Fathom was written by Kim Ryan . =head1 COPYRIGHT AND LICENSE Copyright (c) 2023 Kim Ryan. All rights reserved. This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut #------------------------------------------------------------------------------ package Lingua::EN::Fathom; use Lingua::EN::Syllable; use Lingua::EN::Sentence; use strict; use warnings; our $VERSION = '1.27'; #------------------------------------------------------------------------------ # Create a new instance of a text object. sub new { my $class = shift; my $text = {}; bless($text,$class); $text = &_initialize($text); return($text); } #------------------------------------------------------------------------------ # Analyse text stored in a file, reading from the file one line at a time sub analyse_file { my $text = shift; my ($file_name,$accumulate) = @_; unless ( $accumulate ) { $text = _initialize($text); } $text->{file_name} = $file_name; # Only analyse non-empty text files unless ( -T $file_name and -s $file_name ) { return($text); } open(IN_FH,"<$file_name"); my $in_paragraph = 0; my $all_text; while ( ) { my $one_line = $_; $all_text .= $one_line; ($in_paragraph,$text) = _analyse_line($text,$one_line,$in_paragraph); } close(IN_FH); my $sentences= Lingua::EN::Sentence::get_sentences($all_text); $text->{num_sentences} += scalar(@$sentences); $text->_calculate_readability; return($text); } #------------------------------------------------------------------------------ # Analyse a block of text, stored as a string. The string may contain line # terminators. sub analyse_block { my $text = shift; my ($block,$accumulate) = @_; unless ( $accumulate ) { $text = _initialize($text); } unless ( $block ) { return($text); } my $in_paragraph = 0; # Split on EOL character # repeating trailing line terminators are stripped my @all_lines = split(/\n/,$block); my $one_line; foreach $one_line ( @all_lines ) { ($in_paragraph,$text) = _analyse_line($text,$one_line,$in_paragraph); } my $sentences= Lingua::EN::Sentence::get_sentences($block); if (defined($sentences)) { $text->{num_sentences} += scalar(@$sentences); } $text->_calculate_readability; return($text); } #------------------------------------------------------------------------------ sub num_chars { my $text = shift; return($text->{num_chars}); } #------------------------------------------------------------------------------ sub num_words { my $text = shift; return($text->{num_words}); } #------------------------------------------------------------------------------ sub percent_complex_words { my $text = shift; return($text->{percent_complex_words}); } #------------------------------------------------------------------------------ sub num_sentences { my $text = shift; return($text->{num_sentences}); } #------------------------------------------------------------------------------ sub num_text_lines { my $text = shift; return($text->{num_text_lines}); } #------------------------------------------------------------------------------ sub num_non_text_lines { my $text = shift; return($text->{num_non_text_lines}); } #------------------------------------------------------------------------------ sub num_blank_lines { my $text = shift; return($text->{num_blank_lines}); } #------------------------------------------------------------------------------ sub num_paragraphs { my $text = shift; return($text->{num_paragraphs}); } #------------------------------------------------------------------------------ sub syllables_per_word { my $text = shift; return($text->{syllables_per_word}); } #------------------------------------------------------------------------------ sub words_per_sentence { my $text = shift; return($text->{words_per_sentence}); } #------------------------------------------------------------------------------ sub num_syllables { my $text = shift; return($text->{num_syllables}); } #------------------------------------------------------------------------------ sub complex_words { my $text = shift; return($text->{num_complex_words}); } #------------------------------------------------------------------------------ sub fog { my $text = shift; return($text->{fog}); } #------------------------------------------------------------------------------ sub flesch { my $text = shift; return($text->{flesch}); } #------------------------------------------------------------------------------ sub kincaid { my $text = shift; return($text->{kincaid}); } #------------------------------------------------------------------------------ # Return anonymous hash of all the unique words in analysed text. The words # occurrence count is stored in the hash value. sub unique_words { my $text = shift; if ( $text->{unique_words} ) { return( %{ $text->{unique_words} } ); } else { return(undef); } } #------------------------------------------------------------------------------ # Provide a formatted text report of all statistics for a text object. # Return report as a string. sub report { my $text = shift; my $report = ''; $text->{file_name} and $report .= sprintf("File name : %s\n",$text->{file_name} ); $report .= sprintf("Number of characters : %d\n", $text->num_chars); $report .= sprintf("Number of words : %d\n", $text->num_words); $report .= sprintf("Percent of complex words : %.2f\n",$text->percent_complex_words); $report .= sprintf("Average syllables per word : %.4f\n",$text->syllables_per_word); $report .= sprintf("Number of sentences : %d\n", $text->num_sentences); $report .= sprintf("Average words per sentence : %.4f\n",$text->words_per_sentence); $report .= sprintf("Number of text lines : %d\n", $text->num_text_lines); $report .= sprintf("Number of non-text lines : %d\n", $text->num_non_text_lines); $report .= sprintf("Number of blank lines : %d\n", $text->num_blank_lines); $report .= sprintf("Number of paragraphs : %d\n", $text->num_paragraphs); $report .= "\n\nREADABILITY INDICES\n\n"; $report .= sprintf("Fog : %.4f\n",$text->fog); $report .= sprintf("Flesch : %.4f\n",$text->flesch); $report .= sprintf("Flesch-Kincaid : %.4f\n",$text->kincaid); return($report); } #------------------------------------------------------------------------------ # PRIVATE METHODS #------------------------------------------------------------------------------ sub _initialize { my $text = shift; $text->{num_chars} = 0; $text->{num_syllables} = 0; $text->{num_words} = 0; $text->{num_complex_words} = 0; $text->{syllables_per_word} = 0; $text->{words_per_sentence} = 0; $text->{percent_complex_words} = 0; $text->{num_text_lines} = 0; $text->{num_non_text_lines} = 0; $text->{num_blank_lines} = 0; $text->{num_paragraphs} = 0; $text->{num_sentences} = 0; $text->{unique_words} = (); $text->{file_name} = ''; $text->{fog} = 0; $text->{flesch} = 0; $text->{kincaid} = 0; return($text); } #------------------------------------------------------------------------------ # Increment number of text lines, blank lines and paragraphs sub _analyse_line { my $text = shift; my ($one_line,$in_paragraph) = @_; if ( $one_line =~ /\w/ ) { chomp($one_line); $text = _analyse_words($text,$one_line); $text->{num_text_lines}++; unless ( $in_paragraph ) { $text->{num_paragraphs}++; $in_paragraph = 1; } } elsif ($one_line eq '' ) # empty line { $text->{num_blank_lines}++; $in_paragraph = 0; } elsif ($one_line =~ /^\W+$/ ) # non text { $text->{num_non_text_lines}++; $in_paragraph = 0; } return($in_paragraph,$text); } #------------------------------------------------------------------------------ # Try to detect real words in line. Increment syllable, word, and complex word counters. sub _analyse_words { my $text = shift; my ($one_line) = @_; $text->{num_chars} += length($one_line); # Word found, such as: twice, BOTH, a, I'd, non-plussed .. # Ignore words like 'Mr.', K12, &, X.Y.Z ... # It could be argued that Mr. is a word, but this approach should detect most of the non words # which have punctuation or numbers in them while ( $one_line =~ /\b([a-z][-'a-z]*)\b/ig ) { my $one_word = $1; # Try to filter out acronyms and abbreviations by accepting # words with a vowel sound. This won't work for GPO etc. next unless $one_word =~ /[aeiouy]/i; # Test for valid hyphenated word like be-bop if ( $one_word =~ /-/ ) { next unless $one_word =~ /[a-z]{2,}-[a-z]{2,}/i; } # word frequency count $text->{unique_words}{lc($one_word)}++; $text->{num_words}++; # Use subroutine from Lingua::EN::Syllable my $num_syllables_current_word = syllable($one_word); $text->{num_syllables} += $num_syllables_current_word; # Required for Fog index, count non hyphenated words of 3 or more # syllables. Should add check for proper names in here as well if ( $num_syllables_current_word > 2 and $one_word !~ /-/ ) { $text->{num_complex_words}++; } } return($text); } #------------------------------------------------------------------------------ # Determine the three readability indices sub _calculate_readability { my $text = shift; if ( $text->{num_sentences} and $text->{num_words} ) { $text->{words_per_sentence} = $text->{num_words} / $text->{num_sentences}; $text->{syllables_per_word} = $text->{num_syllables} / $text->{num_words}; $text->{percent_complex_words} = ( $text->{num_complex_words} / $text->{num_words} ) * 100; $text->{fog} = ( $text->{words_per_sentence} + $text->{percent_complex_words} ) * 0.4; $text->{flesch} = 206.835 - (1.015 * $text->{words_per_sentence}) - (84.6 * $text->{syllables_per_word}); $text->{kincaid} = (11.8 * $text->{syllables_per_word}) + (0.39 * $text->{words_per_sentence}) - 15.59; } else { $text->{words_per_sentence} = 0; $text->{syllables_per_word} = 0; $text->{num_complex_words} = 0; $text->{fog} = 0; $text->{flesch} = 0; $text->{kincaid} = 0; } } #------------------------------------------------------------------------------ return(1); Lingua-EN-Fathom-1.27/t000755000000000000 014465274402 15200 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/t/main.t000444000000000000 231014465024233 16435 0ustar00unknownunknown000000000000#------------------------------------------------------------------------------ # Test script for Lingua::EN::::Fathom.pm # # Author : Kim Ryan, Kirk Kimmel #------------------------------------------------------------------------------ use warnings; use strict; use Test::More tests => 8; use lib "./lib"; BEGIN { # does it load properly? require_ok('Lingua::EN::Fathom'); } my $sample = q{ Returns the number of words in the analysed text file or block. A word must consist of letters a-z with at least one vowel sound, and optionally an apostrophe or hyphen. ########################################## Items such as "&, K108, NSW" are not counted as words. Common abbreviations such a U.S. or numbers like 1.23 will not denote the end of a sentence. }; my $text = Lingua::EN::Fathom->new(); $text->analyse_block($sample); is( $text->num_chars, 312, 'sub num_chars' ); is( $text->num_words, 54, 'sub num_words' ); is( $text->num_sentences, 4,'sub num_sentences' ); is( $text->num_text_lines, 6,'sub num_text_lines' ); is( $text->num_non_text_lines, 1,'sub num_non_text_lines' ); is( $text->num_blank_lines, 2,'sub num_blank_lines' ); is( $text->num_paragraphs, 2,'sub num_paragraphs' ); Lingua-EN-Fathom-1.27/xt000755000000000000 014465274402 15370 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/xt/author000755000000000000 014465274402 16672 5ustar00unknownunknown000000000000Lingua-EN-Fathom-1.27/xt/author/pod-coverage.t000444000000000000 27311570574436 21555 0ustar00unknownunknown000000000000#!perl use Test::More; use warnings; use strict; eval "use Test::Pod::Coverage"; plan skip_all => "Test::Pod::Coverage required for testing POD coverage" if $@; all_pod_coverage_ok(); Lingua-EN-Fathom-1.27/xt/author/pod.t000444000000000000 23111570572470 17752 0ustar00unknownunknown000000000000#!perl use Test::More; use warnings; use strict; eval "use Test::Pod"; plan skip_all => "Test::Pod required for testing POD" if $@; all_pod_files_ok();