Message-Passing-Filter-Regexp-0.05/0000755000175000017500000000000012167752622017007 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/lib/0000755000175000017500000000000012167752622017555 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/lib/Message/0000755000175000017500000000000012167752622021141 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/lib/Message/Passing/0000755000175000017500000000000012167752622022545 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/lib/Message/Passing/Filter/0000755000175000017500000000000012167752622023772 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/lib/Message/Passing/Filter/Regexp/0000755000175000017500000000000012167752622025224 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/lib/Message/Passing/Filter/Regexp/Log.pm0000644000175000017500000000273512065322121026272 0ustar chenrynchenrynpackage Message::Passing::Filter::Regexp::Log; use strict; use warnings; use Config::Tiny; use base qw( Regexp::Log ); use vars qw( $VERSION %DEFAULT %FORMAT %REGEXP ); $VERSION = 0.01; sub _read_config { Config::Tiny->read(shift); }; sub new { my $class = shift; my $args = { @_ }; my $regexfile = delete $args->{'regexfile'}; my $cfg = &_read_config($regexfile); %DEFAULT = ( default => '', capture => [] ); %FORMAT = %{ $cfg->{FORMAT} }; %REGEXP = %{ $cfg->{REGEXP} }; my $self = $class->SUPER::new(%$args); bless $self, $class; return $self; }; 1; __END__ =head1 NAME Message::Passing::Filter::Regexp::Log - Extension for Regexp::Log with Config::Tiny =head1 SYNOPSIS use Message::Passing::Filter::Regexp::Log; my $regex = Message::Passing::Filter::Regexp::Log->new( regexfile => "./regexfile", format => ":syslog", capture => [qw(pri host msg time)] ); my @fields = $foo->capture; my $re = $foo->regexp; while (<>) { my %data; @data{@fields} = /$re/; # no need for /o, it's a compiled regexp }; =head1 DESCRIPTION =head1 SEE ALSO C =head1 AUTHOR chenryn, Erao.chenlin@gmail.com =head1 COPYRIGHT AND LICENSE Copyright (C) 2012 by chenryn This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself, either Perl version 5.14.2 or, at your option, any later version of Perl 5 you may have available. =cut Message-Passing-Filter-Regexp-0.05/lib/Message/Passing/Filter/Regexp.pm0000644000175000017500000001300712167752610025560 0ustar chenrynchenrynpackage Message::Passing::Filter::Regexp; use 5.014002; use strict; use warnings; use Moo; use MooX::Types::MooseLike::Base qw/ HashRef ArrayRef Str RegexpRef /; use namespace::clean -except => 'meta'; use DateTime; use JSON::Types; use Message::Passing::Filter::Regexp::Log; with qw/ Message::Passing::Role::Filter /; use vars qw( $VERSION ); $VERSION = 0.05; has format => ( is => 'ro', isa => Str, default => sub { ':default' }, ); has regexfile => ( is => 'ro', isa => Str, default => sub { '/etc/message-passing/regexfile' }, ); has capture => ( is => 'ro', isa => ArrayRef, default => sub { [] }, ); has mutate => ( is => 'ro', isa => HashRef, default => sub { {} }, ); has fromto => ( is => 'ro', isa => HashRef, default => sub { { '@message' => '@fields' } }, ); has _regex => ( is => 'ro', lazy => 1, builder => '_build_regex', ); has _re => ( is => 'ro', isa => RegexpRef, lazy => 1, builder => '_build_re', ); has _fields => ( is => 'ro', isa => ArrayRef, lazy => 1, builder => '_build_fields', ); sub _build_regex { my $self = shift; Message::Passing::Filter::Regexp::Log->new( format => $self->format, capture => $self->capture, regexfile => $self->regexfile, ); } sub _build_re { my $self = shift; return $self->_regex->regexp; } sub _build_fields { my $self = shift; return [ $self->_regex->capture ]; } sub filter { my ( $self, $message ) = @_; while ( my ( $from, $to ) = each %{ $self->fromto } ) { my $log_line = $message->{$from}; my %data; my $re = $self->_re; @data{ @{ $self->_fields } } = $log_line =~ /$re/; for ( keys %{ $self->mutate } ) { my $type = $self->mutate->{$_}; $data{$_} = eval "$type $data{$_}"; } if ( defined $to ) { $message->{$to} = {%data}; } # put parsed fields directly in message hashref else { %$message = ( %$message, %data ); } } return $message; } 1; 1; __END__ =head1 NAME Message::Passing::Filter::Regexp - Regexp Capture Filter For Message::Passing =head1 SYNOPSIS # regexfile [FORMAT] :default = %date %status %remotehost %domain %request %originhost %responsetime %upstreamtime %bytes %referer %ua %xff :nginxaccesslog = %date %status %remotehost %bytes %responsetime [REGEXP] %date = (?#=date)\[(?#=ts)\d{2}\/\w{3}\/\d{4}(?::\d{2}){3}(?#!ts) [-+]\d{4}\](?#!date) %status = (?#=status)\d+(?#!status) %remotehost = (?#=remotehost)\S+(?#!remotehost) %domain = (?#=domain).*?(?#!domain) %request = (?#=request)-|(?#=method)\w+(?#!method) (?#=url).*?(?#!url) (?#=version)HTTP/\d\.\d(?#!version)(?#!request) %originhost = (?#=originhost)-|(?#=oh).*?(?#!oh):\d+(?#!originhost) %responsetime = (?#=responsetime)-|.*?(?#!responsetime) %upstreamtime = (?#=upstreamtime).*?(?#!upstreamtime) %bytes = (?#=bytes)\d+(?#!bytes) %referer = (?#=referer)\"(?#=ref).*?(?#!ref)\"(?#!referer) %useragent = (?#=useragent)\"(?#=ua).*?(?#!ua)\"(?#!useragent) %xforwarderfor = (?#=xforwarderfor)\"(?#=xff).*?(?#!xff)\"(?#!xforwarderfor) # message-passing-cli use Message::Passing::DSL; run_message_server message_chain { output stdout => ( class => 'STDOUT', ); output elasticsearch => ( class => 'ElasticSearch', elasticsearch_servers => ['127.0.0.1:9200'], ); encoder("encoder", class => 'JSON', output_to => 'stdout', output_to => 'es', ); filter regexp => ( class => 'Regexp', format => ':nginxaccesslog', capture => [qw( ts status remotehost url oh responsetime upstreamtime bytes )] output_to => 'encoder', ); filter logstash => ( class => 'ToLogstash', output_to => 'regexp', ); decoder decoder => ( class => 'JSON', output_to => 'logstash', ); input file => ( class => 'FileTail', output_to => 'decoder', ); }; =head1 DESCRIPTION This filter passes all incoming messages through with regexp captures. Note it must be running after Message::Passing::Filter::ToLogstash because it don't process with json format but directly capture C<< $message->{'@message'} >> data lines into C<< %{ $message->{'@fields'} } >> =head1 ATTRIBUTES =head2 regexfile Path of your regexfile. Default is /etc/message-passing/regexfile. =head2 format Name of a defined format in your regexfile. =head2 capture ArrayRef of regex names which you want to capture and has been defined in your regexfile. note delete the prefix C<%>. =head2 fromto HashRef of fields which you want capture from and to. Default as C<< { '@message' => '@fields' } >>. If you set C<< undef >> to one key, such field will exists directly in C<< %$message >>. =head1 SEE ALSO Idea steal from Grok filter Config Format see L =head1 AUTHOR chenryn, Erao.chenlin@gmail.com =head1 CONTRIBUTOR Alexander Hartmaier =head1 COPYRIGHT AND LICENSE Copyright (C) 2012 by chenryn This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself, either Perl version 5.14.2 or, at your option, any later version of Perl 5 you may have available. =cut Message-Passing-Filter-Regexp-0.05/t/0000755000175000017500000000000012167752622017252 5ustar chenrynchenrynMessage-Passing-Filter-Regexp-0.05/t/002-Regexp-Log.t0000644000175000017500000000167312065323717021712 0ustar chenrynchenryn#!/usr/bin/perl use lib 'lib'; use Data::Dumper; use Test::More; BEGIN{ use_ok( 'Message::Passing::Filter::Regexp::Log' ); } my $line = '127.0.0.1 - - [19/Jan/2005:21:42:43 +0000] "POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1" 200 11435'; my $filter = Message::Passing::Filter::Regexp::Log->new( regexfile => 't/regexfile', format => ':common', capture => [ qw( ts req ) ], ); isa_ok( $filter, Message::Passing::Filter::Regexp::Log ); my @fields = $filter->capture; is_deeply( \@fields, [qw(ts req)], "capture fields test"); my $re = $filter->regexp; is( $re, '(?^:^(?:\S+) (?:.*?) (?:.*?) (?:\[(\d{2}\\/\w{3}\\/\d{4}(?::\d{2}){3} [-+]\d{4})\]) (?:\"(.*?)\") (?:\d+) (?:-|\d+)$)', "compiled regex test"); my %data; @data{@fields} = $line =~ /$re/; is_deeply( \%data, { req => 'POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1', ts => '19/Jan/2005:21:42:43 +0000', }, "regexp data match test"); done_testing(); Message-Passing-Filter-Regexp-0.05/t/001-Config-Tiny.t0000644000175000017500000000037712065324100022051 0ustar chenrynchenryn#!/usr/bin/perl use lib 'lib'; use Test::More; BEGIN{ use_ok( 'Config::Tiny' ); } my $parser = Config::Tiny->new(); isa_ok( $parser, Config::Tiny ); my $cfg = $parser->read('t/regexfile'); is( $parser->errstr, '', "read error message"); done_testing(); Message-Passing-Filter-Regexp-0.05/t/003-Message-Passing-Filter-Regexp.t0000644000175000017500000000352712160226171025373 0ustar chenrynchenryn# Before `make install' is performed this script should be runnable with # `make test'. After `make install' it should work as `perl Message-Passing-Filter-Regexp.t' use lib 'lib'; use strict; use warnings; use Data::Dumper; use JSON; use Test::More; use Message::Passing::Output::Test; BEGIN { use_ok('Message::Passing::Filter::Regexp') } my $input = { '@tags' => [], '@message' => '127.0.0.1 - - [19/Jan/2012:21:08:54 +0800] "POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1" 200 11435', '@timestamp' => '2012-06-19T21:08:54+0800', '@fields' => {}, }; my $exp = { '@tags' => [], '@message' => '127.0.0.1 - - [19/Jan/2012:21:08:54 +0800] "POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1" 200 11435', '@timestamp' => '2012-06-19T21:08:54+0800', '@fields' => { req => 'POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1', ts => '19/Jan/2012:21:08:54 +0800', bytes => '11435', }, }; my $json_exp = { '@tags' => [], '@message' => '127.0.0.1 - - [19/Jan/2012:21:08:54 +0800] "POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1" 200 11435', '@timestamp' => '2012-06-19T21:08:54+0800', '@fields' => { req => 'POST /cgi-bin/brum.pl?act=evnt-edit&eventid=24 HTTP/1.1', ts => '19/Jan/2012:21:08:54 +0800', bytes => 11435, }, }; my $out = Message::Passing::Output::Test->new; my $in = Message::Passing::Filter::Regexp->new( output_to => $out, regexfile => 't/regexfile', format => ':common', capture => [qw( ts req bytes )], mutate => { bytes => 'number' }, ); $in->consume($input); my ($output) = $out->messages; is_deeply( $output, $exp, "capture result test" ); is( to_json( [ $output->{'@fields'}->{bytes} ] ), to_json( [ $json_exp->{'@fields'}->{bytes} ] ), "number test" ); done_testing; Message-Passing-Filter-Regexp-0.05/t/regexfile0000644000175000017500000000131212065326465021143 0ustar chenrynchenryn# regexfile transfromed from Regexp::Log::Common [FORMAT] :default = %host %rfc %authuser %date %request %status %bytes :common = %host %rfc %authuser %date %request %status %bytes :extended = %host %rfc %authuser %date %request %status %bytes %referer %useragent [REGEXP] %host = (?#=host)\S+(?#!host) %rfc = (?#=rfc).*?(?#!rfc) %authuser = (?#=authuser).*?(?#!authuser) %date = (?#=date)\[(?#=ts)\d{2}\/\w{3}\/\d{4}(?::\d{2}){3} [-+]\d{4}(?#!ts)\](?#!date) %request = (?#=request)\"(?#=req).*?(?#!req)\"(?#!request) %status = (?#=status)\d+(?#!status) %bytes = (?#=bytes)-|\d+(?#!bytes) %referer = (?#=referer)\"(?#=ref).*?(?#!ref)\"(?#!referer) %useragent = (?#=useragent)\"(?#=ua).*?(?#!ua)\"(?#!useragent) Message-Passing-Filter-Regexp-0.05/Changes0000644000175000017500000000132612167752244020304 0ustar chenrynchenrynRevision history for Perl extension Message::Passing::Filter::Regexp. 0.05 Fri Jul 12 18:03:14 2013 - add support for `fromto => { message => undef }`. 0.04 Wed Jun 19 12:03:00 2013 - add `fromto` hashref attr, now you can capture multi-fields. - modify JSON test for the hash randomization changed by Perl5.18 0.03 Sat Jun 8 10:31:42 2013 - add attrs of _re and _fields, so no more need to call function of Regexp::Log everytime. Save half time in 100k lines testing. 0.02 Sat Dec 29 01:40:00 2012 - use JSON::Types to mutate string to number 0.01 Sat Dec 22 15:05:40 2012 - original version; created by h2xs 1.23 with options -XA -n Message::Passing::Filter::Regexp Message-Passing-Filter-Regexp-0.05/MANIFEST0000644000175000017500000000056312067360303020132 0ustar chenrynchenrynMANIFEST README t/001-Config-Tiny.t t/003-Message-Passing-Filter-Regexp.t t/regexfile t/002-Regexp-Log.t Changes Makefile.PL lib/Message/Passing/Filter/Regexp/Log.pm lib/Message/Passing/Filter/Regexp.pm META.yml Module YAML meta-data (added by MakeMaker) META.json Module JSON meta-data (added by MakeMaker) Message-Passing-Filter-Regexp-0.05/README0000644000175000017500000000655312154513465017674 0ustar chenrynchenrynNAME Message::Passing::Filter::Regexp - Regexp Capture Filter For Message::Passing SYNOPSIS # regexfile *** FORMAT *** :default = "%date %status %remotehost %domain %request %originhost %responsetime %upstreamtime %bytes %referer %ua %xff" :nginxaccesslog = "%date %status %remotehost %bytes %responsetime" *** REGEXP *** %date = '(?#=date)\[(?#=ts)\d{2}\/\w{3}\/\d{4}(?::\d{2}){3}(?#!ts) [-+]\d{4}\](?#!date)' %status = '(?#=status)\d+(?#!status)' %remotehost = '(?#=remotehost)\S+(?#!remotehost)' %domain = '(?#=domain).*?(?#!domain)' %request = '(?#=request)-|(?#=method)\w+(?#!method) (?#=url).*?(?#!url) (?#=version)HTTP/\d\.\d(?#!version)(?#!request)' %originhost = '(?#=originhost)-|(?#=oh).*?(?#!oh):\d+(?#!originhost)' %responsetime = '(?#=responsetime)-|.*?(?#!responsetime)' %upstreamtime = '(?#=upstreamtime).*?(?#!upstreamtime)' %bytes = '(?#=bytes)\d+(?#!bytes)' %referer = '(?#=referer)\"(?#=ref).*?(?#!ref)\"(?#!referer)' %useragent = '(?#=useragent)\"(?#=ua).*?(?#!ua)\"(?#!useragent)' %xforwarderfor = '(?#=xforwarderfor)\"(?#=xff).*?(?#!xff)\"(?#!xforwarderfor)' # message-passing-cli use Message::Passing::DSL; run_message_server message_chain { input file => ( class => 'FileTail', output_to => 'decoder', ); decoder decoder => ( class => 'JSON', output_to => 'logstash', ); filter logstash => ( class => 'ToLogstash', output_to => 'regexp', ); filter regexp => ( class => 'Regexp', format => ':nginxaccesslog', capture => [qw( ts status remotehost url oh responsetime upstreamtime bytes )] output_to => 'encoder', ); encoder("encoder", class => 'JSON', output_to => 'stdout', output_to => 'es', ); output stdout => ( class => 'STDOUT', ); output elasticsearch => ( class => 'ElasticSearch', elasticsearch_servers => ['127.0.0.1:9200'], ); }; DESCRIPTION This filter passes all incoming messages through with regexp captures. Note it must be running after Message::Passing::Filter::ToLogstash because it don't process with json format but directly capture "$message->{'@message'}" data lines into "%{ $message->{'@fields'} }" ATTRIBUTES regexfile Path of your regexfile. Default is /etc/message-passing/regexfile. format Name of a defined format in your regexfile. capture ArrayRef of regex names which you want to capture and has been defined in your regexfile. note delete the prefix "%". SEE ALSO Capture Idea steal from Grok filter Config Format steal from use Config::Grammar AUTHOR chenryn, COPYRIGHT AND LICENSE Copyright (C) 2012 by chenryn This library is free software; you can redistribute it and/or modify it under the same terms as Perl itself, either Perl version 5.14.2 or, at your option, any later version of Perl 5 you may have available. Message-Passing-Filter-Regexp-0.05/Makefile.PL0000644000175000017500000000127612067354556020772 0ustar chenrynchenrynuse 5.014002; use ExtUtils::MakeMaker; # See lib/ExtUtils/MakeMaker.pm for details of how to influence # the contents of the Makefile that is written. WriteMakefile( NAME => 'Message::Passing::Filter::Regexp', VERSION_FROM => 'lib/Message/Passing/Filter/Regexp.pm', # finds $VERSION PREREQ_PM => { 'Message::Passing' => 0.11, 'Config::Tiny' => 0, 'Regexp::Log' => 0, 'JSON::Types' => 0, }, ($] >= 5.005 ? ## Add these new keywords supported since 5.005 (ABSTRACT_FROM => 'lib/Message/Passing/Filter/Regexp.pm', # retrieve abstract from module AUTHOR => 'chenryn ') : ()), ); Message-Passing-Filter-Regexp-0.05/META.yml0000664000175000017500000000110612167752622020260 0ustar chenrynchenryn--- abstract: 'Regexp Capture Filter For Message::Passing' author: - 'chenryn ' build_requires: ExtUtils::MakeMaker: 0 configure_requires: ExtUtils::MakeMaker: 0 dynamic_config: 1 generated_by: 'ExtUtils::MakeMaker version 6.66, CPAN::Meta::Converter version 2.120921' license: unknown meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html version: 1.4 name: Message-Passing-Filter-Regexp no_index: directory: - t - inc requires: Config::Tiny: 0 JSON::Types: 0 Message::Passing: 0.11 Regexp::Log: 0 version: 0.05 Message-Passing-Filter-Regexp-0.05/META.json0000664000175000017500000000177012167752622020437 0ustar chenrynchenryn{ "abstract" : "Regexp Capture Filter For Message::Passing", "author" : [ "chenryn " ], "dynamic_config" : 1, "generated_by" : "ExtUtils::MakeMaker version 6.66, CPAN::Meta::Converter version 2.120921", "license" : [ "unknown" ], "meta-spec" : { "url" : "http://search.cpan.org/perldoc?CPAN::Meta::Spec", "version" : "2" }, "name" : "Message-Passing-Filter-Regexp", "no_index" : { "directory" : [ "t", "inc" ] }, "prereqs" : { "build" : { "requires" : { "ExtUtils::MakeMaker" : "0" } }, "configure" : { "requires" : { "ExtUtils::MakeMaker" : "0" } }, "runtime" : { "requires" : { "Config::Tiny" : "0", "JSON::Types" : "0", "Message::Passing" : "0.11", "Regexp::Log" : "0" } } }, "release_status" : "stable", "version" : "0.05" }