pax_global_header00006660000000000000000000000064131303563640014516gustar00rootroot0000000000000052 comment=33a5564da6dabc579a41bf6d688eefbb51b46480 libsql-tokenizer-perl-0.24/000077500000000000000000000000001313035636400157015ustar00rootroot00000000000000libsql-tokenizer-perl-0.24/._Changes000066400000000000000000000002531313035636400174110ustar00rootroot00000000000000Mac OS X  2y«TEXTATTR¼`«œœcom.apple.TextEncodingUTF-8;134217984libsql-tokenizer-perl-0.24/Changes000066400000000000000000000024451313035636400172010ustar00rootroot00000000000000Revision history for SQL-Tokenizer 0.24 - Dramatically reduce memory usage for quote handling - Luc Lanthier 0.23 - Simplified regex and fix for back slashes in the end of a string - Luc Lanthier 0.22 - Better handling of CR+LF in comments - Devin Withers 0.21 - Better handling of $ placeholders - Nigel Metheringham 0.19 - Allow `` quoted strings (similar to "" and '') - Jonas Kramer - Add dot to punctuation characters - Jonas Kramer 0.17 - fixed issues with square brackets - now exports a function if required 0.16 - now supports database.schema.table dialect. - required at least 5.6.2. 0.15 - doesn't eat `@' anymore. 0.14 - tests passes now in 5.6.2. 0.13 - quote mark wasn't being properly tokenized (fixes DBIx::Placeholder::Named dependency). 0.12 - fixed #34889. 0.11 - added support for single line shell style comments as requested by Evan Harris. 0.10 - added support for SQL operators as requested by Evan Harris. - tokenize() returns an arrayref if called in scalar context, an array if called in list context as requested by Evan Harris. - added remove_white_spaces parameter to tokenize() as suggested by Evan Harris. 0.09 - removed use_ok from BEGIN block on tests (Test::More 0.74). 0.07 - fixed C style comments. 0.06 - first release. libsql-tokenizer-perl-0.24/MANIFEST000066400000000000000000000007261313035636400170370ustar00rootroot00000000000000Changes lib/SQL/Tokenizer.pm Makefile.PL MANIFEST MANIFEST.bak MANIFEST.SKIP META.yml # Will be created by "make dist" README t/00-load.t t/01-single-quotes.t t/02-double-quotes.t t/03-complex.t t/04-c-style-comments.t t/05-ansi-comments.t t/06-empty.t t/07-equality-and-math-operators.t t/08-remove-white-tokens.t t/09-shell-style-comments.t t/10-sql-operators.t t/11-name-sep.t t/98-dbix-placeholder-named.t t/99-rt34889.t t/99-dollars.t t/99-more-dollars.t t/pod.t libsql-tokenizer-perl-0.24/MANIFEST.SKIP000066400000000000000000000000711313035636400175750ustar00rootroot00000000000000.DS_Store .gitignore .git ^Makefile$ ^._ blib pm_to_blib libsql-tokenizer-perl-0.24/MANIFEST.bak000066400000000000000000000007441313035636400175730ustar00rootroot00000000000000Changes lib/SQL/Tokenizer.pm Makefile.old Makefile.PL MANIFEST MANIFEST.bak MANIFEST.SKIP META.yml # Will be created by "make dist" README SQL-Tokenizer-0.14.tar.gz SQL-Tokenizer-0.15.tar.gz t/00-load.t t/01-single-quotes.t t/02-double-quotes.t t/03-complex.t t/04-c-style-comments.t t/05-ansi-comments.t t/06-empty.t t/07-equality-and-math-operators.t t/08-remove-white-tokens.t t/09-shell-style-comments.t t/10-sql-operators.t t/98-dbix-placeholder-named.t t/99-rt34889.t t/pod.t libsql-tokenizer-perl-0.24/META.yml000066400000000000000000000006741313035636400171610ustar00rootroot00000000000000--- #YAML:1.0 name: SQL-Tokenizer version: 0.24 abstract: A simple SQL tokenizer. license: ~ author: - Igor Sutton Lopes generated_by: ExtUtils::MakeMaker version 6.42 distribution_type: module requires: Test::More: 0 meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.3.html version: 1.3 libsql-tokenizer-perl-0.24/Makefile.PL000066400000000000000000000007721313035636400176610ustar00rootroot00000000000000use strict; use warnings; use ExtUtils::MakeMaker; WriteMakefile( NAME => 'SQL::Tokenizer', AUTHOR => 'Igor Sutton Lopes ', VERSION_FROM => 'lib/SQL/Tokenizer.pm', ABSTRACT_FROM => 'lib/SQL/Tokenizer.pm', PL_FILES => {}, PREREQ_PM => { 'Test::More' => '0', }, dist => { COMPRESS => 'gzip -9f', SUFFIX => 'gz', }, clean => { FILES => 'SQL-Tokenizer-*' }, ); libsql-tokenizer-perl-0.24/README000066400000000000000000000016551313035636400165700ustar00rootroot00000000000000SQL-Tokenizer This module is a simple regular expressions based SQL tokenizer. It was created to make DBIx::Placeholder::Named possible. INSTALLATION To install this module, run the following commands: perl Makefile.PL make make test make install SUPPORT AND DOCUMENTATION After installing, you can find documentation for this module with the perldoc command. perldoc SQL::Tokenizer You can also look for information at: Search CPAN http://search.cpan.org/dist/SQL-Tokenizer CPAN Request Tracker: http://rt.cpan.org/NoAuth/Bugs.html?Dist=SQL-Tokenizer AnnoCPAN, annotated CPAN documentation: http://annocpan.org/dist/SQL-Tokenizer CPAN Ratings: http://cpanratings.perl.org/d/SQL-Tokenizer COPYRIGHT AND LICENCE Copyright (C) 2007 Igor Sutton Lopes This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. libsql-tokenizer-perl-0.24/lib/000077500000000000000000000000001313035636400164475ustar00rootroot00000000000000libsql-tokenizer-perl-0.24/lib/SQL/000077500000000000000000000000001313035636400171065ustar00rootroot00000000000000libsql-tokenizer-perl-0.24/lib/SQL/Tokenizer.pm000066400000000000000000000103601313035636400214160ustar00rootroot00000000000000package SQL::Tokenizer; use warnings; use strict; use 5.006002; use Exporter; our @ISA = qw(Exporter); our @EXPORT_OK= qw(tokenize_sql); our $VERSION= '0.24'; my $re= qr{ ( (?:--|\#)[\ \t\S]* # single line comments | (?:<>|<=>|>=|<=|==|=|!=|!|<<|>>|<|>|\|\||\||&&|&|-|\+|\*(?!/)|/(?!\*)|\%|~|\^|\?) # operators and tests | [\[\]\(\),;.] # punctuation (parenthesis, comma) | \'\'(?!\') # empty single quoted string | \"\"(?!\"") # empty double quoted string | "(?>(?:(?>[^"\\]+)|""|\\.)*)+" # anything inside double quotes, ungreedy | `(?>(?:(?>[^`\\]+)|``|\\.)*)+` # anything inside backticks quotes, ungreedy | '(?>(?:(?>[^'\\]+)|''|\\.)*)+' # anything inside single quotes, ungreedy. | /\*[\ \t\r\n\S]*?\*/ # C style comments | (?:[\w:@]+(?:\.(?:\w+|\*)?)*) # words, standard named placeholders, db.table.*, db.* | (?: \$_\$ | \$\d+ | \${1,2} ) # dollar expressions - eg $_$ $3 $$ | \n # newline | [\t\ ]+ # any kind of white spaces ) }smx; sub tokenize_sql { my ( $query, $remove_white_tokens )= @_; my @query= $query =~ m{$re}smxg; if ($remove_white_tokens) { @query= grep( !/^[\s\n\r]*$/, @query ); } return wantarray ? @query : \@query; } sub tokenize { my $class= shift; return tokenize_sql(@_); } 1; =pod =head1 NAME SQL::Tokenizer - A simple SQL tokenizer. =head1 VERSION 0.20 =head1 SYNOPSIS use SQL::Tokenizer qw(tokenize_sql); my $query= q{SELECT 1 + 1}; my @tokens= SQL::Tokenizer->tokenize($query); # @tokens now contains ('SELECT', ' ', '1', ' ', '+', ' ', '1') @tokens= tokenize_sql($query); # procedural interface =head1 DESCRIPTION SQL::Tokenizer is a simple tokenizer for SQL queries. It does not claim to be a parser or query verifier. It just creates sane tokens from a valid SQL query. It supports SQL with comments like: -- This query is used to insert a message into -- logs table INSERT INTO log (application, message) VALUES (?, ?) Also supports C<''>, C<""> and C<\'> escaping methods, so tokenizing queries like the one below should not be a problem: INSERT INTO log (application, message) VALUES ('myapp', 'Hey, this is a ''single quoted string''!') =head1 API =over 4 =item tokenize_sql use SQL::Tokenizer qw(tokenize_sql); my @tokens= tokenize_sql($query); my $tokens= tokenize_sql($query); $tokens= tokenize_sql( $query, $remove_white_tokens ); C can be imported to current namespace on request. It receives a SQL query, and returns an array of tokens if called in list context, or an arrayref if called in scalar context. =item tokenize my @tokens= SQL::Tokenizer->tokenize($query); my $tokens= SQL::Tokenizer->tokenize($query); $tokens= SQL::Tokenizer->tokenize( $query, $remove_white_tokens ); This is the only available class method. It receives a SQL query, and returns an array of tokens if called in list context, or an arrayref if called in scalar context. If C<$remove_white_tokens> is true, white spaces only tokens will be removed from result. =back =head1 ACKNOWLEDGEMENTS =over 4 =item Evan Harris, for implementing Shell comment style and SQL operators. =item Charlie Hills, for spotting a lot of important issues I haven't thought. =item Jonas Kramer, for fixing MySQL quoted strings and treating dot as punctuation character correctly. =item Emanuele Zeppieri, for asking to fix SQL::Tokenizer to support dollars as well. =item Nigel Metheringham, for extending the dollar signal support. =item Devin Withers, for making it not choke on CR+LF in comments. =item Luc Lanthier, for simplifying the regex and make it not choke on backslashes. =back =head1 AUTHOR Copyright (c) 2007, 2008, 2009, 2010, 2011 Igor Sutton Lopes "". All rights reserved. This module is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =cut libsql-tokenizer-perl-0.24/t/000077500000000000000000000000001313035636400161445ustar00rootroot00000000000000libsql-tokenizer-perl-0.24/t/00-load.t000066400000000000000000000002301313035636400174600ustar00rootroot00000000000000#!perl -T use Test::More tests => 1; BEGIN { use_ok( 'SQL::Tokenizer' ); } diag( "Testing SQL::Tokenizer $SQL::Tokenizer::VERSION, Perl $], $^X" ); libsql-tokenizer-perl-0.24/t/01-single-quotes.t000066400000000000000000000074021313035636400213510ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests= ( { description => qq{one C style scaped single quote}, query => 'INSERT INTO table VALUES( \'scaped \\\' single quote\', \'no quote\' )', wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'table', SPACE, 'VALUES', '(', SPACE, q{'scaped \' single quote'}, COMMA, SPACE, q{'no quote'}, SPACE, ')' ], }, { description => qq{no quotes inside string}, query => 'INSERT INTO table VALUES( \'no quote\', \'no quote either\' )', wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'table', SPACE, 'VALUES', '(', SPACE, q{'no quote'}, COMMA, SPACE, q{'no quote either'}, SPACE, ')' ], }, { description => qq{more than one C style escaped single quotes inside string}, query => q{INSERT INTO logs (program, message) VALUES (:program, 'Something \' with \' a \' lot \' of \' scaped quotes')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', COMMA, SPACE, 'message', ')', SPACE, 'VALUES', SPACE, '(', ':program', COMMA, SPACE, q{'Something \' with \' a \' lot \' of \' scaped quotes'}, ')' ], }, { description => qq{more than one C style escaped single quotes inside string, with extra backslashes}, query => q{INSERT INTO logs (program, message) VALUES (:program, 'Something \' with \' a \' lot \' of \' scaped quotes\\\\\\\\\\\\\\\\')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', COMMA, SPACE, 'message', ')', SPACE, 'VALUES', SPACE, '(', ':program', COMMA, SPACE, q{'Something \' with \' a \' lot \' of \' scaped quotes\\\\\\\\\\\\\\\\'}, ')' ], }, { description => qq{SQL style escaped single quotes}, query => q{INSERT INTO logs (program) VALUES ('single''quote')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{'single''quote'}, ')' ], }, { description => qq{SQL style escaped single quotes with surrounding spaces}, query => q{INSERT INTO logs (program) VALUES ('single '' quote '' ')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{'single '' quote '' '}, ')' ], }, { description => qq{C style escaped single quote at end of string}, query => q{INSERT INTO logs (program) VALUES ('single '' quote \'')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{'single '' quote \''}, ')' ], }, { description => qq{multiple SQL style escaped single quotes at end of string}, query => q{INSERT INTO logs (program) VALUES ('single '' quote ''''')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{'single '' quote '''''}, ')' ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized= SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/02-double-quotes.t000066400000000000000000000065101313035636400213420ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => qq{one C style scaped double quote}, query => "INSERT INTO table VALUES( \"scaped \\\" double quote\", \"no quote\" )", wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'table', SPACE, 'VALUES', '(', SPACE, q{"scaped \" double quote"}, COMMA, SPACE, q{"no quote"}, SPACE, ')' ], }, { description => qq{no quotes inside string}, query => "INSERT INTO table VALUES( \"no quote\", \"no quote either\" )", wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'table', SPACE, 'VALUES', '(', SPACE, q{"no quote"}, COMMA, SPACE, q{"no quote either"}, SPACE, ')' ], }, { description => qq{more than one C style escaped double quotes inside string}, query => q{INSERT INTO logs (program, message) VALUES (:program, "Something \" with \" a \" lot \" of \" scaped quotes")}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', COMMA, SPACE, 'message', ')', SPACE, 'VALUES', SPACE, '(', ':program', COMMA, SPACE, q{"Something \" with \" a \" lot \" of \" scaped quotes"}, ')' ], }, { description => qq{SQL style escaped double quotes}, query => q{INSERT INTO logs (program) VALUES ("double""quote")}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{"double""quote"}, ')' ], }, { description => qq{SQL style escaped double quotes with surrounding spaces}, query => q{INSERT INTO logs (program) VALUES ("double "" quote "" ")}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{"double "" quote "" "}, ')' ], }, { description => qq{C style escaped double quote at end of string}, query => q{INSERT INTO logs (program) VALUES ("single "" quote \"")}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', q{"single "" quote \""}, ')' ], }, { description => qq{multiple SQL style escaped double quotes at end of string}, query => qq{INSERT INTO logs (program) VALUES ("double "" quote """"")}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'logs', SPACE, '(', 'program', ')', SPACE, 'VALUES', SPACE, '(', qq{"double "" quote """""}, ')' ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/03-complex.t000066400000000000000000000312301313035636400202170ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => q{complex explain query}, query => q{EXPLAIN PLAN LEFT JOIN user.name, email.address, mobile.number WHERE user.id = 100 AND email.user = user.id AND mobile.user = user.id GROUP BY mobile.country_code HAVING mobile.country_code IN ( '55', '31', '44' )}, wanted => [ 'EXPLAIN', SPACE, 'PLAN', SPACE, 'LEFT', SPACE, 'JOIN', SPACE, 'user.name', COMMA, SPACE, 'email.address', COMMA, SPACE, 'mobile.number', SPACE, 'WHERE', SPACE, 'user.id', SPACE, '=', SPACE, '100', SPACE, 'AND', SPACE, 'email.user', SPACE, '=', SPACE, 'user.id', SPACE, 'AND', SPACE, 'mobile.user', SPACE, '=', SPACE, 'user.id', SPACE, 'GROUP', SPACE, 'BY', SPACE, 'mobile.country_code', SPACE, 'HAVING', SPACE, 'mobile.country_code', SPACE, 'IN', SPACE, '(', SPACE, q{'55'}, COMMA, SPACE, q{'31'}, COMMA, SPACE, q{'44'}, SPACE, ')' ], }, { description => q{SQL script}, query => < [ q{-- drop table}, NL, 'DROP', SPACE, 'TABLE', SPACE, 'test', ';', NL, q{-- create table}, NL, 'CREATE', SPACE, 'TABLE', SPACE, 'test', SPACE, '(', 'id', SPACE, 'INT', COMMA, SPACE, 'name', SPACE, 'VARCHAR', ')', ';', NL, q{-- insert data}, NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '1', COMMA, SPACE, q{'t'}, ')', ';', NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '2', COMMA, SPACE, q{'''quoted'''}, ')', ';', NL, ], }, { description => q{really long SQL query}, query => <<'EOQ', SELECT v.veiculo_id AS veiculo_id, v.imagem1 AS imagem1, v.imagem2 AS imagem2, v.imagem3 AS imagem3, v.imagem4 AS imagem4, v.combustivel AS combustivel, v.modelo_nome AS modelo_nome, COALESCE(m.nome,' ') AS marca, COALESCE(v.versao,' ') AS versao, COALESCE(v.placa,' ') AS placa, COALESCE(v.ano,'0') AS ano, COALESCE(v.modelo_ano,'0') AS modelo_ano, COALESCE(v.cor,' ') AS cor, COALESCE(v.portas,'0') AS portas, COALESCE(v.preco,'0.0') AS valor, c.cliente_id AS cliente_id FROM veiculo AS v INNER JOIN anuncio AS a ON a.veiculo_id = v.veiculo_id INNER JOIN cliente AS c ON c.cliente_id = a.cliente_id LEFT JOIN marca AS m ON v.marca_id = m.marca_id WHERE 1=1 AND ( (a.data_inicio <= '20070502' AND a.data_fim >= '20060502') OR (a.data_inicio IS NULL AND a.data_fim IS NULL) ) AND a.ativo = 1 AND v.veiculo_tipo_id = 3 AND v.imagem1 IS NOT NULL AND ( v.imagem1 is not null OR v.imagem2 is not null OR v.imagem3 is not null OR v.imagem4 is not null ) AND c.cliente_id = 12 ORDER BY v.preco ASC EOQ wanted => [ 'SELECT', NL, ' ', 'v.veiculo_id', SPACE, 'AS', SPACE, 'veiculo_id', COMMA, NL, ' ', 'v.imagem1', SPACE, 'AS', SPACE, 'imagem1', COMMA, NL, ' ', 'v.imagem2', SPACE, 'AS', SPACE, 'imagem2', COMMA, NL, ' ', 'v.imagem3', SPACE, 'AS', SPACE, 'imagem3', COMMA, NL, ' ', 'v.imagem4', SPACE, 'AS', SPACE, 'imagem4', COMMA, NL, ' ', 'v.combustivel', SPACE, 'AS', SPACE, 'combustivel', COMMA, NL, ' ', 'v.modelo_nome', SPACE, 'AS', SPACE, 'modelo_nome', COMMA, NL, ' ', 'COALESCE', '(', 'm.nome', COMMA, q{' '}, ')', SPACE, 'AS', SPACE, 'marca', COMMA, NL, ' ', 'COALESCE', '(', 'v.versao', COMMA, q{' '}, ')', SPACE, 'AS', SPACE, 'versao', COMMA, NL, ' ', 'COALESCE', '(', 'v.placa', COMMA, q{' '}, ')', SPACE, 'AS', SPACE, 'placa', COMMA, NL, ' ', 'COALESCE', '(', 'v.ano', COMMA, q{'0'}, ')', SPACE, 'AS', SPACE, 'ano', COMMA, NL, ' ', 'COALESCE', '(', 'v.modelo_ano', COMMA, q{'0'}, ')', SPACE, 'AS', SPACE, 'modelo_ano', COMMA, NL, ' ', 'COALESCE', '(', 'v.cor', COMMA, q{' '}, ')', SPACE, 'AS', SPACE, 'cor', COMMA, NL, ' ', 'COALESCE', '(', 'v.portas', COMMA, q{'0'}, ')', SPACE, 'AS', SPACE, 'portas', COMMA, NL, ' ', 'COALESCE', '(', 'v.preco', COMMA, q{'0.0'}, ')', SPACE, 'AS', SPACE, 'valor', COMMA, NL, ' ', 'c.cliente_id', SPACE, 'AS', SPACE, 'cliente_id', NL, 'FROM', NL, ' ', 'veiculo', SPACE, 'AS', SPACE, 'v', NL, 'INNER', SPACE, 'JOIN', NL, ' ', 'anuncio', SPACE, 'AS', SPACE, 'a', SPACE, 'ON', SPACE, 'a.veiculo_id', SPACE, '=', SPACE, 'v.veiculo_id', NL, 'INNER', SPACE, 'JOIN', NL, ' ', 'cliente', SPACE, 'AS', SPACE, 'c', SPACE, 'ON', SPACE, 'c.cliente_id', SPACE, '=', SPACE, 'a.cliente_id', NL, 'LEFT', SPACE, 'JOIN', NL, ' ', 'marca', SPACE, 'AS', SPACE, 'm', SPACE, 'ON', SPACE, 'v.marca_id', SPACE, '=', SPACE, 'm.marca_id', NL, 'WHERE', NL, ' ', '1', '=', '1', SPACE, 'AND', NL, ' ', '(', NL, ' ', '(', 'a.data_inicio', SPACE, '<=', SPACE, q{'20070502'}, SPACE, 'AND', SPACE, 'a.data_fim', SPACE, '>=', SPACE, q{'20060502'}, ')', SPACE, 'OR', NL, ' ', '(', 'a.data_inicio', SPACE, 'IS', SPACE, 'NULL', SPACE, 'AND', SPACE, 'a.data_fim', SPACE, 'IS', SPACE, 'NULL', ')', NL, ' ', ')', SPACE, 'AND', NL, ' ', 'a.ativo', SPACE, '=', SPACE, '1', SPACE, 'AND', NL, ' ', 'v.veiculo_tipo_id', SPACE, '=', SPACE, '3', SPACE, 'AND', NL, ' ', 'v.imagem1', SPACE, 'IS', SPACE, 'NOT', SPACE, 'NULL', SPACE, 'AND', NL, ' ', '(', NL, ' ', 'v.imagem1', SPACE, 'is', SPACE, 'not', SPACE, 'null', SPACE, 'OR', NL, ' ', 'v.imagem2', SPACE, 'is', SPACE, 'not', SPACE, 'null', SPACE, 'OR', NL, ' ', 'v.imagem3', SPACE, 'is', SPACE, 'not', SPACE, 'null', SPACE, 'OR', NL, ' ', 'v.imagem4', SPACE, 'is', SPACE, 'not', SPACE, 'null', NL, ' ', ')', SPACE, 'AND', NL, ' ', 'c.cliente_id', SPACE, '=', SPACE, '12', NL, 'ORDER', SPACE, 'BY', SPACE, 'v.preco', SPACE, 'ASC', NL ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/04-c-style-comments.t000066400000000000000000000154761313035636400217720ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => q{C style comment}, query => < [ q{/* drop table */}, NL, 'DROP', SPACE, 'TABLE', SPACE, 'test', ';', NL, q{/* create table */}, NL, 'CREATE', SPACE, 'TABLE', SPACE, 'test', SPACE, '(', 'id', SPACE, 'INT', COMMA, SPACE, 'name', SPACE, 'VARCHAR', ')', ';', NL, q{/* insert data */}, NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '1', COMMA, SPACE, q{'t'}, ')', ';', NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '2', COMMA, SPACE, q{'''quoted'''}, ')', ';', NL, ], }, { description => q{multi-line C style comment}, query => < [ qq{/*\n drop table\n*/}, NL, 'DROP', SPACE, 'TABLE', SPACE, 'test', ';', NL, qq{/*\n create table\n*/}, NL, 'CREATE', SPACE, 'TABLE', SPACE, 'test', SPACE, '(', 'id', SPACE, 'INT', COMMA, SPACE, 'name', SPACE, 'VARCHAR', ')', ';', NL, qq{/*\n insert data\n*/}, NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '1', COMMA, SPACE, q{'t'}, ')', ';', NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '2', COMMA, SPACE, q{'''quoted'''}, ')', ';', NL, ], }, { description => q{multi-line C style comment with CR+LF newline}, query => < [ qq{/*\r\n drop table\r\n*/}, NL, 'DROP', SPACE, 'TABLE', SPACE, 'test', ';', NL, qq{/*\r\n create table\r\n*/}, NL, 'CREATE', SPACE, 'TABLE', SPACE, 'test', SPACE, '(', 'id', SPACE, 'INT', COMMA, SPACE, 'name', SPACE, 'VARCHAR', ')', ';', NL, qq{/*\r\n insert data\r\n*/}, NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '1', COMMA, SPACE, q{'t'}, ')', ';', NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '2', COMMA, SPACE, q{'''quoted'''}, ')', ';', NL, ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/05-ansi-comments.t000066400000000000000000000035331313035636400213340ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => q{ANSI comments}, query => < [ 'DROP', SPACE, 'TABLE', SPACE, 'test', ';', SPACE, q{-- drop table}, NL, 'CREATE', SPACE, 'TABLE', SPACE, 'test', SPACE, '(', 'id', SPACE, 'INT', COMMA, SPACE, 'name', SPACE, 'VARCHAR', ')', ';', SPACE, q{-- create table}, NL, q{-- insert data}, NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '1', COMMA, SPACE, q{'t'}, ')', ';', NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '2', COMMA, SPACE, q{'''quoted'''}, ')', ';', NL, ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/06-empty.t000066400000000000000000000012631313035636400177140ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => qq{empty single quotes, RT #27797}, query => q{nvl(reward_type,'')='' and group_code = 'XXXX'}, wanted => [ 'nvl', '(', 'reward_type', ',', q{''}, ')', '=', q{''}, SPACE, 'and', SPACE, 'group_code', SPACE, '=', SPACE, q{'XXXX'} ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/07-equality-and-math-operators.t000066400000000000000000000022211313035636400241120ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => qq{equality and math operators}, query => q{SELECT a * 2, b / 3, c % 4 FROM table WHERE a <> b AND b >= c AND d <= c AND d <> a}, wanted => [ 'SELECT', SPACE, 'a', SPACE, '*', SPACE, '2', COMMA, SPACE, 'b', SPACE, '/', SPACE, '3', COMMA, SPACE, 'c', SPACE, '%', SPACE, '4', SPACE, 'FROM', SPACE, 'table', SPACE, 'WHERE', SPACE, 'a', SPACE, '<>', SPACE, 'b', SPACE, 'AND', SPACE, 'b', SPACE, '>=', SPACE, 'c', SPACE, 'AND', SPACE, 'd', SPACE, '<=', SPACE, 'c', SPACE, 'AND', SPACE, 'd', SPACE, '<>', SPACE, 'a' ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize($test->{query}); is_deeply(\@tokenized, $test->{wanted}, $test->{description}); } libsql-tokenizer-perl-0.24/t/08-remove-white-tokens.t000066400000000000000000000177351313035636400225070ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests= ( { description => qq{equality and math operators}, query => q{SELECT a * 2, b / 3, c % 4 FROM table WHERE a <> b AND b >= c AND d <= c AND d <> a}, wanted => [ 'SELECT', 'a', '*', '2', COMMA, 'b', '/', '3', COMMA, 'c', '%', '4', 'FROM', 'table', 'WHERE', 'a', '<>', 'b', 'AND', 'b', '>=', 'c', 'AND', 'd', '<=', 'c', 'AND', 'd', '<>', 'a' ], }, { description => q{complex explain query}, query => q{EXPLAIN PLAN LEFT JOIN user.name, email.address, mobile.number WHERE user.id = 100 AND email.user = user.id AND mobile.user = user.id GROUP BY mobile.country_code HAVING mobile.country_code IN ( '55', '31', '44' )}, wanted => [ 'EXPLAIN', 'PLAN', 'LEFT', 'JOIN', 'user.name', COMMA, 'email.address', COMMA, 'mobile.number', 'WHERE', 'user.id', '=', '100', 'AND', 'email.user', '=', 'user.id', 'AND', 'mobile.user', '=', 'user.id', 'GROUP', 'BY', 'mobile.country_code', 'HAVING', 'mobile.country_code', 'IN', '(', q{'55'}, COMMA, q{'31'}, COMMA, q{'44'}, ')' ], }, { description => q{SQL script}, query => < [ q{-- drop table}, 'DROP', 'TABLE', 'test', ';', q{-- create table}, 'CREATE', 'TABLE', 'test', '(', 'id', 'INT', COMMA, 'name', 'VARCHAR', ')', ';', q{-- insert data}, 'INSERT', 'INTO', 'test', '(', 'id', COMMA, 'name', ')', 'VALUES', '(', '1', COMMA, q{'t'}, ')', ';', 'INSERT', 'INTO', 'test', '(', 'id', COMMA, 'name', ')', 'VALUES', '(', '2', COMMA, q{'''quoted'''}, ')', ';', ], }, { description => q{really long SQL query}, query => <<'EOQ', SELECT v.veiculo_id AS veiculo_id, v.imagem1 AS imagem1, v.imagem2 AS imagem2, v.imagem3 AS imagem3, v.imagem4 AS imagem4, v.combustivel AS combustivel, v.modelo_nome AS modelo_nome, COALESCE(m.nome,' ') AS marca, COALESCE(v.versao,' ') AS versao, COALESCE(v.placa,' ') AS placa, COALESCE(v.ano,'0') AS ano, COALESCE(v.modelo_ano,'0') AS modelo_ano, COALESCE(v.cor,' ') AS cor, COALESCE(v.portas,'0') AS portas, COALESCE(v.preco,'0.0') AS valor, c.cliente_id AS cliente_id FROM veiculo AS v INNER JOIN anuncio AS a ON a.veiculo_id = v.veiculo_id INNER JOIN cliente AS c ON c.cliente_id = a.cliente_id LEFT JOIN marca AS m ON v.marca_id = m.marca_id WHERE 1=1 AND ( (a.data_inicio <= '20070502' AND a.data_fim >= '20060502') OR (a.data_inicio IS NULL AND a.data_fim IS NULL) ) AND a.ativo = 1 AND v.veiculo_tipo_id = 3 AND v.imagem1 IS NOT NULL AND ( v.imagem1 is not null OR v.imagem2 is not null OR v.imagem3 is not null OR v.imagem4 is not null ) AND c.cliente_id = 12 ORDER BY v.preco ASC EOQ wanted => [ 'SELECT', 'v.veiculo_id', 'AS', 'veiculo_id', COMMA, 'v.imagem1', 'AS', 'imagem1', COMMA, 'v.imagem2', 'AS', 'imagem2', COMMA, 'v.imagem3', 'AS', 'imagem3', COMMA, 'v.imagem4', 'AS', 'imagem4', COMMA, 'v.combustivel', 'AS', 'combustivel', COMMA, 'v.modelo_nome', 'AS', 'modelo_nome', COMMA, 'COALESCE', '(', 'm.nome', COMMA, q{' '}, ')', 'AS', 'marca', COMMA, 'COALESCE', '(', 'v.versao', COMMA, q{' '}, ')', 'AS', 'versao', COMMA, 'COALESCE', '(', 'v.placa', COMMA, q{' '}, ')', 'AS', 'placa', COMMA, 'COALESCE', '(', 'v.ano', COMMA, q{'0'}, ')', 'AS', 'ano', COMMA, 'COALESCE', '(', 'v.modelo_ano', COMMA, q{'0'}, ')', 'AS', 'modelo_ano', COMMA, 'COALESCE', '(', 'v.cor', COMMA, q{' '}, ')', 'AS', 'cor', COMMA, 'COALESCE', '(', 'v.portas', COMMA, q{'0'}, ')', 'AS', 'portas', COMMA, 'COALESCE', '(', 'v.preco', COMMA, q{'0.0'}, ')', 'AS', 'valor', COMMA, 'c.cliente_id', 'AS', 'cliente_id', 'FROM', 'veiculo', 'AS', 'v', 'INNER', 'JOIN', 'anuncio', 'AS', 'a', 'ON', 'a.veiculo_id', '=', 'v.veiculo_id', 'INNER', 'JOIN', 'cliente', 'AS', 'c', 'ON', 'c.cliente_id', '=', 'a.cliente_id', 'LEFT', 'JOIN', 'marca', 'AS', 'm', 'ON', 'v.marca_id', '=', 'm.marca_id', 'WHERE', '1', '=', '1', 'AND', '(', '(', 'a.data_inicio', '<=', q{'20070502'}, 'AND', 'a.data_fim', '>=', q{'20060502'}, ')', 'OR', '(', 'a.data_inicio', 'IS', 'NULL', 'AND', 'a.data_fim', 'IS', 'NULL', ')', ')', 'AND', 'a.ativo', '=', '1', 'AND', 'v.veiculo_tipo_id', '=', '3', 'AND', 'v.imagem1', 'IS', 'NOT', 'NULL', 'AND', '(', 'v.imagem1', 'is', 'not', 'null', 'OR', 'v.imagem2', 'is', 'not', 'null', 'OR', 'v.imagem3', 'is', 'not', 'null', 'OR', 'v.imagem4', 'is', 'not', 'null', ')', 'AND', 'c.cliente_id', '=', '12', 'ORDER', 'BY', 'v.preco', 'ASC', ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized= SQL::Tokenizer->tokenize( $test->{query}, 1 ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/09-shell-style-comments.t000066400000000000000000000035201313035636400226470ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => q{Shell style comments}, query => < [ 'DROP', SPACE, 'TABLE', SPACE, 'test', ';', SPACE, q{# drop table}, NL, 'CREATE', SPACE, 'TABLE', SPACE, 'test', SPACE, '(', 'id', SPACE, 'INT', COMMA, SPACE, 'name', SPACE, 'VARCHAR', ')', ';', SPACE, q{### create table}, NL, q{# insert data}, NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '1', COMMA, SPACE, q{'t'}, ')', ';', NL, 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '2', COMMA, SPACE, q{'''quoted'''}, ')', ';', NL, ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized = SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/10-sql-operators.t000066400000000000000000000016661313035636400213730ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests= ( { description => q{!}, query => qq{SELECT !1}, wanted => [ 'SELECT', SPACE, '!', 1 ], }, { description => q{Negative number}, query => qq{SELECT -1}, wanted => [ 'SELECT', SPACE, '-', 1 ], } ); for my $operator (qw( - + / * <=> <= >= < > <> != = == % ~ & ^ & && | || << >> )) { push @tests, { description => qq{$operator operator}, query => qq{SELECT 1${operator}2}, wanted => [ 'SELECT', SPACE, 1, $operator, 2 ], }; } plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized= SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } __END__ libsql-tokenizer-perl-0.24/t/11-name-sep.t000066400000000000000000000013431313035636400202560ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests = ( { description => q{PostgreSQL style}, query => qq{SELECT "foo"."bar"}, wanted => [ 'SELECT', SPACE, '"foo"', '.', '"bar"' ], }, { description => q{MySQL style}, query => qq{SELECT `foo`.`bar`}, wanted => [ 'SELECT', SPACE, '`foo`', '.', '`bar`' ], } ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized= SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } __END__ libsql-tokenizer-perl-0.24/t/98-dbix-placeholder-named.t000066400000000000000000000014161313035636400230610ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests= ( { description => q{Selecting all fields from db.table}, query => q{INSERT INTO test (id, name) VALUES (?, '0000-00-00 11:11:11')}, wanted => [ 'INSERT', SPACE, 'INTO', SPACE, 'test', SPACE, '(', 'id', COMMA, SPACE, 'name', ')', SPACE, 'VALUES', SPACE, '(', '?', COMMA, SPACE, q{'0000-00-00 11:11:11'}, ')' ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized= SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/99-dollars.t000066400000000000000000000007601313035636400202330ustar00rootroot00000000000000#!perl use strict; use warnings; use SQL::Tokenizer qw(tokenize_sql); use Test::More tests => 1; my $sql_code = <<'SQL'; CREATE OR REPLACE FUNCTION fib ( fib_for integer ) RETURNS integer AS $$ BEGIN IF fib_for < 2 THEN RETURN fib_for; END IF; RETURN fib(fib_for - 2) + fib(fib_for - 1); END; $$ LANGUAGE plpgsql; SQL my @tokens = tokenize_sql($sql_code); my @dollar_tokens = grep { /\$/ } @tokens; cmp_ok( @dollar_tokens, '==', 2, 'Dollar tokens found' ); libsql-tokenizer-perl-0.24/t/99-more-dollars.t000066400000000000000000000014561313035636400211760ustar00rootroot00000000000000#!perl use strict; use warnings; use SQL::Tokenizer qw(tokenize_sql); use Test::More tests => 4; my $sql_code = <<'SQL'; CREATE OR REPLACE FUNCTION getdatastore(integer, integer) RETURNS SETOF type_datastore AS $_$ SELECT id, name FROM data_store WHERE storage_class = $1 AND data_centre = $2 AND active = true ORDER BY name; $_$ LANGUAGE sql; SQL my @tokens = tokenize_sql($sql_code); is( scalar( grep {/\$/} @tokens ), 4, 'Dollar token count correct' ); # old code would have 6 here is( scalar( grep { $_ eq '$_$' } @tokens ), 2, '$_$ token count correct' ); is( scalar( grep { $_ eq '$1' } @tokens ), 1, '$1 token count correct' ); is( scalar( grep { $_ eq '$2' } @tokens ), 1, '$2 token count correct' ); libsql-tokenizer-perl-0.24/t/99-rt34889.t000066400000000000000000000035121313035636400176360ustar00rootroot00000000000000use strict; use warnings; use Test::More; use SQL::Tokenizer; use constant SPACE => ' '; use constant COMMA => ','; use constant NL => "\n"; my $query; my @query; my @tokenized; my @tests= ( { description => q{rt34889}, query => q{SELECT t.* FROM table t}, wanted => [ 'SELECT', SPACE, 't.*', SPACE, 'FROM', SPACE, 'table', SPACE, 't' ], }, { description => q{Selecting all fields from db.table}, query => q{SELECT t.* FROM db.table t}, wanted => [ 'SELECT', SPACE, 't.*', SPACE, 'FROM', SPACE, 'db.table', SPACE, 't' ], }, { description => q{Selecting all fields from db.table}, query => q{SELECT db.table.* FROM db.table}, wanted => [ 'SELECT', SPACE, 'db.table.*', SPACE, 'FROM', SPACE, 'db.table' ], }, { description => q{Storing to a variable}, query => q{SELECT @id = column FROM db.table}, wanted => [ 'SELECT', SPACE, '@id', SPACE, '=', SPACE, 'column', SPACE, 'FROM', SPACE, 'db.table' ], }, { description => q{Declare a variable}, query => q{DECLARE @id INT}, wanted => [ 'DECLARE', SPACE, '@id', SPACE, 'INT' ], }, { description => q{Declare a variable with two @}, query => q{DECLARE @@id INT}, wanted => [ 'DECLARE', SPACE, '@@id', SPACE, 'INT' ], }, { description => q{SQL full notation for DATABASE.SCHEMA.TABLE}, query => q{SELECT SUM(column) FROM database..table}, wanted => [ 'SELECT', SPACE, 'SUM', '(', 'column', ')', SPACE,'FROM', SPACE, 'database..table' ], }, ); plan tests => scalar @tests; foreach my $test (@tests) { my @tokenized= SQL::Tokenizer->tokenize( $test->{query} ); is_deeply( \@tokenized, $test->{wanted}, $test->{description} ); } libsql-tokenizer-perl-0.24/t/pod.t000066400000000000000000000002141313035636400171100ustar00rootroot00000000000000#!perl -T use Test::More; eval "use Test::Pod 1.14"; plan skip_all => "Test::Pod 1.14 required for testing POD" if $@; all_pod_files_ok();