kpeg-1.0.0/0000755000175000017500000000000012321266363011104 5ustar domidomikpeg-1.0.0/Rakefile0000644000175000017500000000152412321266363012553 0ustar domidomi# -*- ruby -*- require 'rubygems' require 'hoe' Hoe.plugin :gemspec Hoe.plugin :git Hoe.plugin :minitest Hoe.plugin :travis Hoe.spec 'kpeg' do self.readme_file = "README.rdoc" developer 'Evan Phoenix', 'evan@fallingsnow.net' end task :test => :parser task :grammar do require 'kpeg' require 'kpeg/format' require 'kpeg/grammar_renderer' gr = KPeg::GrammarRenderer.new(KPeg::FORMAT) gr.render(STDOUT) end rule ".rb" => ".kpeg" do |t| ruby "-Ilib bin/kpeg -s -o #{t.name} -f #{t.source}" end PARSER_FILES = %w[ lib/kpeg/string_escape.rb lib/kpeg/format_parser.rb ] PARSER_FILES.map do |parser_file| file parser_file => 'lib/kpeg/compiled_parser.rb' file parser_file => 'lib/kpeg/code_generator.rb' file parser_file => 'lib/kpeg/position.rb' end desc "build the parser" task :parser => PARSER_FILES # vim: syntax=ruby kpeg-1.0.0/metadata.yml0000644000175000017500000000763612321266363013423 0ustar domidomi--- !ruby/object:Gem::Specification name: kpeg version: !ruby/object:Gem::Version version: 1.0.0 platform: ruby authors: - Evan Phoenix autorequire: bindir: bin cert_chain: [] date: 2014-01-04 00:00:00.000000000 Z dependencies: - !ruby/object:Gem::Dependency name: minitest requirement: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: '5.2' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: '5.2' - !ruby/object:Gem::Dependency name: rdoc requirement: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: '4.0' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: '4.0' - !ruby/object:Gem::Dependency name: hoe requirement: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: '3.7' type: :development prerelease: false version_requirements: !ruby/object:Gem::Requirement requirements: - - ~> - !ruby/object:Gem::Version version: '3.7' description: |- KPeg is a simple PEG library for Ruby. It provides an API as well as native grammar to build the grammar. KPeg strives to provide a simple, powerful API without being too exotic. KPeg supports direct left recursion of rules via the {OMeta memoization}[http://www.vpri.org/pdf/tr2008003_experimenting.pdf] trick. email: - evan@fallingsnow.net executables: - kpeg extensions: [] extra_rdoc_files: - History.txt - Manifest.txt - README.rdoc - examples/phone_number/README.md - examples/upper/README.md files: - .autotest - .travis.yml - History.txt - LICENSE - Manifest.txt - README.rdoc - Rakefile - bin/kpeg - examples/calculator/calculator.kpeg - examples/calculator/calculator.rb - examples/foreign_reference/literals.kpeg - examples/foreign_reference/matcher.kpeg - examples/foreign_reference/matcher.rb - examples/lua_string/driver.rb - examples/lua_string/lua_string.kpeg - examples/lua_string/lua_string.kpeg.rb - examples/phone_number/README.md - examples/phone_number/phone_number.kpeg - examples/phone_number/phone_number.rb - examples/upper/README.md - examples/upper/upper.kpeg - examples/upper/upper.rb - kpeg.gemspec - lib/hoe/kpeg.rb - lib/kpeg.rb - lib/kpeg/code_generator.rb - lib/kpeg/compiled_parser.rb - lib/kpeg/format_parser.kpeg - lib/kpeg/format_parser.rb - lib/kpeg/grammar.rb - lib/kpeg/grammar_renderer.rb - lib/kpeg/match.rb - lib/kpeg/parser.rb - lib/kpeg/position.rb - lib/kpeg/string_escape.kpeg - lib/kpeg/string_escape.rb - test/inputs/comments.kpeg - test/test_kpeg.rb - test/test_kpeg_code_generator.rb - test/test_kpeg_compiled_parser.rb - test/test_kpeg_format.rb - test/test_kpeg_format_parser_round_trip.rb - test/test_kpeg_grammar.rb - test/test_kpeg_grammar_renderer.rb - vim/syntax_kpeg/ftdetect/kpeg.vim - vim/syntax_kpeg/syntax/kpeg.vim - test/test_kpeg_string_escape.rb - .gemtest homepage: https://github.com/evanphx/kpeg licenses: - MIT metadata: {} post_install_message: rdoc_options: - --main - README.rdoc require_paths: - lib required_ruby_version: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' required_rubygems_version: !ruby/object:Gem::Requirement requirements: - - '>=' - !ruby/object:Gem::Version version: '0' requirements: [] rubyforge_project: kpeg rubygems_version: 2.1.10 signing_key: specification_version: 4 summary: KPeg is a simple PEG library for Ruby test_files: - test/test_kpeg.rb - test/test_kpeg_code_generator.rb - test/test_kpeg_compiled_parser.rb - test/test_kpeg_format.rb - test/test_kpeg_format_parser_round_trip.rb - test/test_kpeg_grammar.rb - test/test_kpeg_grammar_renderer.rb - test/test_kpeg_string_escape.rb kpeg-1.0.0/bin/0000755000175000017500000000000012321266363011654 5ustar domidomikpeg-1.0.0/bin/kpeg0000755000175000017500000000465512321266363012542 0ustar domidomi#!/usr/bin/env ruby require 'kpeg' require 'kpeg/code_generator' require 'kpeg/format_parser' require 'kpeg/grammar_renderer' require 'optparse' options = {} OptionParser.new do |o| o.banner = "Usage: kpeg [options]" o.on("-t", "--test", "Syntax check the file only") do |v| options[:test] = v end o.on("--reformat", "Reformat your grammar and write it back out") do options[:reformat] = true end o.on("-o", "--output FILE", "Where the output should go") do |v| options[:output] = v end o.on("-n", "--name NAME", "Class name to use for the parser") do |v| options[:name] = v end o.on("-f", "--force", "Overwrite the output if it exists") do |v| options[:force] = v end o.on("-s", "--stand-alone", "Write the parser to run standalone") do |v| options[:standalone] = v end o.on("-v", "--[no-]verbose", "Run verbosely") do |v| options[:verbose] = v end o.on("-d", "--debug", "Debug parsing the file") do |v| options[:debug] = v end end.parse! file = ARGV.shift unless File.exists?(file) puts "File '#{file}' does not exist" exit 1 end parser = KPeg::FormatParser.new File.read(file), true unless m = parser.parse puts "Syntax error in grammar #{file}" parser.show_error exit 1 end grammar = parser.grammar if options[:reformat] if !options[:output] puts "Please specify -o for where to write the new grammar" exit 1 end output = options[:output] if File.exists?(output) and !options[:force] puts "Output '#{output}' already exists, not overwriting (use -f)" exit 1 end rend = KPeg::GrammarRenderer.new(parser.grammar) File.open output, "w" do |f| rend.render(f) end puts "Wrote reformatted output to #{output}" exit 0 end if !options[:test] and !options[:name] unless name = grammar.variables["name"] puts "Please specify -n" exit 1 end else name = options[:name] end if options[:output] new_path = options[:output] else new_path = "#{file}.rb" end if !options[:test] and File.exists?(new_path) and !options[:force] puts "Path #{new_path} already exists, not overwriting\n" exit 1 end if options[:test] puts "Syntax ok" if options[:debug] gr = KPeg::GrammarRenderer.new(grammar) gr.render(STDOUT) end exit 0 end cg = KPeg::CodeGenerator.new name, grammar cg.standalone = options[:standalone] output = cg.output open new_path, "w" do |io| io << output end puts "Wrote #{name} to #{new_path}" kpeg-1.0.0/checksums.yaml.gz0000444000175000017500000000041612321266363014373 0ustar domidomivURe;R@D=^`->0lff `#k#O︡VQhoUw686"`DaXsÿण2)I潣B^]/th"hRZQMf\ dgX)){Ҥ^&\0sd0egٶ;8؍uZ .Y{#j <=10cN0F (1wH>1A.?IZWkpeg-1.0.0/Manifest.txt0000644000175000017500000000223512321266363013415 0ustar domidomi.autotest .travis.yml History.txt LICENSE Manifest.txt README.rdoc Rakefile bin/kpeg examples/calculator/calculator.kpeg examples/calculator/calculator.rb examples/foreign_reference/literals.kpeg examples/foreign_reference/matcher.kpeg examples/foreign_reference/matcher.rb examples/lua_string/driver.rb examples/lua_string/lua_string.kpeg examples/lua_string/lua_string.kpeg.rb examples/phone_number/README.md examples/phone_number/phone_number.kpeg examples/phone_number/phone_number.rb examples/upper/README.md examples/upper/upper.kpeg examples/upper/upper.rb kpeg.gemspec lib/hoe/kpeg.rb lib/kpeg.rb lib/kpeg/code_generator.rb lib/kpeg/compiled_parser.rb lib/kpeg/format_parser.kpeg lib/kpeg/format_parser.rb lib/kpeg/grammar.rb lib/kpeg/grammar_renderer.rb lib/kpeg/match.rb lib/kpeg/parser.rb lib/kpeg/position.rb lib/kpeg/string_escape.kpeg lib/kpeg/string_escape.rb test/inputs/comments.kpeg test/test_kpeg.rb test/test_kpeg_code_generator.rb test/test_kpeg_compiled_parser.rb test/test_kpeg_format.rb test/test_kpeg_format_parser_round_trip.rb test/test_kpeg_grammar.rb test/test_kpeg_grammar_renderer.rb vim/syntax_kpeg/ftdetect/kpeg.vim vim/syntax_kpeg/syntax/kpeg.vim kpeg-1.0.0/lib/0000755000175000017500000000000012321266363011652 5ustar domidomikpeg-1.0.0/lib/kpeg.rb0000644000175000017500000000166312321266363013133 0ustar domidomimodule KPeg VERSION = "1.0.0" def self.grammar g = Grammar.new yield g g end def self.match(str, gram) scan = Parser.new(str, gram) scan.parse end def self.load_grammar(file, log=false) parser = KPeg::FormatParser.new File.read(file) if !parser.parse parser.raise_error end return parser.grammar end def self.load(file, name) grammar = load_grammar(file) cg = KPeg::CodeGenerator.new name, grammar code = cg.output warn "[Loading parser '#{name}' => #{code.size} bytes]" Object.module_eval code true end def self.compile(str, name, scope=Object) parser = KPeg::FormatParser.new str unless parser.parse parser.raise_error end cg = KPeg::CodeGenerator.new name, parser.grammar code = cg.output scope.module_eval code true end end require 'kpeg/grammar' require 'kpeg/format_parser' require 'kpeg/code_generator' kpeg-1.0.0/lib/hoe/0000755000175000017500000000000012321266363012425 5ustar domidomikpeg-1.0.0/lib/hoe/kpeg.rb0000644000175000017500000000531612321266363013705 0ustar domidomi## # Kpeg plugin for hoe. # # === Tasks Provided: # # parser :: Generate parsers for all .kpeg files in your manifest # .kpeg -> .rb rule :: Generate a parser using kpeg. # # NOTE: This plugin is derived from the Hoe::Racc and used under the MIT # license: # # Copyright (c) Ryan Davis, seattle.rb # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. module Hoe::Kpeg ## # Optional: Defines what tasks need to generate parsers first. # # Defaults to [:multi, :test, :check_manifest] # # If you have extra tasks that require your parser to be built, add their # names here in your hoe spec. eg: # # kpeg_tasks << :debug attr_accessor :kpeg_tasks ## # Optional: Defines what flags to use for kpeg. default: "-s -v" attr_accessor :kpeg_flags ## # Initialize variables for kpeg plugin. def initialize_kpeg self.kpeg_tasks = [:multi, :test, :check_manifest] # -f = overwrite existing file # -s = parser does not require runtime # -v = verbose self.kpeg_flags ||= "-s -v -f" dependency 'kpeg', '~> 0.9', :development end ## # Define tasks for kpeg plugin def define_kpeg_tasks kpeg_files = self.spec.files.find_all { |f| f =~ /\.kpeg$/ } parser_files = kpeg_files.map { |f| f.sub(/\.kpeg$/, ".rb") } self.clean_globs += parser_files rule ".rb" => ".kpeg" do |t| kpeg = Gem.bin_path "kpeg", "kpeg" begin ruby "-rubygems #{kpeg} #{kpeg_flags} -o #{t.name} #{t.source}" rescue abort "need kpeg, please run rake check_extra_deps" end end desc "build the parser" unless parser_files.empty? task :parser task :parser => parser_files kpeg_tasks.each do |t| task t => :parser end end end kpeg-1.0.0/lib/kpeg/0000755000175000017500000000000012321266363012600 5ustar domidomikpeg-1.0.0/lib/kpeg/grammar_renderer.rb0000644000175000017500000001021712321266363016442 0ustar domidomirequire 'kpeg/string_escape' module KPeg class GrammarRenderer def initialize(gram) @grammar = gram end def render(io) widest = @grammar.rules.keys.sort { |a,b| a.size <=> b.size }.last indent = widest.size @grammar.variables.sort.each do |name, value| io.print "%% #{name} = #{value}\n" end unless @grammar.variables.empty? io.print "\n" end @grammar.directives.sort_by { |name,| name }.each do |name, act| io.print "%% #{name} {" io.print act.action io.print "}\n\n" end @grammar.setup_actions.each do |act| io.print "%% {" io.print act.action io.print "}\n\n" end @grammar.rule_order.each do |name| rule = @grammar.find(name) io.print(' ' * (indent - name.size)) io.print "#{name} = " op = rule.op if op.kind_of? Choice op.ops.each_with_index do |r,idx| unless idx == 0 io.print "\n#{' ' * (indent+1)}| " end render_op io, r end else render_op io, op end io.puts end end def parens?(op) case op when Sequence, AndPredicate, NotPredicate return true end false end def self.escape(str, embed=false) parc = StringEscape.new(str) rule = (embed ? "embed" : nil) unless parc.parse(rule) parc.raise_error end return parc.text end def render_op(io, op) case op when Dot io.print "." when LiteralString esc = GrammarRenderer.escape op.string io.print '"' io.print esc io.print '"' when LiteralRegexp io.print op.regexp.inspect when CharRange io.print "[#{op.start}-#{op.fin}]" when Sequence op.ops.each_with_index do |r,idx| unless idx == 0 io.print " " end render_op io, r end when Choice io.print "(" op.ops.each_with_index do |r,idx| unless idx == 0 io.print " | " end render_op io, r end io.print ")" when Multiple if parens?(op.op) io.print "(" render_op io, op.op io.print ")" else render_op io, op.op end if op.max if op.min == 0 and op.max == 1 io.print "?" else io.print "[#{op.min}, #{op.max}]" end elsif op.min == 0 io.print "*" elsif op.min == 1 io.print "+" else io.print "[#{op.min},*]" end when AndPredicate io.print "&" if parens?(op.op) io.print "(" render_op io, op.op io.print ")" else render_op io, op.op end when NotPredicate io.print "!" if parens?(op.op) io.print "(" render_op io, op.op io.print ")" else render_op io, op.op end when RuleReference if op.arguments io.print "#{op.rule_name}#{op.arguments}" else io.print "#{op.rule_name}" end when InvokeRule if op.arguments io.print "@#{op.rule_name}#{op.arguments}" else io.print "@#{op.rule_name}" end when ForeignInvokeRule if op.arguments io.print "%#{op.grammar_name}.#{op.rule_name}#{op.arguments}" else io.print "%#{op.grammar_name}.#{op.rule_name}" end when Tag if parens?(op.op) io.print "(" render_op io, op.op io.print ")" else render_op io, op.op end if op.tag_name io.print ":#{op.tag_name}" end when Action io.print "{#{op.action}}" when Collect io.print "< " render_op io, op.op io.print " >" when Bounds io.print "@< " render_op io, op.op io.print " >" else raise "Unknown op type - #{op.class}" end end end end kpeg-1.0.0/lib/kpeg/grammar.rb0000644000175000017500000003607612321266363014567 0ustar domidomirequire 'strscan' require 'kpeg/parser' require 'kpeg/match' module KPeg class Rule def initialize(name, op, args=nil) @name = name @op = op @arguments = args end attr_reader :name, :op, :arguments end class Operator def initialize @action = nil @has_tags = false end attr_accessor :action def set_action(act) @action = act end def detect_tags(ops) tags = [] ops.each_with_index do |r,idx| if r.kind_of?(Tag) @has_tags = true tags << idx end end @tags = tags if @has_tags end def prune_values(values) return values unless @has_tags return values.values_at(*@tags) end def inspect_type(tag, body) "#<#{tag} #{body}>" end def |(other) Choice.new(self, Grammar.resolve(other)) end end class Dot < Operator def match(x) if str = x.get_byte MatchString.new(self, str) else x.fail(self) end end def ==(obj) Dot === obj ? true : false end def inspect "#" end end class LiteralString < Operator def initialize(str) super() @string = str @reg = Regexp.new Regexp.quote(str) end attr_reader :string def match(x) if str = x.scan(@reg) MatchString.new(self, str) else x.fail(self) end end def ==(obj) case obj when LiteralString @string == obj.string else super end end def inspect inspect_type 'str', @string.inspect end end class LiteralRegexp < Operator def initialize(reg, opts=nil) super() if reg.kind_of? String flags = 0 lang = nil if opts opts.split("").each do |o| case o when "n", "N", "e", "E", "s", "S" lang = o.downcase when "u", "U" if RUBY_VERSION > "1.8.7" # Ruby 1.9 defaults to UTF-8 for string matching lang = "" else lang = "u" end when "m" flags |= Regexp::MULTILINE when "x" flags |= Regexp::EXTENDED when "i" flags |= Regexp::IGNORECASE end end end @regexp = Regexp.new(reg, flags, lang) else @regexp = reg end end attr_reader :regexp def string @regexp.source end def match(x) if str = x.scan(@regexp) MatchString.new(self, str) else x.fail(self) end end def ==(obj) case obj when LiteralRegexp @regexp == obj.regexp else super end end def inspect inspect_type 'reg', @regexp.inspect end end class CharRange < Operator def initialize(start, fin) super() @start = start @fin = fin @regexp = Regexp.new "[#{Regexp.quote start}-#{Regexp.quote fin}]" end attr_reader :start, :fin def string @regexp.source end def match(x) if str = x.scan(@regexp) MatchString.new(self, str) else x.fail(self) end end def ==(obj) case obj when CharRange @start == obj.start and @fin == obj.fin else super end end def inspect inspect_type 'range', "#{@start}-#{@fin}" end end class Choice < Operator def initialize(*many) super() @ops = many end attr_reader :ops def |(other) @ops << Grammar.resolve(other) self end def match(x) pos = x.pos @ops.each do |c| if m = c.match(x) return m end x.pos = pos end return nil end def ==(obj) case obj when Choice @ops == obj.ops else super end end def inspect inspect_type "any", @ops.map { |i| i.inspect }.join(' | ') end end class Multiple < Operator def initialize(op, min, max) super() @op = op @min = min @max = max @save_values = nil end attr_reader :op, :min, :max, :save_values def save_values! @save_values = true end def match(x) n = 0 matches = [] start = x.pos while true if m = @op.match(x) matches << m else break end n += 1 if @max and n > @max x.pos = start return nil end end if n >= @min return MatchComposition.new(self, matches) end x.pos = start return nil end def ==(obj) case obj when Multiple @op == obj.op and @min == obj.min and @max == obj.max else super end end def inspect inspect_type "multi", "#{@min} #{@max ? @max : "*"} #{@op.inspect}" end end class Sequence < Operator def initialize(*ops) super() @ops = ops detect_tags ops end attr_reader :ops def match(x) start = x.pos matches = @ops.map do |n| m = n.match(x) unless m x.pos = start return nil end m end MatchComposition.new(self, matches) end def ==(obj) case obj when Sequence @ops == obj.ops else super end end def inspect inspect_type "seq", @ops.map { |i| i.inspect }.join(' ') end end class AndPredicate < Operator def initialize(op) super() @op = op end attr_reader :op def match(x) pos = x.pos m = @op.match(x) x.pos = pos return m ? MatchString.new(self, "") : nil end def ==(obj) case obj when AndPredicate @op == obj.op else super end end def inspect inspect_type "andp", @op.inspect end end class NotPredicate < Operator def initialize(op) super() @op = op end attr_reader :op def match(x) pos = x.pos m = @op.match(x) x.pos = pos return m ? nil : MatchString.new(self, "") end def ==(obj) case obj when NotPredicate @op == obj.op else super end end def inspect inspect_type "notp", @op.inspect end end class RuleReference < Operator def initialize(name, grammar=nil, args=nil) super() @rule_name = name @grammar = grammar @arguments = args end attr_reader :rule_name, :arguments def match(x) if @grammar and @grammar != x.grammar x.switch_grammar(@grammar) do rule = @grammar.find(@rule_name) raise "Unknown rule: '#{@rule_name}'" unless rule x.apply rule end else rule = x.grammar.find(@rule_name) raise "Unknown rule: '#{@rule_name}'" unless rule x.apply rule end end def ==(obj) case obj when RuleReference @rule_name == obj.rule_name and @arguments == obj.arguments else super end end def inspect if @arguments body = "#{@rule_name} #{@arguments}" else body = @rule_name end inspect_type "ref", body end end class InvokeRule < Operator def initialize(name, args=nil) super() @rule_name = name @arguments = args end attr_reader :rule_name, :arguments def match(x) rule = x.grammar.find(@rule_name) raise "Unknown rule: '#{@rule_name}'" unless rule x.invoke rule end def ==(obj) case obj when InvokeRule @rule_name == obj.rule_name and @arguments == obj.arguments else super end end def inspect if @arguments body = "#{@rule_name} #{@arguments}" else body = @rule_name end inspect_type "invoke", body end end class ForeignInvokeRule < Operator def initialize(grammar, name, args=nil) super() @grammar_name = grammar @rule_name = name if !args or args.empty? @arguments = nil else @arguments = args end end attr_reader :grammar_name, :rule_name, :arguments def match(x) rule = x.grammar.find(@rule_name) raise "Unknown rule: '#{@rule_name}'" unless rule x.invoke rule end def ==(obj) case obj when ForeignInvokeRule @grammar_name == obj.grammar_name and \ @rule_name == obj.rule_name and @arguments == obj.arguments else super end end def inspect if @arguments body = "%#{@grammar}.#{@rule_name} #{@arguments}" else body = "%#{@grammar}.#{@rule_name}" end inspect_type "invoke", body end end class Tag < Operator def initialize(op, tag_name) super() if op.kind_of? Multiple op.save_values! end @op = op @tag_name = tag_name end attr_reader :op, :tag_name def match(x) if m = @op.match(x) MatchComposition.new(self, [m]) end end def ==(obj) case obj when Tag @op == obj.op and @tag_name == obj.tag_name when Operator @op == obj else super end end def inspect if @tag_name body = "@#{tag_name} " else body = "" end body << @op.inspect inspect_type "tag", body end end class Action < Operator def initialize(action) super() @action = action end attr_reader :action def match(x) return MatchString.new(self, "") end def ==(obj) case obj when Action @action == obj.action else super end end def inspect inspect_type "action", "=> #{action.inspect}" end end class Collect < Operator def initialize(op) super() @op = op end attr_reader :op def match(x) start = x.pos if @op.match(x) MatchString.new(self, x.string[start..x.pos]) end end def ==(obj) case obj when Collect @op == obj.op else super end end def inspect inspect_type "collect", @op.inspect end end class Bounds < Operator def initialize(op) super() @op = op end attr_reader :op def ==(obj) case obj when Bounds @op == obj.op else super end end def inspect inspect_type "bounds", @op.inspect end end class Grammar def initialize @directives = {} @rules = {} @rule_order = [] @setup_actions = [] @foreign_grammars = {} @variables = {} end attr_reader :directives attr_reader :rules, :rule_order, :setup_actions, :foreign_grammars attr_reader :variables def add_directive(name, body) if @directives.include? name warn "directive #{name.inspect} appears more than once" end @directives[name] = body end def add_setup(act) @setup_actions << act end def add_foreign_grammar(name, str) @foreign_grammars[name] = str end def set_variable(name, val) @variables[name] = val end def root @rules["root"] end def set(name, op, args=nil) if @rules.key? name raise "Already set rule named '#{name}'" end op = Grammar.resolve(op) @rule_order << name rule = Rule.new(name, op, args) @rules[name] = rule end def find(name) @rules[name] end def self.resolve(obj) case obj when Operator return obj when Symbol return RuleReference.new(obj.to_s) when String return LiteralString.new(obj) when Array ops = [] obj.each do |x| case x when Sequence ops.concat x.ops when Operator ops << x else ops << resolve(x) end end return Sequence.new(*ops) when Range return CharRange.new(obj.begin.to_s, obj.end.to_s) when Regexp return LiteralRegexp.new(obj) else raise "Unknown obj type - #{obj.inspect}" end end # Use these to access the rules unambigiously def [](rule) ref(rule.to_s) end def []=(name, rule) set(name, rule) end def method_missing(meth, *args) meth_s = meth.to_s if meth_s[-1,1] == "=" rule = args.first set(meth_s[0..-2], rule) return rule elsif !args.empty? super end # Hm, I guess this is fine. It might end up confusing people though. return ref(meth.to_s) end def lit(obj, &b) op = Grammar.resolve(obj) op.set_action(b) if b op end def dot(&b) op = Dot.new op.set_action(b) if b op end def str(str, &b) op = LiteralString.new str op.set_action(b) if b op end def reg(reg, opts=nil, &b) op = LiteralRegexp.new reg, opts op.set_action(b) if b op end def range(start, fin, &b) op = CharRange.new(start, fin) op.set_action(b) if b op end def any(*nodes, &b) nodes.map! { |x| Grammar.resolve(x) } op = Choice.new(*nodes) op.set_action(b) if b op end def multiple(node, min, max, &b) op = Multiple.new Grammar.resolve(node), min, max op.set_action(b) if b op end def maybe(node, &b) multiple Grammar.resolve(node), 0, 1, &b end def many(node, &b) multiple Grammar.resolve(node), 1, nil, &b end def kleene(node, &b) multiple Grammar.resolve(node), 0, nil, &b end def seq(*nodes, &b) ops = [] nodes.each do |x| case x when Sequence ops.concat x.ops when Operator ops << x else ops << Grammar.resolve(x) end end op = Sequence.new(*ops) op.set_action(b) if b op end def andp(node) AndPredicate.new Grammar.resolve(node) end def notp(node) NotPredicate.new Grammar.resolve(node) end def ref(name, other_grammar=nil, args=nil) RuleReference.new name.to_s, other_grammar, args end def invoke(name, args=nil) InvokeRule.new name.to_s, args end # Invoke a rule defined on a foreign grammar # == Parameters: # gram:: # The name of the grammar that the rule will be reference from # name:: # The name of the rule that will be invoked # args:: # Any arguements that should be passed to the rule # == Returns: # A new ForeignInvokeRule def foreign_invoke(gram, name, args=nil) ForeignInvokeRule.new gram, name.to_s, args end def t(op, name=nil) Tag.new Grammar.resolve(op), name end def action(action) Action.new action end def collect(op) Collect.new Grammar.resolve(op) end def bounds(op) Bounds.new Grammar.resolve(op) end end end kpeg-1.0.0/lib/kpeg/compiled_parser.rb0000644000175000017500000001657012321266363016306 0ustar domidomirequire 'kpeg/position' module KPeg class CompiledParser # Must be outside the STANDALONE block because a standalone # parser always injects it's own version of this method. def setup_foreign_grammar end # Leave these markers in! They allow us to generate standalone # code automatically! # INITIALIZE START # This is distinct from setup_parser so that a standalone parser # can redefine #initialize and still have access to the proper # parser setup code. def initialize(str, debug=false) setup_parser(str, debug) end # INITIALIZE END # STANDALONE START # Prepares for parsing +str+. If you define a custom initialize you must # call this method before #parse def setup_parser(str, debug=false) set_string str, 0 @memoizations = Hash.new { |h,k| h[k] = {} } @result = nil @failed_rule = nil @failing_rule_offset = -1 setup_foreign_grammar end attr_reader :string attr_reader :failing_rule_offset attr_accessor :result, :pos include Position def get_text(start) @string[start..@pos-1] end # Sets the string and current parsing position for the parser. def set_string string, pos @string = string @string_size = string ? string.size : 0 @pos = pos end def show_pos width = 10 if @pos < width "#{@pos} (\"#{@string[0,@pos]}\" @ \"#{@string[@pos,width]}\")" else "#{@pos} (\"... #{@string[@pos - width, width]}\" @ \"#{@string[@pos,width]}\")" end end def failure_info l = current_line @failing_rule_offset c = current_column @failing_rule_offset if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "line #{l}, column #{c}: failed rule '#{info.name}' = '#{info.rendered}'" else "line #{l}, column #{c}: failed rule '#{@failed_rule}'" end end def failure_caret l = current_line @failing_rule_offset c = current_column @failing_rule_offset line = lines[l-1] "#{line}\n#{' ' * (c - 1)}^" end def failure_character l = current_line @failing_rule_offset c = current_column @failing_rule_offset lines[l-1][c-1, 1] end def failure_oneline l = current_line @failing_rule_offset c = current_column @failing_rule_offset char = lines[l-1][c-1, 1] if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "@#{l}:#{c} failed rule '#{info.name}', got '#{char}'" else "@#{l}:#{c} failed rule '#{@failed_rule}', got '#{char}'" end end class ParseError < RuntimeError end def raise_error raise ParseError, failure_oneline end def show_error(io=STDOUT) error_pos = @failing_rule_offset line_no = current_line(error_pos) col_no = current_column(error_pos) io.puts "On line #{line_no}, column #{col_no}:" if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] io.puts "Failed to match '#{info.rendered}' (rule '#{info.name}')" else io.puts "Failed to match rule '#{@failed_rule}'" end io.puts "Got: #{string[error_pos,1].inspect}" line = lines[line_no-1] io.puts "=> #{line}" io.print(" " * (col_no + 3)) io.puts "^" end def set_failed_rule(name) if @pos > @failing_rule_offset @failed_rule = name @failing_rule_offset = @pos end end attr_reader :failed_rule def match_string(str) len = str.size if @string[pos,len] == str @pos += len return str end return nil end def scan(reg) if m = reg.match(@string[@pos..-1]) width = m.end(0) @pos += width return true end return nil end if "".respond_to? :ord def get_byte if @pos >= @string_size return nil end s = @string[@pos].ord @pos += 1 s end else def get_byte if @pos >= @string_size return nil end s = @string[@pos] @pos += 1 s end end def parse(rule=nil) # We invoke the rules indirectly via apply # instead of by just calling them as methods because # if the rules use left recursion, apply needs to # manage that. if !rule apply(:_root) else method = rule.gsub("-","_hyphen_") apply :"_#{method}" end end class MemoEntry def initialize(ans, pos) @ans = ans @pos = pos @result = nil @set = false @left_rec = false end attr_reader :ans, :pos, :result, :set attr_accessor :left_rec def move!(ans, pos, result) @ans = ans @pos = pos @result = result @set = true @left_rec = false end end def external_invoke(other, rule, *args) old_pos = @pos old_string = @string set_string other.string, other.pos begin if val = __send__(rule, *args) other.pos = @pos other.result = @result else other.set_failed_rule "#{self.class}##{rule}" end val ensure set_string old_string, old_pos end end def apply_with_args(rule, *args) memo_key = [rule, args] if m = @memoizations[memo_key][@pos] @pos = m.pos if !m.set m.left_rec = true return nil end @result = m.result return m.ans else m = MemoEntry.new(nil, @pos) @memoizations[memo_key][@pos] = m start_pos = @pos ans = __send__ rule, *args lr = m.left_rec m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr return grow_lr(rule, args, start_pos, m) else return ans end return ans end end def apply(rule) if m = @memoizations[rule][@pos] @pos = m.pos if !m.set m.left_rec = true return nil end @result = m.result return m.ans else m = MemoEntry.new(nil, @pos) @memoizations[rule][@pos] = m start_pos = @pos ans = __send__ rule lr = m.left_rec m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr return grow_lr(rule, nil, start_pos, m) else return ans end return ans end end def grow_lr(rule, args, start_pos, m) while true @pos = start_pos @result = m.result if args ans = __send__ rule, *args else ans = __send__ rule end return nil unless ans break if @pos <= m.pos m.move! ans, @pos, @result end @result = m.result @pos = m.pos return m.ans end class RuleInfo def initialize(name, rendered) @name = name @rendered = rendered end attr_reader :name, :rendered end def self.rule_info(name, rendered) RuleInfo.new(name, rendered) end # STANDALONE END end end kpeg-1.0.0/lib/kpeg/format_parser.kpeg0000644000175000017500000001217612321266363016323 0ustar domidomi%% name = KPeg::FormatParser %% custom_initialize = true %% pre-class { require 'kpeg/grammar' } %% { ## # Creates a new kpeg format parser for +str+. def initialize(str, debug=false) setup_parser(str, debug) @g = KPeg::Grammar.new end ## # The parsed grammar attr_reader :g alias_method :grammar, :g } eol = "\n" eof_comment = "#" (!eof .)* comment = "#" (!eol .)* eol space = " " | "\t" | eol - = (space | comment)* kleene = "*" # Allow - by itself, but not at the beginning var = < "-" | /[a-z][\w-]*/i > { text } method = < /[a-z_]\w*/i > { text } dbl_escapes = "n" { "\n" } | "s" { " " } | "r" { "\r" } | "t" { "\t" } | "v" { "\v" } | "f" { "\f" } | "b" { "\b" } | "a" { "\a" } | "e" { "\e" } | "\\" { "\\" } | "\"" { "\"" } | num_escapes | < . > { text } num_escapes = < /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") } | "x" < /[a-f\d]{2}/i > { [text.to_i(16)].pack("U") } # TODO use /\h{2}/ after 1.8 support is dropped dbl_seq = < /[^\\"]+/ > { text } dbl_not_quote = ("\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) } dbl_string = "\"" dbl_not_quote:s "\"" { @g.str(s.join) } sgl_escape_quote = "\\'" { "'" } sgl_seq = < /[^']/ > { text } sgl_not_quote = (sgl_escape_quote | sgl_seq)*:segs { Array(segs) } sgl_string = "'" sgl_not_quote:s "'" { @g.str(s.join) } string = dbl_string | sgl_string not_slash = < ("\\/" | /[^\/]/)+ > { text } regexp_opts = < [a-z]* > { text } regexp = "/" not_slash:body "/" regexp_opts:opts { @g.reg body, opts } char = < /[a-z\d]/i > { text } char_range = "[" char:l "-" char:r "]" { @g.range(l,r) } range_num = < /[1-9]\d*/ > { text } range_elem = < range_num|kleene > { text } mult_range = "[" - range_elem:l - "," - range_elem:r - "]" { [l == "*" ? nil : l.to_i, r == "*" ? nil : r.to_i] } | "[" - range_num:e - "]" { [e.to_i, e.to_i] } curly_block = curly curly = "{" < (/[^{}"']+/ | string | curly)* > "}" { @g.action(text) } nested_paren = "(" (/[^()"']+/ | string | nested_paren)* ")" value = value:v ":" var:n { @g.t(v,n) } | value:v "?" { @g.maybe(v) } | value:v "+" { @g.many(v) } | value:v "*" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | "&" value:v { @g.andp(v) } | "!" value:v { @g.notp(v) } | "(" - expression:o - ")" { o } | "@<" - expression:o - ">" { @g.bounds(o) } | "<" - expression:o - ">" { @g.collect(o) } | curly_block | "~" method:m < nested_paren? > { @g.action("#{m}#{text}") } | "." { @g.dot } | "@" var:name < nested_paren? > !(- "=") { @g.invoke(name, text.empty? ? nil : text) } | "^" var:name < nested_paren? > { @g.foreign_invoke("parent", name, text) } | "%" var:gram "." var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- "=") { @g.ref(name, nil, text.empty? ? nil : text) } | char_range | regexp | string spaces = (space | comment)+ values = values:s spaces value:v { @g.seq(s, v) } | value:l spaces value:r { @g.seq(l, r) } | value choose_cont = - "|" - values:v { v } expression = values:v choose_cont+:alts { @g.any(v, *alts) } | values args = args:a "," - var:n - { a + [n] } | - var:n - { [n] } statement = - var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[:\w]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - curly:act { @g.add_directive name, act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) } statements = statement (- statements)? eof = !. root = statements - eof_comment? eof # These are a seperate set of rules used to parse an ast declaration ast_constant = < /[A-Z]\w*/ > { text } ast_word = < /[a-z_]\w*/i > { text } ast_sp = (" " | "\t")* ast_words = ast_words:r ast_sp "," ast_sp ast_word:w { r + [w] } | ast_word:w { [w] } ast_root = ast_constant:c "(" ast_words:w ")" { [c, w] } | ast_constant:c "()"? { [c, []] } kpeg-1.0.0/lib/kpeg/position.rb0000644000175000017500000000106312321266363014771 0ustar domidomimodule KPeg module Position # STANDALONE START def current_column(target=pos) if c = string.rindex("\n", target-1) return target - c - 1 end target + 1 end def current_line(target=pos) cur_offset = 0 cur_line = 0 string.each_line do |line| cur_line += 1 cur_offset += line.size return cur_line if cur_offset >= target end -1 end def lines lines = [] string.each_line { |l| lines << l } lines end # STANDALONE END end end kpeg-1.0.0/lib/kpeg/parser.rb0000644000175000017500000000717112321266363014427 0ustar domidomirequire 'kpeg/position' module KPeg class Parser < StringScanner def initialize(str, grammar, log=false) super str @grammar = grammar # A 2 level hash. @memoizations = Hash.new { |h,k| h[k] = {} } @failing_offset = nil @failing_op = nil @log = log end attr_reader :grammar, :memoizations, :failing_offset attr_accessor :failing_op include Position def switch_grammar(gram) begin old = @grammar @grammar = gram yield ensure @grammar = old end end def fail(op) @failing_offset = pos @failing_op = op return nil end def expected_string case @failing_op when Choice return Range.new(@failing_op.start, @failing_op.fin) when Dot return nil else @failing_op.string end end class LeftRecursive def initialize(detected=false) @detected = detected end attr_accessor :detected end class MemoEntry def initialize(ans, pos) @ans = ans @pos = pos @uses = 1 end attr_reader :ans, :pos, :uses def inc! @uses += 1 end def move!(ans, pos) @ans = ans @pos = pos end end # Call a rule without memoization def invoke(rule) rule.op.match(self) end def apply(rule) ans = nil if m = @memoizations[rule][pos] m.inc! self.pos = m.pos if m.ans.kind_of? LeftRecursive m.ans.detected = true if @log puts "LR #{rule.name} @ #{self.inspect}" end return nil end ans = m.ans else lr = LeftRecursive.new(false) m = MemoEntry.new(lr, pos) @memoizations[rule][pos] = m start_pos = pos if @log puts "START #{rule.name} @ #{self.inspect}" end ans = rule.op.match(self) m.move! ans, pos # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr.detected ans = grow_lr(rule, start_pos, m) end end if @log if ans puts " OK #{rule.name} @ #{self.inspect}" else puts " FAIL #{rule.name} @ #{self.inspect}" end end return ans end def grow_lr(rule, start_pos, m) while true self.pos = start_pos ans = rule.op.match(self) return nil unless ans break if pos <= m.pos m.move! ans, pos end self.pos = m.pos return m.ans end def failed? !!@failing_op end def parse(name=nil) if name rule = @grammar.find(name) unless rule raise "Unknown rule - #{name}" end else rule = @grammar.root end match = apply rule if pos == string.size @failing_op = nil end return match end def expectation error_pos = @failing_offset line_no = current_line(error_pos) col_no = current_column(error_pos) expected = expected_string() prefix = nil case expected when String prefix = expected.inspect when Range prefix = "to be between #{expected.begin} and #{expected.end}" when Array prefix = "to be one of #{expected.inspect}" when nil prefix = "anything (no more input)" else prefix = "unknown" end return "Expected #{prefix} at line #{line_no}, column #{col_no} (offset #{error_pos})" end end end kpeg-1.0.0/lib/kpeg/format_parser.rb0000644000175000017500000021361712321266363016003 0ustar domidomirequire 'kpeg/grammar' class KPeg::FormatParser # :stopdoc: # Prepares for parsing +str+. If you define a custom initialize you must # call this method before #parse def setup_parser(str, debug=false) set_string str, 0 @memoizations = Hash.new { |h,k| h[k] = {} } @result = nil @failed_rule = nil @failing_rule_offset = -1 setup_foreign_grammar end attr_reader :string attr_reader :failing_rule_offset attr_accessor :result, :pos def current_column(target=pos) if c = string.rindex("\n", target-1) return target - c - 1 end target + 1 end def current_line(target=pos) cur_offset = 0 cur_line = 0 string.each_line do |line| cur_line += 1 cur_offset += line.size return cur_line if cur_offset >= target end -1 end def lines lines = [] string.each_line { |l| lines << l } lines end def get_text(start) @string[start..@pos-1] end # Sets the string and current parsing position for the parser. def set_string string, pos @string = string @string_size = string ? string.size : 0 @pos = pos end def show_pos width = 10 if @pos < width "#{@pos} (\"#{@string[0,@pos]}\" @ \"#{@string[@pos,width]}\")" else "#{@pos} (\"... #{@string[@pos - width, width]}\" @ \"#{@string[@pos,width]}\")" end end def failure_info l = current_line @failing_rule_offset c = current_column @failing_rule_offset if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "line #{l}, column #{c}: failed rule '#{info.name}' = '#{info.rendered}'" else "line #{l}, column #{c}: failed rule '#{@failed_rule}'" end end def failure_caret l = current_line @failing_rule_offset c = current_column @failing_rule_offset line = lines[l-1] "#{line}\n#{' ' * (c - 1)}^" end def failure_character l = current_line @failing_rule_offset c = current_column @failing_rule_offset lines[l-1][c-1, 1] end def failure_oneline l = current_line @failing_rule_offset c = current_column @failing_rule_offset char = lines[l-1][c-1, 1] if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "@#{l}:#{c} failed rule '#{info.name}', got '#{char}'" else "@#{l}:#{c} failed rule '#{@failed_rule}', got '#{char}'" end end class ParseError < RuntimeError end def raise_error raise ParseError, failure_oneline end def show_error(io=STDOUT) error_pos = @failing_rule_offset line_no = current_line(error_pos) col_no = current_column(error_pos) io.puts "On line #{line_no}, column #{col_no}:" if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] io.puts "Failed to match '#{info.rendered}' (rule '#{info.name}')" else io.puts "Failed to match rule '#{@failed_rule}'" end io.puts "Got: #{string[error_pos,1].inspect}" line = lines[line_no-1] io.puts "=> #{line}" io.print(" " * (col_no + 3)) io.puts "^" end def set_failed_rule(name) if @pos > @failing_rule_offset @failed_rule = name @failing_rule_offset = @pos end end attr_reader :failed_rule def match_string(str) len = str.size if @string[pos,len] == str @pos += len return str end return nil end def scan(reg) if m = reg.match(@string[@pos..-1]) width = m.end(0) @pos += width return true end return nil end if "".respond_to? :ord def get_byte if @pos >= @string_size return nil end s = @string[@pos].ord @pos += 1 s end else def get_byte if @pos >= @string_size return nil end s = @string[@pos] @pos += 1 s end end def parse(rule=nil) # We invoke the rules indirectly via apply # instead of by just calling them as methods because # if the rules use left recursion, apply needs to # manage that. if !rule apply(:_root) else method = rule.gsub("-","_hyphen_") apply :"_#{method}" end end class MemoEntry def initialize(ans, pos) @ans = ans @pos = pos @result = nil @set = false @left_rec = false end attr_reader :ans, :pos, :result, :set attr_accessor :left_rec def move!(ans, pos, result) @ans = ans @pos = pos @result = result @set = true @left_rec = false end end def external_invoke(other, rule, *args) old_pos = @pos old_string = @string set_string other.string, other.pos begin if val = __send__(rule, *args) other.pos = @pos other.result = @result else other.set_failed_rule "#{self.class}##{rule}" end val ensure set_string old_string, old_pos end end def apply_with_args(rule, *args) memo_key = [rule, args] if m = @memoizations[memo_key][@pos] @pos = m.pos if !m.set m.left_rec = true return nil end @result = m.result return m.ans else m = MemoEntry.new(nil, @pos) @memoizations[memo_key][@pos] = m start_pos = @pos ans = __send__ rule, *args lr = m.left_rec m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr return grow_lr(rule, args, start_pos, m) else return ans end return ans end end def apply(rule) if m = @memoizations[rule][@pos] @pos = m.pos if !m.set m.left_rec = true return nil end @result = m.result return m.ans else m = MemoEntry.new(nil, @pos) @memoizations[rule][@pos] = m start_pos = @pos ans = __send__ rule lr = m.left_rec m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr return grow_lr(rule, nil, start_pos, m) else return ans end return ans end end def grow_lr(rule, args, start_pos, m) while true @pos = start_pos @result = m.result if args ans = __send__ rule, *args else ans = __send__ rule end return nil unless ans break if @pos <= m.pos m.move! ans, @pos, @result end @result = m.result @pos = m.pos return m.ans end class RuleInfo def initialize(name, rendered) @name = name @rendered = rendered end attr_reader :name, :rendered end def self.rule_info(name, rendered) RuleInfo.new(name, rendered) end # :startdoc: ## # Creates a new kpeg format parser for +str+. def initialize(str, debug=false) setup_parser(str, debug) @g = KPeg::Grammar.new end ## # The parsed grammar attr_reader :g alias_method :grammar, :g # :stopdoc: def setup_foreign_grammar; end # eol = "\n" def _eol _tmp = match_string("\n") set_failed_rule :_eol unless _tmp return _tmp end # eof_comment = "#" (!eof .)* def _eof_comment _save = self.pos while true # sequence _tmp = match_string("#") unless _tmp self.pos = _save break end while true _save2 = self.pos while true # sequence _save3 = self.pos _tmp = apply(:_eof) _tmp = _tmp ? nil : true self.pos = _save3 unless _tmp self.pos = _save2 break end _tmp = get_byte unless _tmp self.pos = _save2 end break end # end sequence break unless _tmp end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_eof_comment unless _tmp return _tmp end # comment = "#" (!eol .)* eol def _comment _save = self.pos while true # sequence _tmp = match_string("#") unless _tmp self.pos = _save break end while true _save2 = self.pos while true # sequence _save3 = self.pos _tmp = apply(:_eol) _tmp = _tmp ? nil : true self.pos = _save3 unless _tmp self.pos = _save2 break end _tmp = get_byte unless _tmp self.pos = _save2 end break end # end sequence break unless _tmp end _tmp = true unless _tmp self.pos = _save break end _tmp = apply(:_eol) unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_comment unless _tmp return _tmp end # space = (" " | "\t" | eol) def _space _save = self.pos while true # choice _tmp = match_string(" ") break if _tmp self.pos = _save _tmp = match_string("\t") break if _tmp self.pos = _save _tmp = apply(:_eol) break if _tmp self.pos = _save break end # end choice set_failed_rule :_space unless _tmp return _tmp end # - = (space | comment)* def __hyphen_ while true _save1 = self.pos while true # choice _tmp = apply(:_space) break if _tmp self.pos = _save1 _tmp = apply(:_comment) break if _tmp self.pos = _save1 break end # end choice break unless _tmp end _tmp = true set_failed_rule :__hyphen_ unless _tmp return _tmp end # kleene = "*" def _kleene _tmp = match_string("*") set_failed_rule :_kleene unless _tmp return _tmp end # var = < ("-" | /[a-z][\w-]*/i) > { text } def _var _save = self.pos while true # sequence _text_start = self.pos _save1 = self.pos while true # choice _tmp = match_string("-") break if _tmp self.pos = _save1 _tmp = scan(/\A(?i-mx:[a-z][\w-]*)/) break if _tmp self.pos = _save1 break end # end choice if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_var unless _tmp return _tmp end # method = < /[a-z_]\w*/i > { text } def _method _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?i-mx:[a-z_]\w*)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_method unless _tmp return _tmp end # dbl_escapes = ("n" { "\n" } | "s" { " " } | "r" { "\r" } | "t" { "\t" } | "v" { "\v" } | "f" { "\f" } | "b" { "\b" } | "a" { "\a" } | "e" { "\e" } | "\\" { "\\" } | "\"" { "\"" } | num_escapes | < . > { text }) def _dbl_escapes _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = match_string("n") unless _tmp self.pos = _save1 break end @result = begin; "\n" ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = match_string("s") unless _tmp self.pos = _save2 break end @result = begin; " " ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save _save3 = self.pos while true # sequence _tmp = match_string("r") unless _tmp self.pos = _save3 break end @result = begin; "\r" ; end _tmp = true unless _tmp self.pos = _save3 end break end # end sequence break if _tmp self.pos = _save _save4 = self.pos while true # sequence _tmp = match_string("t") unless _tmp self.pos = _save4 break end @result = begin; "\t" ; end _tmp = true unless _tmp self.pos = _save4 end break end # end sequence break if _tmp self.pos = _save _save5 = self.pos while true # sequence _tmp = match_string("v") unless _tmp self.pos = _save5 break end @result = begin; "\v" ; end _tmp = true unless _tmp self.pos = _save5 end break end # end sequence break if _tmp self.pos = _save _save6 = self.pos while true # sequence _tmp = match_string("f") unless _tmp self.pos = _save6 break end @result = begin; "\f" ; end _tmp = true unless _tmp self.pos = _save6 end break end # end sequence break if _tmp self.pos = _save _save7 = self.pos while true # sequence _tmp = match_string("b") unless _tmp self.pos = _save7 break end @result = begin; "\b" ; end _tmp = true unless _tmp self.pos = _save7 end break end # end sequence break if _tmp self.pos = _save _save8 = self.pos while true # sequence _tmp = match_string("a") unless _tmp self.pos = _save8 break end @result = begin; "\a" ; end _tmp = true unless _tmp self.pos = _save8 end break end # end sequence break if _tmp self.pos = _save _save9 = self.pos while true # sequence _tmp = match_string("e") unless _tmp self.pos = _save9 break end @result = begin; "\e" ; end _tmp = true unless _tmp self.pos = _save9 end break end # end sequence break if _tmp self.pos = _save _save10 = self.pos while true # sequence _tmp = match_string("\\") unless _tmp self.pos = _save10 break end @result = begin; "\\" ; end _tmp = true unless _tmp self.pos = _save10 end break end # end sequence break if _tmp self.pos = _save _save11 = self.pos while true # sequence _tmp = match_string("\"") unless _tmp self.pos = _save11 break end @result = begin; "\"" ; end _tmp = true unless _tmp self.pos = _save11 end break end # end sequence break if _tmp self.pos = _save _tmp = apply(:_num_escapes) break if _tmp self.pos = _save _save12 = self.pos while true # sequence _text_start = self.pos _tmp = get_byte if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save12 break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save12 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_dbl_escapes unless _tmp return _tmp end # num_escapes = (< /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") } | "x" < /[a-f\d]{2}/i > { [text.to_i(16)].pack("U") }) def _num_escapes _save = self.pos while true # choice _save1 = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?-mix:[0-7]{1,3})/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save1 break end @result = begin; [text.to_i(8)].pack("U") ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = match_string("x") unless _tmp self.pos = _save2 break end _text_start = self.pos _tmp = scan(/\A(?i-mx:[a-f\d]{2})/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save2 break end @result = begin; [text.to_i(16)].pack("U") ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_num_escapes unless _tmp return _tmp end # dbl_seq = < /[^\\"]+/ > { text } def _dbl_seq _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?-mix:[^\\"]+)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_dbl_seq unless _tmp return _tmp end # dbl_not_quote = ("\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) } def _dbl_not_quote _save = self.pos while true # sequence _ary = [] while true _save2 = self.pos while true # choice _save3 = self.pos while true # sequence _tmp = match_string("\\") unless _tmp self.pos = _save3 break end _tmp = apply(:_dbl_escapes) s = @result unless _tmp self.pos = _save3 end break end # end sequence break if _tmp self.pos = _save2 _tmp = apply(:_dbl_seq) s = @result break if _tmp self.pos = _save2 break end # end choice _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary ary = @result unless _tmp self.pos = _save break end @result = begin; Array(ary) ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_dbl_not_quote unless _tmp return _tmp end # dbl_string = "\"" dbl_not_quote:s "\"" { @g.str(s.join) } def _dbl_string _save = self.pos while true # sequence _tmp = match_string("\"") unless _tmp self.pos = _save break end _tmp = apply(:_dbl_not_quote) s = @result unless _tmp self.pos = _save break end _tmp = match_string("\"") unless _tmp self.pos = _save break end @result = begin; @g.str(s.join) ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_dbl_string unless _tmp return _tmp end # sgl_escape_quote = "\\'" { "'" } def _sgl_escape_quote _save = self.pos while true # sequence _tmp = match_string("\\'") unless _tmp self.pos = _save break end @result = begin; "'" ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_sgl_escape_quote unless _tmp return _tmp end # sgl_seq = < /[^']/ > { text } def _sgl_seq _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?-mix:[^'])/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_sgl_seq unless _tmp return _tmp end # sgl_not_quote = (sgl_escape_quote | sgl_seq)*:segs { Array(segs) } def _sgl_not_quote _save = self.pos while true # sequence _ary = [] while true _save2 = self.pos while true # choice _tmp = apply(:_sgl_escape_quote) break if _tmp self.pos = _save2 _tmp = apply(:_sgl_seq) break if _tmp self.pos = _save2 break end # end choice _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary segs = @result unless _tmp self.pos = _save break end @result = begin; Array(segs) ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_sgl_not_quote unless _tmp return _tmp end # sgl_string = "'" sgl_not_quote:s "'" { @g.str(s.join) } def _sgl_string _save = self.pos while true # sequence _tmp = match_string("'") unless _tmp self.pos = _save break end _tmp = apply(:_sgl_not_quote) s = @result unless _tmp self.pos = _save break end _tmp = match_string("'") unless _tmp self.pos = _save break end @result = begin; @g.str(s.join) ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_sgl_string unless _tmp return _tmp end # string = (dbl_string | sgl_string) def _string _save = self.pos while true # choice _tmp = apply(:_dbl_string) break if _tmp self.pos = _save _tmp = apply(:_sgl_string) break if _tmp self.pos = _save break end # end choice set_failed_rule :_string unless _tmp return _tmp end # not_slash = < ("\\/" | /[^\/]/)+ > { text } def _not_slash _save = self.pos while true # sequence _text_start = self.pos _save1 = self.pos _save2 = self.pos while true # choice _tmp = match_string("\\/") break if _tmp self.pos = _save2 _tmp = scan(/\A(?-mix:[^\/])/) break if _tmp self.pos = _save2 break end # end choice if _tmp while true _save3 = self.pos while true # choice _tmp = match_string("\\/") break if _tmp self.pos = _save3 _tmp = scan(/\A(?-mix:[^\/])/) break if _tmp self.pos = _save3 break end # end choice break unless _tmp end _tmp = true else self.pos = _save1 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_not_slash unless _tmp return _tmp end # regexp_opts = < [a-z]* > { text } def _regexp_opts _save = self.pos while true # sequence _text_start = self.pos while true _save2 = self.pos _tmp = get_byte if _tmp unless _tmp >= 97 and _tmp <= 122 self.pos = _save2 _tmp = nil end end break unless _tmp end _tmp = true if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_regexp_opts unless _tmp return _tmp end # regexp = "/" not_slash:body "/" regexp_opts:opts { @g.reg body, opts } def _regexp _save = self.pos while true # sequence _tmp = match_string("/") unless _tmp self.pos = _save break end _tmp = apply(:_not_slash) body = @result unless _tmp self.pos = _save break end _tmp = match_string("/") unless _tmp self.pos = _save break end _tmp = apply(:_regexp_opts) opts = @result unless _tmp self.pos = _save break end @result = begin; @g.reg body, opts ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_regexp unless _tmp return _tmp end # char = < /[a-z\d]/i > { text } def _char _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?i-mx:[a-z\d])/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_char unless _tmp return _tmp end # char_range = "[" char:l "-" char:r "]" { @g.range(l,r) } def _char_range _save = self.pos while true # sequence _tmp = match_string("[") unless _tmp self.pos = _save break end _tmp = apply(:_char) l = @result unless _tmp self.pos = _save break end _tmp = match_string("-") unless _tmp self.pos = _save break end _tmp = apply(:_char) r = @result unless _tmp self.pos = _save break end _tmp = match_string("]") unless _tmp self.pos = _save break end @result = begin; @g.range(l,r) ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_char_range unless _tmp return _tmp end # range_num = < /[1-9]\d*/ > { text } def _range_num _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?-mix:[1-9]\d*)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_range_num unless _tmp return _tmp end # range_elem = < (range_num | kleene) > { text } def _range_elem _save = self.pos while true # sequence _text_start = self.pos _save1 = self.pos while true # choice _tmp = apply(:_range_num) break if _tmp self.pos = _save1 _tmp = apply(:_kleene) break if _tmp self.pos = _save1 break end # end choice if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_range_elem unless _tmp return _tmp end # mult_range = ("[" - range_elem:l - "," - range_elem:r - "]" { [l == "*" ? nil : l.to_i, r == "*" ? nil : r.to_i] } | "[" - range_num:e - "]" { [e.to_i, e.to_i] }) def _mult_range _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = match_string("[") unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = apply(:_range_elem) l = @result unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = match_string(",") unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = apply(:_range_elem) r = @result unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = match_string("]") unless _tmp self.pos = _save1 break end @result = begin; [l == "*" ? nil : l.to_i, r == "*" ? nil : r.to_i] ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = match_string("[") unless _tmp self.pos = _save2 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = apply(:_range_num) e = @result unless _tmp self.pos = _save2 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = match_string("]") unless _tmp self.pos = _save2 break end @result = begin; [e.to_i, e.to_i] ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_mult_range unless _tmp return _tmp end # curly_block = curly def _curly_block _tmp = apply(:_curly) set_failed_rule :_curly_block unless _tmp return _tmp end # curly = "{" < (/[^{}"']+/ | string | curly)* > "}" { @g.action(text) } def _curly _save = self.pos while true # sequence _tmp = match_string("{") unless _tmp self.pos = _save break end _text_start = self.pos while true _save2 = self.pos while true # choice _tmp = scan(/\A(?-mix:[^{}"']+)/) break if _tmp self.pos = _save2 _tmp = apply(:_string) break if _tmp self.pos = _save2 _tmp = apply(:_curly) break if _tmp self.pos = _save2 break end # end choice break unless _tmp end _tmp = true if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end _tmp = match_string("}") unless _tmp self.pos = _save break end @result = begin; @g.action(text) ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_curly unless _tmp return _tmp end # nested_paren = "(" (/[^()"']+/ | string | nested_paren)* ")" def _nested_paren _save = self.pos while true # sequence _tmp = match_string("(") unless _tmp self.pos = _save break end while true _save2 = self.pos while true # choice _tmp = scan(/\A(?-mix:[^()"']+)/) break if _tmp self.pos = _save2 _tmp = apply(:_string) break if _tmp self.pos = _save2 _tmp = apply(:_nested_paren) break if _tmp self.pos = _save2 break end # end choice break unless _tmp end _tmp = true unless _tmp self.pos = _save break end _tmp = match_string(")") unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_nested_paren unless _tmp return _tmp end # value = (value:v ":" var:n { @g.t(v,n) } | value:v "?" { @g.maybe(v) } | value:v "+" { @g.many(v) } | value:v "*" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | "&" value:v { @g.andp(v) } | "!" value:v { @g.notp(v) } | "(" - expression:o - ")" { o } | "@<" - expression:o - ">" { @g.bounds(o) } | "<" - expression:o - ">" { @g.collect(o) } | curly_block | "~" method:m < nested_paren? > { @g.action("#{m}#{text}") } | "." { @g.dot } | "@" var:name < nested_paren? > !(- "=") { @g.invoke(name, text.empty? ? nil : text) } | "^" var:name < nested_paren? > { @g.foreign_invoke("parent", name, text) } | "%" var:gram "." var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- "=") { @g.ref(name, nil, text.empty? ? nil : text) } | char_range | regexp | string) def _value _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:_value) v = @result unless _tmp self.pos = _save1 break end _tmp = match_string(":") unless _tmp self.pos = _save1 break end _tmp = apply(:_var) n = @result unless _tmp self.pos = _save1 break end @result = begin; @g.t(v,n) ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = apply(:_value) v = @result unless _tmp self.pos = _save2 break end _tmp = match_string("?") unless _tmp self.pos = _save2 break end @result = begin; @g.maybe(v) ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save _save3 = self.pos while true # sequence _tmp = apply(:_value) v = @result unless _tmp self.pos = _save3 break end _tmp = match_string("+") unless _tmp self.pos = _save3 break end @result = begin; @g.many(v) ; end _tmp = true unless _tmp self.pos = _save3 end break end # end sequence break if _tmp self.pos = _save _save4 = self.pos while true # sequence _tmp = apply(:_value) v = @result unless _tmp self.pos = _save4 break end _tmp = match_string("*") unless _tmp self.pos = _save4 break end @result = begin; @g.kleene(v) ; end _tmp = true unless _tmp self.pos = _save4 end break end # end sequence break if _tmp self.pos = _save _save5 = self.pos while true # sequence _tmp = apply(:_value) v = @result unless _tmp self.pos = _save5 break end _tmp = apply(:_mult_range) r = @result unless _tmp self.pos = _save5 break end @result = begin; @g.multiple(v, *r) ; end _tmp = true unless _tmp self.pos = _save5 end break end # end sequence break if _tmp self.pos = _save _save6 = self.pos while true # sequence _tmp = match_string("&") unless _tmp self.pos = _save6 break end _tmp = apply(:_value) v = @result unless _tmp self.pos = _save6 break end @result = begin; @g.andp(v) ; end _tmp = true unless _tmp self.pos = _save6 end break end # end sequence break if _tmp self.pos = _save _save7 = self.pos while true # sequence _tmp = match_string("!") unless _tmp self.pos = _save7 break end _tmp = apply(:_value) v = @result unless _tmp self.pos = _save7 break end @result = begin; @g.notp(v) ; end _tmp = true unless _tmp self.pos = _save7 end break end # end sequence break if _tmp self.pos = _save _save8 = self.pos while true # sequence _tmp = match_string("(") unless _tmp self.pos = _save8 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save8 break end _tmp = apply(:_expression) o = @result unless _tmp self.pos = _save8 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save8 break end _tmp = match_string(")") unless _tmp self.pos = _save8 break end @result = begin; o ; end _tmp = true unless _tmp self.pos = _save8 end break end # end sequence break if _tmp self.pos = _save _save9 = self.pos while true # sequence _tmp = match_string("@<") unless _tmp self.pos = _save9 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save9 break end _tmp = apply(:_expression) o = @result unless _tmp self.pos = _save9 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save9 break end _tmp = match_string(">") unless _tmp self.pos = _save9 break end @result = begin; @g.bounds(o) ; end _tmp = true unless _tmp self.pos = _save9 end break end # end sequence break if _tmp self.pos = _save _save10 = self.pos while true # sequence _tmp = match_string("<") unless _tmp self.pos = _save10 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save10 break end _tmp = apply(:_expression) o = @result unless _tmp self.pos = _save10 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save10 break end _tmp = match_string(">") unless _tmp self.pos = _save10 break end @result = begin; @g.collect(o) ; end _tmp = true unless _tmp self.pos = _save10 end break end # end sequence break if _tmp self.pos = _save _tmp = apply(:_curly_block) break if _tmp self.pos = _save _save11 = self.pos while true # sequence _tmp = match_string("~") unless _tmp self.pos = _save11 break end _tmp = apply(:_method) m = @result unless _tmp self.pos = _save11 break end _text_start = self.pos _save12 = self.pos _tmp = apply(:_nested_paren) unless _tmp _tmp = true self.pos = _save12 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save11 break end @result = begin; @g.action("#{m}#{text}") ; end _tmp = true unless _tmp self.pos = _save11 end break end # end sequence break if _tmp self.pos = _save _save13 = self.pos while true # sequence _tmp = match_string(".") unless _tmp self.pos = _save13 break end @result = begin; @g.dot ; end _tmp = true unless _tmp self.pos = _save13 end break end # end sequence break if _tmp self.pos = _save _save14 = self.pos while true # sequence _tmp = match_string("@") unless _tmp self.pos = _save14 break end _tmp = apply(:_var) name = @result unless _tmp self.pos = _save14 break end _text_start = self.pos _save15 = self.pos _tmp = apply(:_nested_paren) unless _tmp _tmp = true self.pos = _save15 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save14 break end _save16 = self.pos _save17 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save17 break end _tmp = match_string("=") unless _tmp self.pos = _save17 end break end # end sequence _tmp = _tmp ? nil : true self.pos = _save16 unless _tmp self.pos = _save14 break end @result = begin; @g.invoke(name, text.empty? ? nil : text) ; end _tmp = true unless _tmp self.pos = _save14 end break end # end sequence break if _tmp self.pos = _save _save18 = self.pos while true # sequence _tmp = match_string("^") unless _tmp self.pos = _save18 break end _tmp = apply(:_var) name = @result unless _tmp self.pos = _save18 break end _text_start = self.pos _save19 = self.pos _tmp = apply(:_nested_paren) unless _tmp _tmp = true self.pos = _save19 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save18 break end @result = begin; @g.foreign_invoke("parent", name, text) ; end _tmp = true unless _tmp self.pos = _save18 end break end # end sequence break if _tmp self.pos = _save _save20 = self.pos while true # sequence _tmp = match_string("%") unless _tmp self.pos = _save20 break end _tmp = apply(:_var) gram = @result unless _tmp self.pos = _save20 break end _tmp = match_string(".") unless _tmp self.pos = _save20 break end _tmp = apply(:_var) name = @result unless _tmp self.pos = _save20 break end _text_start = self.pos _save21 = self.pos _tmp = apply(:_nested_paren) unless _tmp _tmp = true self.pos = _save21 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save20 break end @result = begin; @g.foreign_invoke(gram, name, text) ; end _tmp = true unless _tmp self.pos = _save20 end break end # end sequence break if _tmp self.pos = _save _save22 = self.pos while true # sequence _tmp = apply(:_var) name = @result unless _tmp self.pos = _save22 break end _text_start = self.pos _save23 = self.pos _tmp = apply(:_nested_paren) unless _tmp _tmp = true self.pos = _save23 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save22 break end _save24 = self.pos _save25 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save25 break end _tmp = match_string("=") unless _tmp self.pos = _save25 end break end # end sequence _tmp = _tmp ? nil : true self.pos = _save24 unless _tmp self.pos = _save22 break end @result = begin; @g.ref(name, nil, text.empty? ? nil : text) ; end _tmp = true unless _tmp self.pos = _save22 end break end # end sequence break if _tmp self.pos = _save _tmp = apply(:_char_range) break if _tmp self.pos = _save _tmp = apply(:_regexp) break if _tmp self.pos = _save _tmp = apply(:_string) break if _tmp self.pos = _save break end # end choice set_failed_rule :_value unless _tmp return _tmp end # spaces = (space | comment)+ def _spaces _save = self.pos _save1 = self.pos while true # choice _tmp = apply(:_space) break if _tmp self.pos = _save1 _tmp = apply(:_comment) break if _tmp self.pos = _save1 break end # end choice if _tmp while true _save2 = self.pos while true # choice _tmp = apply(:_space) break if _tmp self.pos = _save2 _tmp = apply(:_comment) break if _tmp self.pos = _save2 break end # end choice break unless _tmp end _tmp = true else self.pos = _save end set_failed_rule :_spaces unless _tmp return _tmp end # values = (values:s spaces value:v { @g.seq(s, v) } | value:l spaces value:r { @g.seq(l, r) } | value) def _values _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:_values) s = @result unless _tmp self.pos = _save1 break end _tmp = apply(:_spaces) unless _tmp self.pos = _save1 break end _tmp = apply(:_value) v = @result unless _tmp self.pos = _save1 break end @result = begin; @g.seq(s, v) ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = apply(:_value) l = @result unless _tmp self.pos = _save2 break end _tmp = apply(:_spaces) unless _tmp self.pos = _save2 break end _tmp = apply(:_value) r = @result unless _tmp self.pos = _save2 break end @result = begin; @g.seq(l, r) ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save _tmp = apply(:_value) break if _tmp self.pos = _save break end # end choice set_failed_rule :_values unless _tmp return _tmp end # choose_cont = - "|" - values:v { v } def _choose_cont _save = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save break end _tmp = match_string("|") unless _tmp self.pos = _save break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save break end _tmp = apply(:_values) v = @result unless _tmp self.pos = _save break end @result = begin; v ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_choose_cont unless _tmp return _tmp end # expression = (values:v choose_cont+:alts { @g.any(v, *alts) } | values) def _expression _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:_values) v = @result unless _tmp self.pos = _save1 break end _save2 = self.pos _ary = [] _tmp = apply(:_choose_cont) if _tmp _ary << @result while true _tmp = apply(:_choose_cont) _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary else self.pos = _save2 end alts = @result unless _tmp self.pos = _save1 break end @result = begin; @g.any(v, *alts) ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _tmp = apply(:_values) break if _tmp self.pos = _save break end # end choice set_failed_rule :_expression unless _tmp return _tmp end # args = (args:a "," - var:n - { a + [n] } | - var:n - { [n] }) def _args _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:_args) a = @result unless _tmp self.pos = _save1 break end _tmp = match_string(",") unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = apply(:_var) n = @result unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end @result = begin; a + [n] ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = apply(:_var) n = @result unless _tmp self.pos = _save2 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end @result = begin; [n] ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_args unless _tmp return _tmp end # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[:\w]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - curly:act { @g.add_directive name, act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) }) def _statement _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = apply(:_var) v = @result unless _tmp self.pos = _save1 break end _tmp = match_string("(") unless _tmp self.pos = _save1 break end _tmp = apply(:_args) a = @result unless _tmp self.pos = _save1 break end _tmp = match_string(")") unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = match_string("=") unless _tmp self.pos = _save1 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save1 break end _tmp = apply(:_expression) o = @result unless _tmp self.pos = _save1 break end @result = begin; @g.set(v, o, a) ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = apply(:_var) v = @result unless _tmp self.pos = _save2 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = match_string("=") unless _tmp self.pos = _save2 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = apply(:_expression) o = @result unless _tmp self.pos = _save2 break end @result = begin; @g.set(v, o) ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save _save3 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save3 break end _tmp = match_string("%") unless _tmp self.pos = _save3 break end _tmp = apply(:_var) name = @result unless _tmp self.pos = _save3 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save3 break end _tmp = match_string("=") unless _tmp self.pos = _save3 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save3 break end _text_start = self.pos _tmp = scan(/\A(?-mix:[:\w]+)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save3 break end @result = begin; @g.add_foreign_grammar(name, text) ; end _tmp = true unless _tmp self.pos = _save3 end break end # end sequence break if _tmp self.pos = _save _save4 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save4 break end _tmp = match_string("%%") unless _tmp self.pos = _save4 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save4 break end _tmp = apply(:_curly) act = @result unless _tmp self.pos = _save4 break end @result = begin; @g.add_setup act ; end _tmp = true unless _tmp self.pos = _save4 end break end # end sequence break if _tmp self.pos = _save _save5 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save5 break end _tmp = match_string("%%") unless _tmp self.pos = _save5 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save5 break end _tmp = apply(:_var) name = @result unless _tmp self.pos = _save5 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save5 break end _tmp = apply(:_curly) act = @result unless _tmp self.pos = _save5 break end @result = begin; @g.add_directive name, act ; end _tmp = true unless _tmp self.pos = _save5 end break end # end sequence break if _tmp self.pos = _save _save6 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save6 break end _tmp = match_string("%%") unless _tmp self.pos = _save6 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save6 break end _tmp = apply(:_var) name = @result unless _tmp self.pos = _save6 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save6 break end _tmp = match_string("=") unless _tmp self.pos = _save6 break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save6 break end _text_start = self.pos _save7 = self.pos _save8 = self.pos while true # sequence _save9 = self.pos _tmp = match_string("\n") _tmp = _tmp ? nil : true self.pos = _save9 unless _tmp self.pos = _save8 break end _tmp = get_byte unless _tmp self.pos = _save8 end break end # end sequence if _tmp while true _save10 = self.pos while true # sequence _save11 = self.pos _tmp = match_string("\n") _tmp = _tmp ? nil : true self.pos = _save11 unless _tmp self.pos = _save10 break end _tmp = get_byte unless _tmp self.pos = _save10 end break end # end sequence break unless _tmp end _tmp = true else self.pos = _save7 end if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save6 break end @result = begin; @g.set_variable(name, text) ; end _tmp = true unless _tmp self.pos = _save6 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_statement unless _tmp return _tmp end # statements = statement (- statements)? def _statements _save = self.pos while true # sequence _tmp = apply(:_statement) unless _tmp self.pos = _save break end _save1 = self.pos _save2 = self.pos while true # sequence _tmp = apply(:__hyphen_) unless _tmp self.pos = _save2 break end _tmp = apply(:_statements) unless _tmp self.pos = _save2 end break end # end sequence unless _tmp _tmp = true self.pos = _save1 end unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_statements unless _tmp return _tmp end # eof = !. def _eof _save = self.pos _tmp = get_byte _tmp = _tmp ? nil : true self.pos = _save set_failed_rule :_eof unless _tmp return _tmp end # root = statements - eof_comment? eof def _root _save = self.pos while true # sequence _tmp = apply(:_statements) unless _tmp self.pos = _save break end _tmp = apply(:__hyphen_) unless _tmp self.pos = _save break end _save1 = self.pos _tmp = apply(:_eof_comment) unless _tmp _tmp = true self.pos = _save1 end unless _tmp self.pos = _save break end _tmp = apply(:_eof) unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end # ast_constant = < /[A-Z]\w*/ > { text } def _ast_constant _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?-mix:[A-Z]\w*)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_ast_constant unless _tmp return _tmp end # ast_word = < /[a-z_]\w*/i > { text } def _ast_word _save = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?i-mx:[a-z_]\w*)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_ast_word unless _tmp return _tmp end # ast_sp = (" " | "\t")* def _ast_sp while true _save1 = self.pos while true # choice _tmp = match_string(" ") break if _tmp self.pos = _save1 _tmp = match_string("\t") break if _tmp self.pos = _save1 break end # end choice break unless _tmp end _tmp = true set_failed_rule :_ast_sp unless _tmp return _tmp end # ast_words = (ast_words:r ast_sp "," ast_sp ast_word:w { r + [w] } | ast_word:w { [w] }) def _ast_words _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:_ast_words) r = @result unless _tmp self.pos = _save1 break end _tmp = apply(:_ast_sp) unless _tmp self.pos = _save1 break end _tmp = match_string(",") unless _tmp self.pos = _save1 break end _tmp = apply(:_ast_sp) unless _tmp self.pos = _save1 break end _tmp = apply(:_ast_word) w = @result unless _tmp self.pos = _save1 break end @result = begin; r + [w] ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = apply(:_ast_word) w = @result unless _tmp self.pos = _save2 break end @result = begin; [w] ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_ast_words unless _tmp return _tmp end # ast_root = (ast_constant:c "(" ast_words:w ")" { [c, w] } | ast_constant:c "()"? { [c, []] }) def _ast_root _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = apply(:_ast_constant) c = @result unless _tmp self.pos = _save1 break end _tmp = match_string("(") unless _tmp self.pos = _save1 break end _tmp = apply(:_ast_words) w = @result unless _tmp self.pos = _save1 break end _tmp = match_string(")") unless _tmp self.pos = _save1 break end @result = begin; [c, w] ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = apply(:_ast_constant) c = @result unless _tmp self.pos = _save2 break end _save3 = self.pos _tmp = match_string("()") unless _tmp _tmp = true self.pos = _save3 end unless _tmp self.pos = _save2 break end @result = begin; [c, []] ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_ast_root unless _tmp return _tmp end Rules = {} Rules[:_eol] = rule_info("eol", "\"\\n\"") Rules[:_eof_comment] = rule_info("eof_comment", "\"\#\" (!eof .)*") Rules[:_comment] = rule_info("comment", "\"\#\" (!eol .)* eol") Rules[:_space] = rule_info("space", "(\" \" | \"\\t\" | eol)") Rules[:__hyphen_] = rule_info("-", "(space | comment)*") Rules[:_kleene] = rule_info("kleene", "\"*\"") Rules[:_var] = rule_info("var", "< (\"-\" | /[a-z][\\w-]*/i) > { text }") Rules[:_method] = rule_info("method", "< /[a-z_]\\w*/i > { text }") Rules[:_dbl_escapes] = rule_info("dbl_escapes", "(\"n\" { \"\\n\" } | \"s\" { \" \" } | \"r\" { \"\\r\" } | \"t\" { \"\\t\" } | \"v\" { \"\\v\" } | \"f\" { \"\\f\" } | \"b\" { \"\\b\" } | \"a\" { \"\\a\" } | \"e\" { \"\\e\" } | \"\\\\\" { \"\\\\\" } | \"\\\"\" { \"\\\"\" } | num_escapes | < . > { text })") Rules[:_num_escapes] = rule_info("num_escapes", "(< /[0-7]{1,3}/ > { [text.to_i(8)].pack(\"U\") } | \"x\" < /[a-f\\d]{2}/i > { [text.to_i(16)].pack(\"U\") })") Rules[:_dbl_seq] = rule_info("dbl_seq", "< /[^\\\\\"]+/ > { text }") Rules[:_dbl_not_quote] = rule_info("dbl_not_quote", "(\"\\\\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) }") Rules[:_dbl_string] = rule_info("dbl_string", "\"\\\"\" dbl_not_quote:s \"\\\"\" { @g.str(s.join) }") Rules[:_sgl_escape_quote] = rule_info("sgl_escape_quote", "\"\\\\'\" { \"'\" }") Rules[:_sgl_seq] = rule_info("sgl_seq", "< /[^']/ > { text }") Rules[:_sgl_not_quote] = rule_info("sgl_not_quote", "(sgl_escape_quote | sgl_seq)*:segs { Array(segs) }") Rules[:_sgl_string] = rule_info("sgl_string", "\"'\" sgl_not_quote:s \"'\" { @g.str(s.join) }") Rules[:_string] = rule_info("string", "(dbl_string | sgl_string)") Rules[:_not_slash] = rule_info("not_slash", "< (\"\\\\/\" | /[^\\/]/)+ > { text }") Rules[:_regexp_opts] = rule_info("regexp_opts", "< [a-z]* > { text }") Rules[:_regexp] = rule_info("regexp", "\"/\" not_slash:body \"/\" regexp_opts:opts { @g.reg body, opts }") Rules[:_char] = rule_info("char", "< /[a-z\\d]/i > { text }") Rules[:_char_range] = rule_info("char_range", "\"[\" char:l \"-\" char:r \"]\" { @g.range(l,r) }") Rules[:_range_num] = rule_info("range_num", "< /[1-9]\\d*/ > { text }") Rules[:_range_elem] = rule_info("range_elem", "< (range_num | kleene) > { text }") Rules[:_mult_range] = rule_info("mult_range", "(\"[\" - range_elem:l - \",\" - range_elem:r - \"]\" { [l == \"*\" ? nil : l.to_i, r == \"*\" ? nil : r.to_i] } | \"[\" - range_num:e - \"]\" { [e.to_i, e.to_i] })") Rules[:_curly_block] = rule_info("curly_block", "curly") Rules[:_curly] = rule_info("curly", "\"{\" < (/[^{}\"']+/ | string | curly)* > \"}\" { @g.action(text) }") Rules[:_nested_paren] = rule_info("nested_paren", "\"(\" (/[^()\"']+/ | string | nested_paren)* \")\"") Rules[:_value] = rule_info("value", "(value:v \":\" var:n { @g.t(v,n) } | value:v \"?\" { @g.maybe(v) } | value:v \"+\" { @g.many(v) } | value:v \"*\" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | \"&\" value:v { @g.andp(v) } | \"!\" value:v { @g.notp(v) } | \"(\" - expression:o - \")\" { o } | \"@<\" - expression:o - \">\" { @g.bounds(o) } | \"<\" - expression:o - \">\" { @g.collect(o) } | curly_block | \"~\" method:m < nested_paren? > { @g.action(\"\#{m}\#{text}\") } | \".\" { @g.dot } | \"@\" var:name < nested_paren? > !(- \"=\") { @g.invoke(name, text.empty? ? nil : text) } | \"^\" var:name < nested_paren? > { @g.foreign_invoke(\"parent\", name, text) } | \"%\" var:gram \".\" var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- \"=\") { @g.ref(name, nil, text.empty? ? nil : text) } | char_range | regexp | string)") Rules[:_spaces] = rule_info("spaces", "(space | comment)+") Rules[:_values] = rule_info("values", "(values:s spaces value:v { @g.seq(s, v) } | value:l spaces value:r { @g.seq(l, r) } | value)") Rules[:_choose_cont] = rule_info("choose_cont", "- \"|\" - values:v { v }") Rules[:_expression] = rule_info("expression", "(values:v choose_cont+:alts { @g.any(v, *alts) } | values)") Rules[:_args] = rule_info("args", "(args:a \",\" - var:n - { a + [n] } | - var:n - { [n] })") Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[:\\w]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - curly:act { @g.add_directive name, act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })") Rules[:_statements] = rule_info("statements", "statement (- statements)?") Rules[:_eof] = rule_info("eof", "!.") Rules[:_root] = rule_info("root", "statements - eof_comment? eof") Rules[:_ast_constant] = rule_info("ast_constant", "< /[A-Z]\\w*/ > { text }") Rules[:_ast_word] = rule_info("ast_word", "< /[a-z_]\\w*/i > { text }") Rules[:_ast_sp] = rule_info("ast_sp", "(\" \" | \"\\t\")*") Rules[:_ast_words] = rule_info("ast_words", "(ast_words:r ast_sp \",\" ast_sp ast_word:w { r + [w] } | ast_word:w { [w] })") Rules[:_ast_root] = rule_info("ast_root", "(ast_constant:c \"(\" ast_words:w \")\" { [c, w] } | ast_constant:c \"()\"? { [c, []] })") # :startdoc: end kpeg-1.0.0/lib/kpeg/string_escape.kpeg0000644000175000017500000000065712321266363016306 0ustar domidomi%% name = KPeg::StringEscape %% { attr_reader :text } segment = < /[\w ]+/ > { text } # Don't use \s because that matchs \n | "\\" { "\\\\" } | "\n" { "\\n" } | "\r" { "\\r" } | "\t" { "\\t" } | "\b" { "\\b" } | "\"" { "\\\"" } | < . > { text } root = segment*:s { @text = s.join } embed_seg = "#" { "\\#" } | segment embed = embed_seg*:s { @text = s.join } kpeg-1.0.0/lib/kpeg/string_escape.rb0000644000175000017500000003215612321266363015762 0ustar domidomiclass KPeg::StringEscape # :stopdoc: # This is distinct from setup_parser so that a standalone parser # can redefine #initialize and still have access to the proper # parser setup code. def initialize(str, debug=false) setup_parser(str, debug) end # Prepares for parsing +str+. If you define a custom initialize you must # call this method before #parse def setup_parser(str, debug=false) set_string str, 0 @memoizations = Hash.new { |h,k| h[k] = {} } @result = nil @failed_rule = nil @failing_rule_offset = -1 setup_foreign_grammar end attr_reader :string attr_reader :failing_rule_offset attr_accessor :result, :pos def current_column(target=pos) if c = string.rindex("\n", target-1) return target - c - 1 end target + 1 end def current_line(target=pos) cur_offset = 0 cur_line = 0 string.each_line do |line| cur_line += 1 cur_offset += line.size return cur_line if cur_offset >= target end -1 end def lines lines = [] string.each_line { |l| lines << l } lines end def get_text(start) @string[start..@pos-1] end # Sets the string and current parsing position for the parser. def set_string string, pos @string = string @string_size = string ? string.size : 0 @pos = pos end def show_pos width = 10 if @pos < width "#{@pos} (\"#{@string[0,@pos]}\" @ \"#{@string[@pos,width]}\")" else "#{@pos} (\"... #{@string[@pos - width, width]}\" @ \"#{@string[@pos,width]}\")" end end def failure_info l = current_line @failing_rule_offset c = current_column @failing_rule_offset if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "line #{l}, column #{c}: failed rule '#{info.name}' = '#{info.rendered}'" else "line #{l}, column #{c}: failed rule '#{@failed_rule}'" end end def failure_caret l = current_line @failing_rule_offset c = current_column @failing_rule_offset line = lines[l-1] "#{line}\n#{' ' * (c - 1)}^" end def failure_character l = current_line @failing_rule_offset c = current_column @failing_rule_offset lines[l-1][c-1, 1] end def failure_oneline l = current_line @failing_rule_offset c = current_column @failing_rule_offset char = lines[l-1][c-1, 1] if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "@#{l}:#{c} failed rule '#{info.name}', got '#{char}'" else "@#{l}:#{c} failed rule '#{@failed_rule}', got '#{char}'" end end class ParseError < RuntimeError end def raise_error raise ParseError, failure_oneline end def show_error(io=STDOUT) error_pos = @failing_rule_offset line_no = current_line(error_pos) col_no = current_column(error_pos) io.puts "On line #{line_no}, column #{col_no}:" if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] io.puts "Failed to match '#{info.rendered}' (rule '#{info.name}')" else io.puts "Failed to match rule '#{@failed_rule}'" end io.puts "Got: #{string[error_pos,1].inspect}" line = lines[line_no-1] io.puts "=> #{line}" io.print(" " * (col_no + 3)) io.puts "^" end def set_failed_rule(name) if @pos > @failing_rule_offset @failed_rule = name @failing_rule_offset = @pos end end attr_reader :failed_rule def match_string(str) len = str.size if @string[pos,len] == str @pos += len return str end return nil end def scan(reg) if m = reg.match(@string[@pos..-1]) width = m.end(0) @pos += width return true end return nil end if "".respond_to? :ord def get_byte if @pos >= @string_size return nil end s = @string[@pos].ord @pos += 1 s end else def get_byte if @pos >= @string_size return nil end s = @string[@pos] @pos += 1 s end end def parse(rule=nil) # We invoke the rules indirectly via apply # instead of by just calling them as methods because # if the rules use left recursion, apply needs to # manage that. if !rule apply(:_root) else method = rule.gsub("-","_hyphen_") apply :"_#{method}" end end class MemoEntry def initialize(ans, pos) @ans = ans @pos = pos @result = nil @set = false @left_rec = false end attr_reader :ans, :pos, :result, :set attr_accessor :left_rec def move!(ans, pos, result) @ans = ans @pos = pos @result = result @set = true @left_rec = false end end def external_invoke(other, rule, *args) old_pos = @pos old_string = @string set_string other.string, other.pos begin if val = __send__(rule, *args) other.pos = @pos other.result = @result else other.set_failed_rule "#{self.class}##{rule}" end val ensure set_string old_string, old_pos end end def apply_with_args(rule, *args) memo_key = [rule, args] if m = @memoizations[memo_key][@pos] @pos = m.pos if !m.set m.left_rec = true return nil end @result = m.result return m.ans else m = MemoEntry.new(nil, @pos) @memoizations[memo_key][@pos] = m start_pos = @pos ans = __send__ rule, *args lr = m.left_rec m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr return grow_lr(rule, args, start_pos, m) else return ans end return ans end end def apply(rule) if m = @memoizations[rule][@pos] @pos = m.pos if !m.set m.left_rec = true return nil end @result = m.result return m.ans else m = MemoEntry.new(nil, @pos) @memoizations[rule][@pos] = m start_pos = @pos ans = __send__ rule lr = m.left_rec m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr return grow_lr(rule, nil, start_pos, m) else return ans end return ans end end def grow_lr(rule, args, start_pos, m) while true @pos = start_pos @result = m.result if args ans = __send__ rule, *args else ans = __send__ rule end return nil unless ans break if @pos <= m.pos m.move! ans, @pos, @result end @result = m.result @pos = m.pos return m.ans end class RuleInfo def initialize(name, rendered) @name = name @rendered = rendered end attr_reader :name, :rendered end def self.rule_info(name, rendered) RuleInfo.new(name, rendered) end # :startdoc: attr_reader :text # :stopdoc: def setup_foreign_grammar; end # segment = (< /[\w ]+/ > { text } | "\\" { "\\\\" } | "\n" { "\\n" } | "\r" { "\\r" } | "\t" { "\\t" } | "\b" { "\\b" } | "\"" { "\\\"" } | < . > { text }) def _segment _save = self.pos while true # choice _save1 = self.pos while true # sequence _text_start = self.pos _tmp = scan(/\A(?-mix:[\w ]+)/) if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save1 break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _save2 = self.pos while true # sequence _tmp = match_string("\\") unless _tmp self.pos = _save2 break end @result = begin; "\\\\" ; end _tmp = true unless _tmp self.pos = _save2 end break end # end sequence break if _tmp self.pos = _save _save3 = self.pos while true # sequence _tmp = match_string("\n") unless _tmp self.pos = _save3 break end @result = begin; "\\n" ; end _tmp = true unless _tmp self.pos = _save3 end break end # end sequence break if _tmp self.pos = _save _save4 = self.pos while true # sequence _tmp = match_string("\r") unless _tmp self.pos = _save4 break end @result = begin; "\\r" ; end _tmp = true unless _tmp self.pos = _save4 end break end # end sequence break if _tmp self.pos = _save _save5 = self.pos while true # sequence _tmp = match_string("\t") unless _tmp self.pos = _save5 break end @result = begin; "\\t" ; end _tmp = true unless _tmp self.pos = _save5 end break end # end sequence break if _tmp self.pos = _save _save6 = self.pos while true # sequence _tmp = match_string("\b") unless _tmp self.pos = _save6 break end @result = begin; "\\b" ; end _tmp = true unless _tmp self.pos = _save6 end break end # end sequence break if _tmp self.pos = _save _save7 = self.pos while true # sequence _tmp = match_string("\"") unless _tmp self.pos = _save7 break end @result = begin; "\\\"" ; end _tmp = true unless _tmp self.pos = _save7 end break end # end sequence break if _tmp self.pos = _save _save8 = self.pos while true # sequence _text_start = self.pos _tmp = get_byte if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save8 break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save8 end break end # end sequence break if _tmp self.pos = _save break end # end choice set_failed_rule :_segment unless _tmp return _tmp end # root = segment*:s { @text = s.join } def _root _save = self.pos while true # sequence _ary = [] while true _tmp = apply(:_segment) _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary s = @result unless _tmp self.pos = _save break end @result = begin; @text = s.join ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end # embed_seg = ("#" { "\\#" } | segment) def _embed_seg _save = self.pos while true # choice _save1 = self.pos while true # sequence _tmp = match_string("#") unless _tmp self.pos = _save1 break end @result = begin; "\\#" ; end _tmp = true unless _tmp self.pos = _save1 end break end # end sequence break if _tmp self.pos = _save _tmp = apply(:_segment) break if _tmp self.pos = _save break end # end choice set_failed_rule :_embed_seg unless _tmp return _tmp end # embed = embed_seg*:s { @text = s.join } def _embed _save = self.pos while true # sequence _ary = [] while true _tmp = apply(:_embed_seg) _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary s = @result unless _tmp self.pos = _save break end @result = begin; @text = s.join ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_embed unless _tmp return _tmp end Rules = {} Rules[:_segment] = rule_info("segment", "(< /[\\w ]+/ > { text } | \"\\\\\" { \"\\\\\\\\\" } | \"\\n\" { \"\\\\n\" } | \"\\r\" { \"\\\\r\" } | \"\\t\" { \"\\\\t\" } | \"\\b\" { \"\\\\b\" } | \"\\\"\" { \"\\\\\\\"\" } | < . > { text })") Rules[:_root] = rule_info("root", "segment*:s { @text = s.join }") Rules[:_embed_seg] = rule_info("embed_seg", "(\"\#\" { \"\\\\\#\" } | segment)") Rules[:_embed] = rule_info("embed", "embed_seg*:s { @text = s.join }") # :startdoc: end kpeg-1.0.0/lib/kpeg/match.rb0000644000175000017500000000257312321266363014230 0ustar domidomimodule KPeg class Match; end class MatchString < Match def initialize(op, string) @op = op @string = string end attr_reader :op, :string def explain(indent="") puts "#{indent}KPeg::Match:#{object_id.to_s(16)}" puts "#{indent} op: #{@op.inspect}" puts "#{indent} string: #{@string.inspect}" end alias_method :total_string, :string def value(obj=nil) return @string unless @op.action if obj obj.instance_exec(@string, &@op.action) else @op.action.call(@string) end end end class MatchComposition < Match def initialize(op, matches) @op = op @matches = matches end attr_reader :op, :matches def explain(indent="") puts "#{indent}KPeg::Match:#{object_id.to_s(16)}" puts "#{indent} op: #{@op.inspect}" puts "#{indent} matches:" @matches.each do |m| m.explain("#{indent} ") end end def total_string @matches.map { |m| m.total_string }.join end def value(obj=nil) values = @matches.map { |m| m.value(obj) } values = @op.prune_values(values) unless @op.action return values.first if values.size == 1 return values end if obj obj.instance_exec(*values, &@op.action) else @op.action.call(*values) end end end end kpeg-1.0.0/lib/kpeg/code_generator.rb0000644000175000017500000003510212321266363016106 0ustar domidomirequire 'kpeg/grammar_renderer' require 'stringio' module KPeg class CodeGenerator def initialize(name, gram, debug=false) @name = name @grammar = gram @debug = debug @saves = 0 @output = nil @standalone = false end attr_accessor :standalone def method_name(name) name = name.gsub("-","_hyphen_") "_#{name}" end def save if @saves == 0 str = "_save" else str = "_save#{@saves}" end @saves += 1 str end def reset_saves @saves = 0 end def output_ast(short, code, description) parser = FormatParser.new description # just skip it if it's bad. return unless parser.parse "ast_root" name, attrs = parser.result code << " class #{name} < Node\n" code << " def initialize(#{attrs.join(', ')})\n" attrs.each do |at| code << " @#{at} = #{at}\n" end code << " end\n" attrs.each do |at| code << " attr_reader :#{at}\n" end code << " end\n" [short, name, attrs] end def handle_ast(code) output_node = false root = @grammar.variables["ast-location"] || "AST" methods = [] vars = @grammar.variables.keys.sort vars.each do |name| val = @grammar.variables[name] if val.index("ast ") == 0 unless output_node code << "\n" code << " module #{root}\n" code << " class Node; end\n" output_node = true end if m = output_ast(name, code, val[4..-1]) methods << m end end end if output_node code << " end\n" code << " module #{root}Construction\n" methods.each do |short, name, attrs| code << " def #{short}(#{attrs.join(', ')})\n" code << " #{root}::#{name}.new(#{attrs.join(', ')})\n" code << " end\n" end code << " end\n" code << " include #{root}Construction\n" end end def indentify(code, indent) "#{" " * indent}#{code}" end # Default indent is 4 spaces (indent=2) def output_op(code, op, indent=2) case op when Dot code << indentify("_tmp = get_byte\n", indent) when LiteralString code << indentify("_tmp = match_string(#{op.string.dump})\n", indent) when LiteralRegexp if op.regexp.respond_to?(:kcode) lang = op.regexp.kcode.to_s[0,1] else # Let default ruby string handling figure it out lang = "" end code << indentify("_tmp = scan(/\\A#{op.regexp}/#{lang})\n", indent) when CharRange ss = save() if op.start.bytesize == 1 and op.fin.bytesize == 1 code << indentify("#{ss} = self.pos\n", indent) code << indentify("_tmp = get_byte\n", indent) code << indentify("if _tmp\n", indent) if op.start.respond_to? :getbyte left = op.start.getbyte 0 right = op.fin.getbyte 0 else left = op.start[0] right = op.fin[0] end code << indentify(" unless _tmp >= #{left} and _tmp <= #{right}\n", indent) code << indentify(" self.pos = #{ss}\n", indent) code << indentify(" _tmp = nil\n", indent) code << indentify(" end\n", indent) code << indentify("end\n", indent) else raise "Unsupported char range - #{op.inspect}" end when Choice ss = save() code << "\n" code << indentify("#{ss} = self.pos\n", indent) code << indentify("while true # choice\n", indent) op.ops.each_with_index do |n,idx| output_op code, n, (indent+1) code << indentify(" break if _tmp\n", indent) code << indentify(" self.pos = #{ss}\n", indent) if idx == op.ops.size - 1 code << indentify(" break\n", indent) end end code << indentify("end # end choice\n\n", indent) when Multiple ss = save() if op.min == 0 and op.max == 1 code << indentify("#{ss} = self.pos\n", indent) output_op code, op.op, indent if op.save_values code << indentify("@result = nil unless _tmp\n", indent) end code << indentify("unless _tmp\n", indent) code << indentify(" _tmp = true\n", indent) code << indentify(" self.pos = #{ss}\n", indent) code << indentify("end\n", indent) elsif op.min == 0 and !op.max if op.save_values code << indentify("_ary = []\n", indent) end code << indentify("while true\n", indent) output_op code, op.op, (indent+1) if op.save_values code << indentify(" _ary << @result if _tmp\n", indent) end code << indentify(" break unless _tmp\n", indent) code << indentify("end\n", indent) code << indentify("_tmp = true\n", indent) if op.save_values code << indentify("@result = _ary\n", indent) end elsif op.min == 1 and !op.max code << indentify("#{ss} = self.pos\n", indent) if op.save_values code << indentify("_ary = []\n", indent) end output_op code, op.op, indent code << indentify("if _tmp\n", indent) if op.save_values code << indentify(" _ary << @result\n", indent) end code << indentify(" while true\n", indent) output_op code, op.op, (indent+2) if op.save_values code << indentify(" _ary << @result if _tmp\n", indent) end code << indentify(" break unless _tmp\n", indent) code << indentify(" end\n", indent) code << indentify(" _tmp = true\n", indent) if op.save_values code << indentify(" @result = _ary\n", indent) end code << indentify("else\n", indent) code << indentify(" self.pos = #{ss}\n", indent) code << indentify("end\n", indent) else code << indentify("#{ss} = self.pos\n", indent) code << indentify("_count = 0\n", indent) code << indentify("while true\n", indent) output_op code, op.op, (indent+1) code << indentify(" if _tmp\n", indent) code << indentify(" _count += 1\n", indent) code << indentify(" break if _count == #{op.max}\n", indent) code << indentify(" else\n", indent) code << indentify(" break\n", indent) code << indentify(" end\n", indent) code << indentify("end\n", indent) code << indentify("if _count >= #{op.min}\n", indent) code << indentify(" _tmp = true\n", indent) code << indentify("else\n", indent) code << indentify(" self.pos = #{ss}\n", indent) code << indentify(" _tmp = nil\n", indent) code << indentify("end\n", indent) end when Sequence ss = save() code << "\n" code << indentify("#{ss} = self.pos\n", indent) code << indentify("while true # sequence\n", indent) op.ops.each_with_index do |n, idx| output_op code, n, (indent+1) if idx == op.ops.size - 1 code << indentify(" unless _tmp\n", indent) code << indentify(" self.pos = #{ss}\n", indent) code << indentify(" end\n", indent) code << indentify(" break\n", indent) else code << indentify(" unless _tmp\n", indent) code << indentify(" self.pos = #{ss}\n", indent) code << indentify(" break\n", indent) code << indentify(" end\n", indent) end end code << indentify("end # end sequence\n\n", indent) when AndPredicate ss = save() code << indentify("#{ss} = self.pos\n", indent) if op.op.kind_of? Action code << indentify("_tmp = begin; #{op.op.action}; end\n", indent) else output_op code, op.op, indent end code << indentify("self.pos = #{ss}\n", indent) when NotPredicate ss = save() code << indentify("#{ss} = self.pos\n", indent) if op.op.kind_of? Action code << indentify("_tmp = begin; #{op.op.action}; end\n", indent) else output_op code, op.op, indent end code << indentify("_tmp = _tmp ? nil : true\n", indent) code << indentify("self.pos = #{ss}\n", indent) when RuleReference if op.arguments code << indentify("_tmp = apply_with_args(:#{method_name op.rule_name}, #{op.arguments[1..-2]})\n", indent) else code << indentify("_tmp = apply(:#{method_name op.rule_name})\n", indent) end when InvokeRule if op.arguments code << indentify("_tmp = #{method_name op.rule_name}#{op.arguments}\n", indent) else code << indentify("_tmp = #{method_name op.rule_name}()\n", indent) end when ForeignInvokeRule if op.arguments code << indentify("_tmp = @_grammar_#{op.grammar_name}.external_invoke(self, :#{method_name op.rule_name}, #{op.arguments[1..-2]})\n", indent) else code << indentify("_tmp = @_grammar_#{op.grammar_name}.external_invoke(self, :#{method_name op.rule_name})\n", indent) end when Tag if op.tag_name and !op.tag_name.empty? output_op code, op.op, indent code << indentify("#{op.tag_name} = @result\n", indent) else output_op code, op.op, indent end when Action code << indentify("@result = begin; ", indent) code << op.action << "; end\n" if @debug code << indentify("puts \" => \" #{op.action.dump} \" => \#{@result.inspect} \\n\"\n", indent) end code << indentify("_tmp = true\n", indent) when Collect code << indentify("_text_start = self.pos\n", indent) output_op code, op.op, indent code << indentify("if _tmp\n", indent) code << indentify(" text = get_text(_text_start)\n", indent) code << indentify("end\n", indent) when Bounds code << indentify("_bounds_start = self.pos\n", indent) output_op code, op.op, indent code << indentify("if _tmp\n", indent) code << indentify(" bounds = [_bounds_start, self.pos]\n", indent) code << indentify("end\n", indent) else raise "Unknown op - #{op.class}" end end def standalone_region(path, marker = "STANDALONE") expanded_path = File.expand_path("../#{path}", __FILE__) cp = File.read(expanded_path) start_marker = "# #{marker} START" end_marker = /^\s*# #{Regexp.escape marker} END/ start = cp.index(start_marker) + start_marker.length + 1 # \n fin = cp.index(end_marker) unless start and fin abort("#{marker} boundaries in #{path} missing " \ "for standalone generation") end cp[start..fin] end def output return @output if @output code = [] output_header(code) output_grammar(code) output_footer(code) @output = code.join end ## # Output of class end and footer def output_footer(code) code << "end\n" if footer = @grammar.directives['footer'] code << footer.action end end ## # Output of grammar and rules def output_grammar(code) code << " # :stopdoc:\n" handle_ast(code) fg = @grammar.foreign_grammars if fg.empty? if @standalone code << " def setup_foreign_grammar; end\n" end else code << " def setup_foreign_grammar\n" @grammar.foreign_grammars.each do |name, gram| code << " @_grammar_#{name} = #{gram}.new(nil)\n" end code << " end\n" end render = GrammarRenderer.new(@grammar) renderings = {} @grammar.rule_order.each do |name| reset_saves rule = @grammar.rules[name] io = StringIO.new render.render_op io, rule.op rend = io.string rend.gsub! "\n", " " renderings[name] = rend code << "\n" code << " # #{name} = #{rend}\n" if rule.arguments code << " def #{method_name name}(#{rule.arguments.join(',')})\n" else code << " def #{method_name name}\n" end if @debug code << " puts \"START #{name} @ \#{show_pos}\\n\"\n" end output_op code, rule.op if @debug code << " if _tmp\n" code << " puts \" OK #{name} @ \#{show_pos}\\n\"\n" code << " else\n" code << " puts \" FAIL #{name} @ \#{show_pos}\\n\"\n" code << " end\n" end code << " set_failed_rule :#{method_name name} unless _tmp\n" code << " return _tmp\n" code << " end\n" end code << "\n Rules = {}\n" @grammar.rule_order.each do |name| rend = GrammarRenderer.escape renderings[name], true code << " Rules[:#{method_name name}] = rule_info(\"#{name}\", \"#{rend}\")\n" end code << " # :startdoc:\n" end ## # Output up to the user-defined setup actions def output_header(code) if header = @grammar.directives['header'] code << header.action.strip code << "\n" end pre_class = @grammar.directives['pre-class'] if @standalone if pre_class code << pre_class.action.strip code << "\n" end code << "class #{@name}\n" cp = standalone_region("compiled_parser.rb") cpi = standalone_region("compiled_parser.rb", "INITIALIZE") pp = standalone_region("position.rb") cp.gsub!(/^\s*include Position/, pp) code << " # :stopdoc:\n" code << cpi << "\n" unless @grammar.variables['custom_initialize'] code << cp << "\n" code << " # :startdoc:\n" else code << "require 'kpeg/compiled_parser'\n\n" if pre_class code << pre_class.action.strip code << "\n" end code << "class #{@name} < KPeg::CompiledParser\n" end @grammar.setup_actions.each do |act| code << "\n#{act.action}\n\n" end end def make(str) m = Module.new m.module_eval output, "(kpeg parser #{@name})" cls = m.const_get(@name) cls.new(str) end def parse(str) make(str).parse end end end kpeg-1.0.0/test/0000755000175000017500000000000012321266363012063 5ustar domidomikpeg-1.0.0/test/test_kpeg_grammar.rb0000644000175000017500000000525712321266363016114 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'kpeg/format_parser' require 'kpeg/code_generator' require 'stringio' class TestKpegGrammar < Minitest::Test LEFT_RECURSION = <<-'STR' name = name:n "[]" { [:array, n] } | < /\w+/ > { [:word, text] } root = name STR def test_left_recursion_invoke_rule_directly parc = KPeg::FormatParser.new(LEFT_RECURSION) assert parc.parse, "Unable to parse" gram = parc.grammar # gr = KPeg::GrammarRenderer.new(gram) # puts # gr.render(STDOUT) cg = KPeg::CodeGenerator.new "TestCalc", gram code = cg.make("blah[]") assert_equal true, code.parse("name") assert_equal [:array, [:word, "blah"]], code.result end def test_left_recursion_invoke_rule_via_another parc = KPeg::FormatParser.new(LEFT_RECURSION) assert parc.parse, "Unable to parse" gram = parc.grammar # gr = KPeg::GrammarRenderer.new(gram) # puts # gr.render(STDOUT) cg = KPeg::CodeGenerator.new "TestCalc", gram code = cg.make("blah[]") assert_equal true, code.parse("root") assert_equal [:array, [:word, "blah"]], code.result end def test_gen_calc parc = KPeg::FormatParser.new(<<-'STR') Stmt = - Expr:e EOL { @answers << e } | ( !EOL . )* EOL { puts "error" } Expr = ID:i ASSIGN Sum:s { @vars[i] = s } | Sum:s { s } Sum = Product:l ( PLUS Product:r { l += r } | MINUS Product:r { l -= r } )* { l } Product = Value:l ( TIMES Value:r { l *= r } | DIVIDE Value:r { l /= r } )* { l } Value = NUMBER:i { i } | ID:i !ASSIGN { @vars[i] } | OPEN Expr:i CLOSE { i } NUMBER = < [0-9]+ > - { text.to_i } ID = < [a-z] > - { text } ASSIGN = '=' - PLUS = '+' - MINUS = '-' - TIMES = '*' - DIVIDE = '/' - OPEN = '(' - CLOSE = ')' - - = (' ' | '\t')* EOL = ('\n' | '\r\n' | '\r' | ';') - root = Stmt+ STR assert parc.parse, "Unable to parse" gram = parc.grammar # gr = KPeg::GrammarRenderer.new(gram) # puts # gr.render(STDOUT) cg = KPeg::CodeGenerator.new "TestCalc", gram code = cg.make("i = 3+4; j = i*8; i + j * 2;") code.instance_variable_set(:@vars, {}) code.instance_variable_set(:@answers, []) assert_equal true, code.parse assert_equal [7,56,119], code.instance_variable_get(:@answers) end end kpeg-1.0.0/test/inputs/0000755000175000017500000000000012321266363013405 5ustar domidomikpeg-1.0.0/test/inputs/comments.kpeg0000644000175000017500000000007112321266363016100 0ustar domidomi%% name = Name word = /\w+/ root = word # kjkjk # asldkjfkpeg-1.0.0/test/test_kpeg_format_parser_round_trip.rb0000644000175000017500000000551512321266363021574 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'kpeg/format_parser' require 'kpeg/grammar_renderer' require 'kpeg/code_generator' require 'stringio' class TestKPegFormatParserRoundtrip < Minitest::Test PATH = File.expand_path("../../lib/kpeg/format_parser.kpeg", __FILE__) def test_roundtrip data = File.read(PATH) pr = KPeg::FormatParser.new data assert pr.parse, "Couldn't parse with builtin parser" io = StringIO.new gr = KPeg::GrammarRenderer.new(pr.g) gr.render io cg1 = KPeg::CodeGenerator.new("Test1", pr.g, false) pr2 = cg1.make(io.string) g2 = KPeg::Grammar.new pr2.instance_variable_set(:@g, g2) assert pr2.parse, "Couldn't parse with 2nd generation parser" io2 = StringIO.new gr2 = KPeg::GrammarRenderer.new(g2) gr2.render io2 assert_equal io2.string, io.string cg2 = KPeg::CodeGenerator.new("Test2", g2, false) pr3 = cg2.make(io2.string) g3 = KPeg::Grammar.new pr3.instance_variable_set(:@g, g3) assert pr3.parse, "Couldn't parse with 3rd generation parser" io3 = StringIO.new gr3 = KPeg::GrammarRenderer.new(g3) gr3.render io3 assert_equal io3.string, io2.string cg3 = KPeg::CodeGenerator.new("Test3", g3, false) pr4 = cg3.make(io3.string) g4 = KPeg::Grammar.new pr4.instance_variable_set(:@g, g4) assert pr4.parse, "Couldn't parse with 4th generation parser" io4 = StringIO.new gr4 = KPeg::GrammarRenderer.new(g4) gr4.render io4 assert_equal io4.string, io3.string end def test_roundtrip_standalone data = File.read(PATH) pr = KPeg::FormatParser.new data assert pr.parse, "Couldn't parse with builtin parser" io = StringIO.new gr = KPeg::GrammarRenderer.new(pr.g) gr.render io cg1 = KPeg::CodeGenerator.new("Test1", pr.g, false) cg1.standalone = true pr2 = cg1.make(io.string) g2 = KPeg::Grammar.new pr2.instance_variable_set(:@g, g2) assert pr2.parse, "Couldn't parse with 2nd generation parser" io2 = StringIO.new gr2 = KPeg::GrammarRenderer.new(g2) gr2.render io2 assert_equal io2.string, io.string cg2 = KPeg::CodeGenerator.new("Test2", g2, false) cg2.standalone = true pr3 = cg2.make(io2.string) g3 = KPeg::Grammar.new pr3.instance_variable_set(:@g, g3) assert pr3.parse, "Couldn't parse with 3rd generation parser" io3 = StringIO.new gr3 = KPeg::GrammarRenderer.new(g3) gr3.render io3 assert_equal io3.string, io2.string cg3 = KPeg::CodeGenerator.new("Test3", g3, false) cg3.standalone = true pr4 = cg3.make(io3.string) g4 = KPeg::Grammar.new pr4.instance_variable_set(:@g, g4) assert pr4.parse, "Couldn't parse with 4th generation parser" io4 = StringIO.new gr4 = KPeg::GrammarRenderer.new(g4) gr4.render io4 assert_equal io4.string, io3.string end end kpeg-1.0.0/test/test_kpeg_grammar_renderer.rb0000644000175000017500000001305512321266363017775 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'kpeg/grammar_renderer' require 'stringio' class TestKPegGrammarRenderer < Minitest::Test def test_escape str = "hello\nbob" assert_equal 'hello\nbob', KPeg::GrammarRenderer.escape(str) str = "hello\tbob" assert_equal 'hello\tbob', KPeg::GrammarRenderer.escape(str) str = "\\" assert_equal '\\\\', KPeg::GrammarRenderer.escape(str) str = 'hello"bob"' assert_equal 'hello\\"bob\\"', KPeg::GrammarRenderer.escape(str) end def test_invoke gram = KPeg.grammar do |g| g.root = g.invoke("greeting") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = @greeting\n", io.string end def test_invoke_with_args gram = KPeg.grammar do |g| g.root = g.invoke("greeting", "(1,2)") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = @greeting(1,2)\n", io.string end def test_foreign_invoke gram = KPeg.grammar do |g| g.root = g.foreign_invoke("blah", "greeting") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = %blah.greeting\n", io.string end def test_foreign_invoke_with_args gram = KPeg.grammar do |g| g.root = g.foreign_invoke("blah", "greeting", "(1,2)") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = %blah.greeting(1,2)\n", io.string end def test_dot_render gram = KPeg.grammar do |g| g.root = g.dot end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = .\n", io.string end def test_tag_render gram = KPeg.grammar do |g| g.root = g.seq("+", g.t("hello", "greeting")) end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = \"+\" \"hello\":greeting\n", io.string end def test_tag_render_parens gram = KPeg.grammar do |g| g.root = g.t(g.seq(:b, :c), "greeting") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = (b c):greeting\n", io.string end def test_grammar_renderer gram = KPeg.grammar do |g| g.some = g.range('0', '9') g.num = g.reg(/[0-9]/) g.term = g.any( [:term, "+", :term], [:term, "-", :term], :fact) g.fact = g.any( [:fact, "*", :fact], [:fact, "/", :fact], :num ) g.root = g.term end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-GRAM some = [0-9] num = /[0-9]/ term = term "+" term | term "-" term | fact fact = fact "*" fact | fact "/" fact | num root = term GRAM assert_equal expected, io.string end def test_grammar_renderer2 gram = KPeg.grammar do |g| g.num = g.reg(/[0-9]/) g.term = g.any( [:term, g.t("+"), :term], [:term, g.any("-", "$"), :term], :fact) g.fact = g.any( [:fact, g.t("*", "op"), :fact], [:fact, "/", :fact], :num ) g.root = g.term end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-GRAM num = /[0-9]/ term = term "+" term | term ("-" | "$") term | fact fact = fact "*":op fact | fact "/" fact | num root = term GRAM assert_equal expected, io.string end def test_action gram = KPeg.grammar do |g| g.root = g.seq("hello", g.action("3 + 4")) end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-GRAM root = "hello" {3 + 4} GRAM assert_equal expected, io.string end def test_collect gram = KPeg.grammar do |g| g.root = g.collect(g.many(g.range("a", "z"))) end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-GRAM root = < [a-z]+ > GRAM assert_equal expected, io.string end def test_directives gram = KPeg.grammar do |g| g.root = g.dot g.add_directive "header", g.action("\n# coding: UTF-8\n") g.add_directive "footer", g.action("\nrequire 'something'\n") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-TXT %% footer { require 'something' } %% header { # coding: UTF-8 } root = . TXT assert_equal expected, io.string end def test_setup_actions gram = KPeg.grammar do |g| g.root = g.dot g.add_setup g.action(" attr_reader :foo ") end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-TXT %% { attr_reader :foo } root = . TXT assert_equal expected, io.string end def test_variables gram = KPeg.grammar do |g| g.root = g.dot g.set_variable "name", "Foo" g.set_variable "custom_initialize", "true" end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) expected = <<-TXT %% custom_initialize = true %% name = Foo root = . TXT assert_equal expected, io.string end def test_multiple_render gram = KPeg.grammar do |g| g.root = g.multiple("a", 3, 5) end io = StringIO.new gr = KPeg::GrammarRenderer.new(gram) gr.render(io) assert_equal "root = \"a\"[3, 5]\n", io.string end end kpeg-1.0.0/test/test_kpeg_code_generator.rb0000644000175000017500000010061712321266363017442 0ustar domidomi# encoding: utf-8 require 'minitest/autorun' require 'kpeg' require 'kpeg/code_generator' require 'stringio' class TestKPegCodeGenerator < Minitest::Test def test_dot gram = KPeg.grammar do |g| g.root = g.dot end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_str gram = KPeg.grammar do |g| g.root = g.str("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello" def _root _tmp = match_string("hello") set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\"") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_reg gram = KPeg.grammar do |g| g.root = g.reg(/[0-9]/) end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = /[0-9]/ def _root _tmp = scan(/\\A(?-mix:[0-9])/) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "/[0-9]/") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("9") assert cg.parse("1") assert !cg.parse("a") end def test_reg_unicode gram = KPeg.grammar do |g| g.root = g.reg(/./u) end if RUBY_VERSION > "1.8.7" str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = /./ def _root _tmp = scan(/\\A(?-mix:.)/) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "/./") # :startdoc: end STR else str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = /./u def _root _tmp = scan(/\\A(?-mix:.)/u) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "/./u") # :startdoc: end STR end cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("う") assert cg.parse("a") end def test_char_range gram = KPeg.grammar do |g| g.root = g.range("a", "z") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = [a-z] def _root _save = self.pos _tmp = get_byte if _tmp unless _tmp >= 97 and _tmp <= 122 self.pos = _save _tmp = nil end end set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "[a-z]") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("z") assert cg.parse("a") assert !cg.parse("0") end def test_char_range_in_seq gram = KPeg.grammar do |g| g.root = g.seq(g.range("a", "z"), "hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = [a-z] "hello" def _root _save = self.pos while true # sequence _save1 = self.pos _tmp = get_byte if _tmp unless _tmp >= 97 and _tmp <= 122 self.pos = _save1 _tmp = nil end end unless _tmp self.pos = _save break end _tmp = match_string("hello") unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "[a-z] \\\"hello\\\"") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("ahello") assert cg.parse("zhello") assert !cg.parse("0hello") assert !cg.parse("ajello") end def test_any gram = KPeg.grammar do |g| g.root = g.any("hello", "world") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = ("hello" | "world") def _root _save = self.pos while true # choice _tmp = match_string("hello") break if _tmp self.pos = _save _tmp = match_string("world") break if _tmp self.pos = _save break end # end choice set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "(\\\"hello\\\" | \\\"world\\\")") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") assert cg.parse("world") assert !cg.parse("jello") end def test_any_resets_pos gram = KPeg.grammar do |g| g.root = g.any(g.seq("hello", "world"), "hello balloons") end cg = KPeg::CodeGenerator.new "Test", gram code = cg.make("helloworld") assert code.parse assert_equal 10, code.pos assert cg.parse("hello balloons") end def test_maybe gram = KPeg.grammar do |g| g.root = g.maybe("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello"? def _root _save = self.pos _tmp = match_string("hello") unless _tmp _tmp = true self.pos = _save end set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\"?") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") assert cg.parse("jello") end def test_maybe_resets_pos gram = KPeg.grammar do |g| g.root = g.maybe(g.seq("hello", "world")) end cg = KPeg::CodeGenerator.new "Test", gram assert cg.parse("helloworld") code = cg.make("hellojello") assert code.parse assert_equal 0, code.pos end def test_kleene gram = KPeg.grammar do |g| g.root = g.kleene("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello"* def _root while true _tmp = match_string("hello") break unless _tmp end _tmp = true set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\"*") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hellohellohello") assert code.parse assert_equal 15, code.pos end def test_kleene_reset_pos gram = KPeg.grammar do |g| g.root = g.kleene(g.seq("hello", "world")) end cg = KPeg::CodeGenerator.new "Test", gram code = cg.make("helloworldhelloworld") assert code.parse assert_equal 20, code.pos code = cg.make("hellojello") assert code.parse assert_equal 0, code.pos end def test_many gram = KPeg.grammar do |g| g.root = g.many("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello"+ def _root _save = self.pos _tmp = match_string("hello") if _tmp while true _tmp = match_string("hello") break unless _tmp end _tmp = true else self.pos = _save end set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\"+") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hellohello") assert code.parse assert_equal 10, code.pos code = cg.make("hello") assert code.parse assert_equal 5, code.pos code = cg.make("") assert !code.parse end def test_many_resets_pos gram = KPeg.grammar do |g| g.root = g.many(g.seq("hello", "world")) end cg = KPeg::CodeGenerator.new "Test", gram code = cg.make("helloworldhelloworld") assert code.parse assert_equal 20, code.pos code = cg.make("hellojello") assert !code.parse assert_equal 0, code.pos end def test_multiple gram = KPeg.grammar do |g| g.root = g.multiple("hello", 5, 9) end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello"[5, 9] def _root _save = self.pos _count = 0 while true _tmp = match_string("hello") if _tmp _count += 1 break if _count == 9 else break end end if _count >= 5 _tmp = true else self.pos = _save _tmp = nil end set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\"[5, 9]") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output end def test_seq gram = KPeg.grammar do |g| g.root = g.seq("hello", "world") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello" "world" def _root _save = self.pos while true # sequence _tmp = match_string("hello") unless _tmp self.pos = _save break end _tmp = match_string("world") unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\" \\\"world\\\"") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output end def test_seq_resets_pos gram = KPeg.grammar do |g| g.root = g.seq("hello", "world") end cg = KPeg::CodeGenerator.new "Test", gram code = cg.make("helloworld") assert code.parse code = cg.make("hellojello") assert !code.parse assert_equal 0, code.pos end def test_andp gram = KPeg.grammar do |g| g.root = g.andp("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = &"hello" def _root _save = self.pos _tmp = match_string("hello") self.pos = _save set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "&\\\"hello\\\"") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert code.parse assert_equal 0, code.pos code = cg.make("jello") assert !code.parse assert_equal 0, code.pos end def test_andp_for_action gram = KPeg.grammar do |g| g.root = g.andp(g.action(" !defined? @fail ")) end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = &{ !defined? @fail } def _root _save = self.pos _tmp = begin; !defined? @fail ; end self.pos = _save set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "&{ !defined? @fail }") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert code.parse assert_equal 0, code.pos code = cg.make("jello") code.instance_variable_set :@fail, true assert !code.parse assert_equal 0, code.pos end def test_notp gram = KPeg.grammar do |g| g.root = g.notp("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = !"hello" def _root _save = self.pos _tmp = match_string("hello") _tmp = _tmp ? nil : true self.pos = _save set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "!\\\"hello\\\"") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert !code.parse assert_equal 0, code.pos code = cg.make("jello") assert code.parse assert_equal 0, code.pos end def test_notp_for_action gram = KPeg.grammar do |g| g.root = g.notp(g.action(" defined? @fail ")) end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = !{ defined? @fail } def _root _save = self.pos _tmp = begin; defined? @fail ; end _tmp = _tmp ? nil : true self.pos = _save set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "!{ defined? @fail }") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert code.parse assert_equal 0, code.pos code = cg.make("jello") code.instance_variable_set :@fail, true assert !code.parse assert_equal 0, code.pos end def test_ref gram = KPeg.grammar do |g| g.greeting = "hello" g.root = g.ref("greeting") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # greeting = "hello" def _greeting _tmp = match_string("hello") set_failed_rule :_greeting unless _tmp return _tmp end # root = greeting def _root _tmp = apply(:_greeting) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_greeting] = rule_info("greeting", "\\\"hello\\\"") Rules[:_root] = rule_info("root", "greeting") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_invoke gram = KPeg.grammar do |g| g.greeting = "hello" g.root = g.invoke("greeting") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # greeting = "hello" def _greeting _tmp = match_string("hello") set_failed_rule :_greeting unless _tmp return _tmp end # root = @greeting def _root _tmp = _greeting() set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_greeting] = rule_info("greeting", "\\\"hello\\\"") Rules[:_root] = rule_info("root", "@greeting") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_invoke_with_args gram = KPeg.grammar do |g| g.set("greeting", "hello", ["a", "b"]) g.root = g.invoke("greeting", "(1,2)") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # greeting = "hello" def _greeting(a,b) _tmp = match_string("hello") set_failed_rule :_greeting unless _tmp return _tmp end # root = @greeting(1,2) def _root _tmp = _greeting(1,2) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_greeting] = rule_info("greeting", "\\\"hello\\\"") Rules[:_root] = rule_info("root", "@greeting(1,2)") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end gram = <<-GRAM greeting = "hello" greeting2(a,b) = "hello" GRAM KPeg.compile gram, "TestParser", self def test_foreign_invoke gram = KPeg.grammar do |g| g.add_foreign_grammar "blah", "TestKPegCodeGenerator::TestParser" g.root = g.foreign_invoke("blah", "greeting") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: def setup_foreign_grammar @_grammar_blah = TestKPegCodeGenerator::TestParser.new(nil) end # root = %blah.greeting def _root _tmp = @_grammar_blah.external_invoke(self, :_greeting) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "%blah.greeting") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_foreign_invoke_with_args gram = KPeg.grammar do |g| g.add_foreign_grammar "blah", "TestKPegCodeGenerator::TestParser" g.root = g.foreign_invoke("blah", "greeting2", "(1,2)") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: def setup_foreign_grammar @_grammar_blah = TestKPegCodeGenerator::TestParser.new(nil) end # root = %blah.greeting2(1,2) def _root _tmp = @_grammar_blah.external_invoke(self, :_greeting2, 1,2) set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "%blah.greeting2(1,2)") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_tag gram = KPeg.grammar do |g| g.root = g.t g.str("hello"), "t" end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello":t def _root _tmp = match_string("hello") t = @result set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\":t") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output end def test_noname_tag gram = KPeg.grammar do |g| g.root = g.t g.str("hello") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = "hello" def _root _tmp = match_string("hello") set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "\\\"hello\\\"") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output end def test_tag_maybe gram = KPeg.grammar do |g| g.hello = g.seq(g.collect("hello"), g.action("text")) g.root = g.seq g.t(g.maybe(:hello), "lots"), g.action("lots") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # hello = < "hello" > {text} def _hello _save = self.pos while true # sequence _text_start = self.pos _tmp = match_string("hello") if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_hello unless _tmp return _tmp end # root = hello?:lots {lots} def _root _save = self.pos while true # sequence _save1 = self.pos _tmp = apply(:_hello) @result = nil unless _tmp unless _tmp _tmp = true self.pos = _save1 end lots = @result unless _tmp self.pos = _save break end @result = begin; lots; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_hello] = rule_info("hello", "< \\\"hello\\\" > {text}") Rules[:_root] = rule_info("root", "hello?:lots {lots}") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert code.parse assert_equal "hello", code.result code = cg.make("") assert code.parse assert_equal nil, code.result end def test_tag_multiple gram = KPeg.grammar do |g| g.hello = g.seq(g.collect("hello"), g.action("text")) g.root = g.seq g.t(g.kleene(:hello), "lots"), g.action("lots") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # hello = < "hello" > {text} def _hello _save = self.pos while true # sequence _text_start = self.pos _tmp = match_string("hello") if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_hello unless _tmp return _tmp end # root = hello*:lots {lots} def _root _save = self.pos while true # sequence _ary = [] while true _tmp = apply(:_hello) _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary lots = @result unless _tmp self.pos = _save break end @result = begin; lots; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_hello] = rule_info("hello", "< \\\"hello\\\" > {text}") Rules[:_root] = rule_info("root", "hello*:lots {lots}") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hellohello") assert code.parse assert_equal ["hello", "hello"], code.result code = cg.make("hello") assert code.parse assert_equal ["hello"], code.result code = cg.make("") assert code.parse assert_equal [], code.result end def test_tag_many gram = KPeg.grammar do |g| g.hello = g.seq(g.collect("hello"), g.action("text")) g.root = g.seq g.t(g.many(:hello), "lots"), g.action("lots") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # hello = < "hello" > {text} def _hello _save = self.pos while true # sequence _text_start = self.pos _tmp = match_string("hello") if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_hello unless _tmp return _tmp end # root = hello+:lots {lots} def _root _save = self.pos while true # sequence _save1 = self.pos _ary = [] _tmp = apply(:_hello) if _tmp _ary << @result while true _tmp = apply(:_hello) _ary << @result if _tmp break unless _tmp end _tmp = true @result = _ary else self.pos = _save1 end lots = @result unless _tmp self.pos = _save break end @result = begin; lots; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_hello] = rule_info("hello", "< \\\"hello\\\" > {text}") Rules[:_root] = rule_info("root", "hello+:lots {lots}") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hellohello") assert code.parse assert_equal ["hello", "hello"], code.result code = cg.make("hello") assert code.parse assert_equal ["hello"], code.result code = cg.make("") assert !code.parse end def test_action gram = KPeg.grammar do |g| g.root = g.action "3 + 4" end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = {3 + 4} def _root @result = begin; 3 + 4; end _tmp = true set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "{3 + 4}") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("") assert code.parse assert_equal 7, code.result end def test_collect gram = KPeg.grammar do |g| g.root = g.seq(g.collect("hello"), g.action(" text ")) end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = < "hello" > { text } def _root _save = self.pos while true # sequence _text_start = self.pos _tmp = match_string("hello") if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "< \\\"hello\\\" > { text }") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert code.parse assert_equal "hello", code.result end def test_bounds gram = KPeg.grammar do |g| g.root = g.seq(g.bounds("hello"), g.action(" bounds ")) end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = @< "hello" > { bounds } def _root _save = self.pos while true # sequence _bounds_start = self.pos _tmp = match_string("hello") if _tmp bounds = [_bounds_start, self.pos] end unless _tmp self.pos = _save break end @result = begin; bounds ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", "@< \\\"hello\\\" > { bounds }") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output code = cg.make("hello") assert code.parse assert_equal [0,5], code.result end def test_standalone_region gram = KPeg.grammar do |g| g.root = g.dot end cg = KPeg::CodeGenerator.new "Test", gram expected = <<-EXPECTED # This is distinct from setup_parser so that a standalone parser # can redefine #initialize and still have access to the proper # parser setup code. def initialize(str, debug=false) setup_parser(str, debug) end EXPECTED assert_equal expected, cg.standalone_region('compiled_parser.rb', 'INITIALIZE') end def test_parse_error gram = KPeg.grammar do |g| g.world = "world" g.root = g.seq("hello", :world) end cg = KPeg::CodeGenerator.new "Test", gram code = cg.make("no") assert !code.parse assert_equal 0, code.failing_rule_offset cg2 = KPeg::CodeGenerator.new "Test", gram code = cg2.make("hellono") assert !code.parse assert_equal 5, code.failing_rule_offset end def test_directive_footer gram = KPeg.grammar do |g| g.root = g.dot g.directives['footer'] = g.action("\n# require 'some/subclass'\n") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end # require 'some/subclass' STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_directive_header gram = KPeg.grammar do |g| g.root = g.dot g.directives['header'] = g.action("\n# coding: UTF-8\n") end str = <<-STR # coding: UTF-8 require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_directive_pre_class gram = KPeg.grammar do |g| g.root = g.dot g.directives['pre-class'] = g.action("\n# some comment\n") end str = <<-STR require 'kpeg/compiled_parser' # some comment class Test < KPeg::CompiledParser # :stopdoc: # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_directive_pre_class_standalone gram = KPeg.grammar do |g| g.root = g.dot g.directives['pre-class'] = g.action("\n# some comment\n") end cg = KPeg::CodeGenerator.new "Test", gram cg.standalone = true assert_match %r%^# some comment%, cg.output end def test_setup_actions gram = KPeg.grammar do |g| g.root = g.dot g.add_setup g.action(" attr_reader :foo ") end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser attr_reader :foo # :stopdoc: # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_output_standalone gram = KPeg.grammar do |g| g.root = g.dot end cg = KPeg::CodeGenerator.new "Test", gram cg.standalone = true # if this fails, also change test_variable_custom_initialize assert_match 'def initialize(str, debug=false)', cg.output assert_match '# :stopdoc:', cg.output assert_match '# :startdoc:', cg.output assert cg.parse("hello") end def test_variable_custom_initialize gram = KPeg.grammar do |g| g.root = g.dot g.variables['custom_initialize'] = 'whatever' end cg = KPeg::CodeGenerator.new "Test", gram cg.standalone = true refute_match 'def initialize(str, debug=false)', cg.output end def test_ast_generation gram = KPeg.grammar do |g| g.root = g.dot g.set_variable "bracket", "ast BracketOperator(receiver, argument)" g.set_variable "simple", "ast Simple()" g.set_variable "simple2", "ast Simple2" end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: module AST class Node; end class BracketOperator < Node def initialize(receiver, argument) @receiver = receiver @argument = argument end attr_reader :receiver attr_reader :argument end class Simple < Node def initialize() end end class Simple2 < Node def initialize() end end end module ASTConstruction def bracket(receiver, argument) AST::BracketOperator.new(receiver, argument) end def simple() AST::Simple.new() end def simple2() AST::Simple2.new() end end include ASTConstruction # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end def test_ast_generation_in_different_location gram = KPeg.grammar do |g| g.root = g.dot g.set_variable "bracket", "ast BracketOperator(receiver, argument)" g.set_variable "ast-location", "MegaAST" end str = <<-STR require 'kpeg/compiled_parser' class Test < KPeg::CompiledParser # :stopdoc: module MegaAST class Node; end class BracketOperator < Node def initialize(receiver, argument) @receiver = receiver @argument = argument end attr_reader :receiver attr_reader :argument end end module MegaASTConstruction def bracket(receiver, argument) MegaAST::BracketOperator.new(receiver, argument) end end include MegaASTConstruction # root = . def _root _tmp = get_byte set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_root] = rule_info("root", ".") # :startdoc: end STR cg = KPeg::CodeGenerator.new "Test", gram assert_equal str, cg.output assert cg.parse("hello") end end kpeg-1.0.0/test/test_kpeg_compiled_parser.rb0000644000175000017500000000344712321266363017635 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'kpeg/compiled_parser' require 'stringio' class TestKPegCompiledParser < Minitest::Test gram = <<-GRAM letter = [a-z] root = letter GRAM KPeg.compile gram, "TestParser", self gram = <<-GRAM %test = TestKPegCompiledParser::TestParser root = %test.letter "!" GRAM KPeg.compile gram, "CompTestParser", self def test_current_column r = TestParser.new "hello\nsir" assert_equal 2, r.current_column(1) assert_equal 6, r.current_column(5) assert_equal 1, r.current_column(7) assert_equal 4, r.current_column(10) end def test_failed_rule r = TestParser.new "9" assert !r.parse, "shouldn't parse" assert_equal :_letter, r.failed_rule end def test_failure_info r = TestParser.new "9" assert !r.parse, "shouldn't parse" expected = "line 1, column 1: failed rule 'letter' = '[a-z]'" assert_equal 0, r.failing_rule_offset assert_equal expected, r.failure_info end def test_failure_caret r = TestParser.new "9" assert !r.parse, "shouldn't parse" assert_equal "9\n^", r.failure_caret end def test_failure_character r = TestParser.new "9" assert !r.parse, "shouldn't parse" assert_equal "9", r.failure_character end def test_failure_oneline r = TestParser.new "9" assert !r.parse, "shouldn't parse" expected = "@1:1 failed rule 'letter', got '9'" assert_equal expected, r.failure_oneline end def test_composite_grammar r = CompTestParser.new "l!" assert r.parse, "should parse" end def test_composite_grammar_failure r = CompTestParser.new "9" assert !r.parse, "should parse" expected = "@1:1 failed rule 'TestKPegCompiledParser::TestParser#_letter', got '9'" assert_equal expected, r.failure_oneline end end kpeg-1.0.0/test/test_kpeg.rb0000644000175000017500000002241212321266363014376 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'stringio' class TestKPeg < Minitest::Test def assert_match(m, str) assert_kind_of KPeg::MatchString, m assert_equal str, m.string end def test_dot gram = KPeg.grammar do |g| g.root = g.dot end assert_match KPeg.match("q", gram), "q" end def test_str gram = KPeg.grammar do |g| g.root = g.str("hello") end assert_match KPeg.match("hello", gram), "hello" assert_equal nil, KPeg.match("vador", gram) end def test_reg gram = KPeg.grammar do |g| g.root = g.reg(/[0-9]/) end assert_match KPeg.match("3", gram), "3" end def test_char_range gram = KPeg.grammar do |g| g.root = g.range('0', '9') end assert_match KPeg.match("3", gram), "3" end def test_any gram = KPeg.grammar do |g| g.root = g.any g.str("hello"), g.str("chicken") end assert_match KPeg.match("hello", gram), "hello" assert_match KPeg.match("chicken", gram), "chicken" assert_equal nil, KPeg.match("vador", gram) end def test_maybe gram = KPeg.grammar do |g| g.root = g.maybe g.str("hello") end m = KPeg.match "hello", gram assert_kind_of KPeg::Match, m assert_equal 1, m.matches.size assert_match m.matches[0], "hello" m = KPeg.match "surprise", gram assert_kind_of KPeg::Match, m assert_equal 0, m.matches.size end def test_many gram = KPeg.grammar do |g| g.root = g.many g.str("run") end m = KPeg.match "runrunrun", gram assert_kind_of KPeg::Match, m assert_equal 3, m.matches.size m.matches.each do |sm| assert_match sm, "run" end assert_equal nil, KPeg.match("vador", gram) end def test_kleene gram = KPeg.grammar do |g| g.root = g.kleene g.str("run") end m = KPeg.match "runrunrun", gram assert_kind_of KPeg::Match, m assert_equal 3, m.matches.size m.matches.each do |sm| assert_match sm, "run" end m = KPeg.match "chicken", gram assert_kind_of KPeg::Match, m assert_equal 0, m.matches.size end def test_multiple gram = KPeg.grammar do |g| g.root = g.multiple g.str("run"), 2, 4 end m = KPeg.match "runrun", gram assert_kind_of KPeg::Match, m assert_equal 2, m.matches.size m.matches.each do |sm| assert_match sm, "run" end m = KPeg.match "runrunrun", gram assert_kind_of KPeg::Match, m assert_equal 3, m.matches.size m.matches.each do |sm| assert_match sm, "run" end m = KPeg.match "runrunrunrun", gram assert_kind_of KPeg::Match, m assert_equal 4, m.matches.size m.matches.each do |sm| assert_match sm, "run" end assert_equal nil, KPeg.match("run", gram) assert_equal nil, KPeg.match("runrunrunrunrun", gram) assert_equal nil, KPeg.match("vador", gram) end def test_seq gram = KPeg.grammar do |g| g.root = g.seq g.str("hello"), g.str(", world") end m = KPeg.match "hello, world", gram assert_kind_of KPeg::Match, m assert_match m.matches[0], "hello" assert_match m.matches[1], ", world" assert_equal m.value, ["hello", ", world"] assert_equal nil, KPeg.match("vador", gram) assert_equal nil, KPeg.match("hello, vador", gram) end def test_andp gram = KPeg.grammar do |g| g.root = g.seq g.andp(g.str("h")), g.str("hello") end m = KPeg.match "hello", gram assert_equal m.matches.size, 2 assert_match m.matches[0], "" assert_match m.matches[1], "hello" end def test_notp gram = KPeg.grammar do |g| g.root = g.seq g.notp(g.str("g")), g.str("hello") end m = KPeg.match "hello", gram assert_equal m.matches.size, 2 assert_match m.matches[0], "" assert_match m.matches[1], "hello" end def test_ref gram = KPeg.grammar do |g| g.greeting = g.str("hello") g.root = g.ref "greeting" end m = KPeg.match "hello", gram assert_match m, "hello" end def test_invoke gram = KPeg.grammar do |g| g.greeting = g.str("hello") g.root = g.invoke "greeting" end m = KPeg.match "hello", gram assert_match m, "hello" end def test_foreign_ref g1 = KPeg.grammar do |g| g.greeting = "hello" end g2 = KPeg.grammar do |g| g.root = g.ref("greeting", g1) end m = KPeg.match "hello", g2 assert_match m, "hello" end def test_foreign_ref_with_ref g1 = KPeg.grammar do |g| g.name = ", evan" g.greeting = g.seq("hello", :name) end g2 = KPeg.grammar do |g| g.root = g.ref("greeting", g1) end m = KPeg.match "hello, evan", g2 assert_match m.matches[0], "hello" assert_match m.matches[1], ", evan" end def test_tag_with_name gram = KPeg.grammar do |g| g.root = g.seq(" ", g.t("hello", "greeting")) end m = KPeg.match " hello", gram assert_equal 2, m.matches.size tag = m.matches[1] assert_kind_of KPeg::Tag, tag.op assert_equal 1, tag.matches.size assert_match tag.matches[0], "hello" # show that tag influences the value of the sequence assert_equal m.value, "hello" end def test_tag_without_name gram = KPeg.grammar do |g| g.root = g.seq(" ", g.t("hello")) end m = KPeg.match " hello", gram assert_equal m.value, "hello" end def test_action gram = KPeg.grammar do |g| g.root = g.seq("hello", g.action("b + c")) end m = KPeg.match "hello", gram assert_equal 2, m.matches.size assert_match m.matches[0], "hello" action = m.matches[1] assert_equal action.op.action, "b + c" end def test_naming gram = KPeg.grammar do |g| g.greeting = g.str("hello") g.root = g.greeting end m = KPeg.match "hello", gram assert_match m, "hello" end def test_matching_curly gram = KPeg.grammar do |g| g.curly = g.seq("{", g.kleene(g.any(/[^{}]+/, :curly)), "}") g.root = :curly end m = KPeg.match "{ hello }", gram assert_match m.matches[0], "{" assert_match m.matches[1].matches[0], " hello " assert_match m.matches[2], "}" parc = KPeg::Parser.new "{ foo { bar } }", gram m = parc.parse assert_equal "{ foo { bar } }", m.total_string parc = KPeg::Parser.new "{ foo {\nbar }\n }", gram m = parc.parse assert_equal "{ foo {\nbar }\n }", m.total_string end def test_collect gram = KPeg.grammar do |g| g.root = g.collect(g.many(/[a-z]/)) end m = KPeg.match "hellomatch", gram assert_equal "hellomatch", m.value end def test_memoization gram = KPeg.grammar do |g| g.one = g.str("1") g.two = g.str("2") g.root = g.any( [:one, "-", :two], [:one, "+", :two] ) end parser = KPeg::Parser.new "1+2", gram m = parser.parse assert_equal 3, m.matches.size assert_match m.matches[0], "1" assert_match m.matches[1], "+" assert_match m.matches[2], "2" one = gram.find("one") two = gram.find("two") # We try 1 twice assert_equal 2, parser.memoizations[one][0].uses # but we only get as far as 2 once assert_equal 1, parser.memoizations[two][2].uses end def test_left_recursion gram = KPeg.grammar do |g| g.num = g.reg(/[0-9]/) g.expr = g.any [:expr, "-", :num], :num g.root = g.expr end parser = KPeg::Parser.new "1-2-3", gram m = parser.parse assert_equal 3, m.matches.size left = m.matches[0] assert_equal 3, left.matches.size assert_match left.matches[0], "1" assert_match left.matches[1], "-" assert_match left.matches[2], "2" assert_match m.matches[1], "-" assert_match m.matches[2], "3" parser = KPeg::Parser.new "hello", gram m = parser.parse assert_equal nil, m end def test_math_grammar gram = KPeg.grammar do |g| g.num = '0'..'9' g.term = g.seq(:term, "+", :term) \ | g.seq(:term, "-", :term) \ | :fact g.fact = g.seq(:fact, "*", :fact) \ | g.seq(:fact, "/", :fact) \ | :num g.root = g.term end sub = KPeg.match "4*3-8/9", gram mul = sub.matches[0] div = sub.matches[2] assert_match mul.matches[0], "4" assert_match mul.matches[1], "*" assert_match mul.matches[2], "3" assert_match sub.matches[1], "-" assert_match div.matches[0], "8" assert_match div.matches[1], "/" assert_match div.matches[2], "9" end def test_calc vars = {} gram = KPeg.grammar do |g| g.spaces = /\s*/ g.var = 'a'..'z' g.num = g.lit(/[0-9]+/) { |i| i.to_i } g.pri = g.seq(:spaces, :var) { |s,v| vars[v] } \ | g.seq(:spaces, :num) { |s,n| n } \ | g.seq('(', :expr, ')') { |_,e,_| e } g.mul = g.seq(:mul, "*", :pri) { |x,_,y| x * y } \ | g.seq(:mul, "/", :pri) { |x,_,y| x / y } \ | :pri g.add = g.seq(:add, "+", :mul) { |x,_,y| x + y } \ | g.seq(:add, "-", :mul) { |x,_,y| x - y } \ | :mul g.expr = g.seq(:var, "=", :expr) { |v,_,e| vars[v] = e } \ | :add g.root = g.seq(g.kleene(:expr), :spaces) { |e,_| e } end m = KPeg.match "3+4*5", gram assert_equal 23, m.value m = KPeg.match "x=2", gram assert_equal 2, m.value m = KPeg.match "x=x*7", gram assert_equal 14, m.value end end kpeg-1.0.0/test/test_kpeg_string_escape.rb0000644000175000017500000000122312321266363017301 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'kpeg/string_escape' class TestKPegStringEscape < Minitest::Test def test_bell assert_equal '\b', parse("\b") end def test_carriage_return assert_equal '\r', parse("\r") end def test_newline assert_equal '\n', parse("\n") end def test_quote assert_equal '\\\\\"', parse('\\"') end def test_slash assert_equal '\\\\', parse('\\') end def test_tab assert_equal '\t', parse("\t") end def parse(str, embed = false) se = KPeg::StringEscape.new(str) rule = (embed ? 'embed' : nil) se.raise_error unless se.parse(rule) se.text end end kpeg-1.0.0/test/test_kpeg_format.rb0000644000175000017500000003272012321266363015751 0ustar domidomirequire 'minitest/autorun' require 'kpeg' require 'kpeg/format_parser' require 'kpeg/grammar_renderer' require 'stringio' require 'rubygems' class TestKPegFormat < Minitest::Test G = KPeg::Grammar.new gram = File.read File.expand_path("../../lib/kpeg/format_parser.kpeg", __FILE__) KPeg.compile gram, "TestParser", self def match(str, gram=nil, log=false) parc = TestParser.new str parc.raise_error unless parc.parse return parc.grammar end def assert_rule(expect, gram, name="a") actual = gram.find name.to_s assert_equal expect, actual.op end def test_assignment assert_rule G.ref("b"), match("a=b"), "a" end def test_apply_with_arg assert_rule G.ref("b", nil, "(x)"), match("a=b(x)"), "a" end def test_invoke assert_rule G.invoke("b"), match("a=@b"), "a" end def test_assignment_hyphen_only assert_rule G.ref("b"), match("-=b"), "-" end def test_assigment_sp assert_rule G.ref("b"), match(" a=b") assert_rule G.ref("b"), match(" a =b") assert_rule G.ref("b"), match(" a = b") assert_rule G.ref("b"), match(" a = b ") end def test_assign_with_arg gram = match("a(t) = b") rule = gram.find "a" assert_equal ["t"], rule.arguments end def test_assign_with_arg_disambiguated_from_grouping str = <<-STR a = c b(p) = x STR assert match(str) # HACK what is this testing? end def test_assign_with_multiple_args gram = match("a(t,x) = b") rule = gram.find "a" assert_equal ["t", "x"], rule.arguments end def test_assign_with_args_spacing gram = match("a( t) = b") rule = gram.find "a" assert_equal ["t"], rule.arguments gram = match("a( t ) = b") rule = gram.find "a" assert_equal ["t"], rule.arguments gram = match("a( t,x) = b") rule = gram.find "a" assert_equal ["t", "x"], rule.arguments gram = match("a( t,x ) = b") rule = gram.find "a" assert_equal ["t", "x"], rule.arguments gram = match("a( t ,x ) = b") rule = gram.find "a" assert_equal ["t", "x"], rule.arguments gram = match("a( t , x ) = b") rule = gram.find "a" assert_equal ["t", "x"], rule.arguments end def test_invoke_with_arg gram = match("a=b(1)") rule = gram.find "a" assert_equal "(1)", rule.op.arguments end def test_invoke_with_double_quoted_strings m = match "a=b(\")\")" assert_equal "(\")\")", m.find("a").op.arguments end def test_invoke_with_single_quoted_strings m = match "a=b(')')" assert_equal "(')')", m.find("a").op.arguments end def test_invoke_with_multiple_args assert_rule G.invoke("b", "(1,2)"), match("a=@b(1,2)"), "a" end def test_invoke_foreign_rule assert_rule G.foreign_invoke("blah", "letters"), match("a=%blah.letters"), "a" end def test_add_foreign_grammar gram = match "%blah = OtherGrammar" assert_equal "OtherGrammar", gram.foreign_grammars["blah"] end def test_add_foreign_grammar_with_numbers gram = match "%blah = Thing1::OtherGrammar" assert_equal "Thing1::OtherGrammar", gram.foreign_grammars["blah"] end def test_add_foreign_grammar_with_undescore gram = match "%blah = Other_Grammar" assert_equal "Other_Grammar", gram.foreign_grammars["blah"] end def test_invoke_parent_rule assert_rule G.foreign_invoke("parent", "letters"), match("a=^letters"), "a" end def test_dot assert_rule G.dot, match("a=.") end def test_string assert_rule G.str(""), match('a=""') assert_rule G.str("hello"), match('a="hello"') assert_rule G.str("hello\ngoodbye"), match('a="hello\ngoodbye"') assert_rule G.str("hello\n\0goodbye"), match('a="hello\n\0goodbye"') assert_rule G.str("hello\n\017goodbye"), match('a="hello\n\017goodbye"') assert_rule G.str("hello\n\017goodbye"), match('a="hello\n\017goodbye"') assert_rule G.str("hello\n\9goodbye"), match('a="hello\n\9goodbye"') assert_rule G.str("\n\s\r\t\v\f\b\a\r\\\"\0172\x1b"), match('a="\n\s\r\t\v\f\b\a\r\\\\\\"\0172\x1b"') assert_rule G.str("h\"ello"), match('a="h\"ello"') end def test_regexp assert_rule G.reg('foo'), match('a=/foo/') assert_rule G.reg('foo\\/bar'), match('a=/foo\/bar/') assert_rule G.reg('[^"]'), match('a=/[^"]/') end def test_regexp_options if RUBY_VERSION > "1.8.7" assert_rule G.reg(/foo/n), match('a=/foo/n') else assert_rule G.reg(/foo/u), match('a=/foo/u') end end def test_char_range assert_rule G.range("a", "z"), match('a=[a-z]') end def test_maybe assert_rule G.maybe(:b), match('a=b?') end def test_many assert_rule G.many(:b), match('a=b+') end def test_many_sequence assert_rule G.many([:b, :c]), match('a=(b c)+') end def test_many_sequence_with_action assert_rule G.seq(G.many([:b, :c]), G.action(" 1 ")), match('a=(b c)+ { 1 }') end def test_kleene assert_rule G.kleene(:b), match('a=b*') end def test_arbitrary_multiple assert_rule G.multiple(:b, 5, 9), match('a=b[5,9]') end def test_single_value_for_multiple assert_rule G.multiple(:b, 5, 5), match('a=b[5]') end def test_no_max_multiple assert_rule G.multiple(:b, 5, nil), match('a=b[5,*]') end def test_no_max_multiple_sp assert_rule G.multiple(:b, 5, nil), match('a=b[5, *]') assert_rule G.multiple(:b, 5, nil), match('a=b[5, * ]') assert_rule G.multiple(:b, 5, nil), match('a=b[5 , * ]') assert_rule G.multiple(:b, 5, nil), match('a=b[ 5 , * ]') end def test_andp assert_rule G.andp(:c), match('a=&c') end def test_notp assert_rule G.notp(:c), match('a=!c') end def test_choice assert_rule G.any(:b, :c), match('a=b|c') end def test_choice_seq_priority assert_rule G.any([:num, :b], :c), match('a=num b|c') end def test_choice_sp m = match 'a=num "+" dig | dig' expected = G.any([:num, "+", :dig], :dig) assert_rule expected, m end def test_choice_sp2 str = <<-STR Stmt = - Expr:e EOL | ( !EOL . )* EOL STR m = match str expected = G.any( [:"-", G.t(:Expr, "e"), :EOL], [G.kleene([G.notp(:EOL), G.dot]), :EOL]) assert_rule expected, m, "Stmt" end def test_choice_with_actions str = <<-STR Stmt = - Expr:e EOL { p e } | ( !EOL . )* EOL { puts "error" } STR m = match str expected = G.any( [:"-", G.t(:Expr, "e"), :EOL, G.action(" p e ")], [G.kleene([G.notp(:EOL), G.dot]), :EOL, G.action(" puts \"error\" ")]) assert_rule expected, m, "Stmt" end def test_multiline_seq str = <<-STR Sum = Product:l ( PLUS Product:r { l += r } | MINUS Product:r { l -= r } )* { l } STR m = match str expected = G.seq( G.t(:Product, "l"), G.kleene( G.any( [:PLUS, G.t(:Product, "r"), G.action(" l += r ")], [:MINUS, G.t(:Product, "r"), G.action(" l -= r ")] )), G.action(" l ")) assert_rule expected, m, "Sum" end def test_multiline_seq2 str = <<-STR Value = NUMBER:i { i } | ID:i !ASSIGN { vars[i] } | OPEN Expr:i CLOSE { i } STR assert match(str) # HACK what is this testing? end def test_seq m = match 'a=b c' assert_rule G.seq(:b, :c), m m = match 'a=b c d' assert_rule G.seq(:b, :c, :d), m m = match 'a=b c d e f' assert_rule G.seq(:b, :c, :d, :e, :f), m end def test_tag m = match 'a=b:x' assert_rule G.t(:b, "x"), m end def test_tag_parens m = match 'a=(b c):x' assert_rule G.t([:b, :c], "x"), m end def test_tag_priority m = match 'a=d (b c):x' assert_rule G.seq(:d, G.t([:b, :c], "x")), m m = match 'a=d c*:x' assert_rule G.seq(:d, G.t(G.kleene(:c), "x")), m end def test_parens m = match 'a=(b c)' assert_rule G.seq(:b, :c), m end def test_parens_sp m = match 'a=( b c )' assert_rule G.seq(:b, :c), m end def test_parens_as_outer m = match 'a=b (c|d)' assert_rule G.seq(:b, G.any(:c, :d)), m end def test_action m = match 'a=b c { b + c }' assert_rule G.seq(:b, :c, G.action(" b + c ")), m end def test_action_nested_curly m = match 'a=b c { b + { c + d } }' assert_rule G.seq(:b, :c, G.action(" b + { c + d } ")), m end def test_actions_handle_double_quoted_strings m = match 'a=b c { b + c + "}" }' assert_rule G.seq(:b, :c, G.action(' b + c + "}" ')), m end def test_actions_handle_single_quoted_strings m = match "a=b c { b + c + '}' }" assert_rule G.seq(:b, :c, G.action(" b + c + '}' ")), m end def test_action_send m = match 'a=b c ~d' assert_rule G.seq(:b, :c, G.action("d")), m end def test_action_send_with_args m = match 'a=b c ~d(b,c)' assert_rule G.seq(:b, :c, G.action("d(b,c)")), m end def test_collect m = match 'a = < b c >' assert_rule G.collect(G.seq(:b, :c)), m end def test_bounds m = match 'a = @< b c >' assert_rule G.bounds(G.seq(:b, :c)), m end def test_comment m = match "a=b # this is a comment\n" assert_rule G.ref('b'), m end def test_comment_span m = match "a=b # this is a comment\n c" assert_rule G.seq(G.ref('b'), G.ref("c")), m end def test_parser_directive m = match <<-GRAMMAR %% header { # coding: UTF-8 } a=b GRAMMAR assert_rule G.ref("b"), m expected = { "header" => KPeg::Action.new("\n# coding: UTF-8\n") } assert_equal expected, m.directives end def test_parser_directive_duplicate m = nil out, err = capture_io do m = match <<-GRAMMAR %% header { # coding: UTF-8 } a=b %% header { # coding: ISO-8859-1 } GRAMMAR end assert_empty out assert_equal "directive \"header\" appears more than once\n", err expected = { "header" => KPeg::Action.new("\n# coding: ISO-8859-1\n") } assert_equal expected, m.directives end def test_parser_setup m = match "%% { def initialize; end }\na=b" assert_rule G.ref("b"), m assert_equal " def initialize; end ", m.setup_actions.first.action end def test_parser_name m = match "%%name = BlahParser" assert_equal "BlahParser", m.variables["name"] end def test_multiple_rules m = match "a=b\nc=d\ne=f" assert_rule G.ref("b"), m, "a" assert_rule G.ref("d"), m, "c" assert_rule G.ref("f"), m, "e" end def test_multiline_choice gram = <<-GRAM expr = num "+" num | num "-" num GRAM m = match gram expected = G.seq(:num, "+", :num) | G.seq(:num, "-", :num) assert_rule expected, m, "expr" end def test_multiline_choice_many2 gram = <<-GRAM term = term "+" fact | term "-" fact | fact fact = fact "*" num | fact "/" num | num GRAM m = match gram term = G.any([:term, "+", :fact], [:term, "-", :fact], :fact) fact = G.any([:fact, "*", :num], [:fact, "/", :num], :num) assert_equal term, m.find("term").op assert_equal fact, m.find("fact").op end def test_multiline_choice_many gram = <<-GRAM term = term "+" fact | term "-" fact fact = fact "*" num | fact "/" num GRAM m = match gram term = G.any([:term, "+", :fact], [:term, "-", :fact]) fact = G.any([:fact, "*", :num], [:fact, "/", :num]) assert_equal term, m.find("term").op assert_equal fact, m.find("fact").op end def make_parser(str, gram, debug=false) cg = KPeg::CodeGenerator.new "Test", gram, debug inst = cg.make(str) return inst end def test_allow_ends_with_comment path = File.expand_path("../inputs/comments.kpeg", __FILE__) parser = KPeg::FormatParser.new File.read(path), true assert_equal true, parser.parse end def test_roundtrip path = File.expand_path("../../lib/kpeg/format_parser.kpeg", __FILE__) parser = KPeg::FormatParser.new File.read(path) assert parser.parse, "Unable to parse" start = parser.grammar gr = KPeg::GrammarRenderer.new(start) io = StringIO.new gr.render(io) scan = make_parser io.string, start unless scan.parse puts io.string scan.show_error assert !scan.failed?, "parsing the grammar" end g2 = scan.grammar gr2 = KPeg::GrammarRenderer.new(g2) io2 = StringIO.new gr2.render(io2) unless io.string == io2.string require 'tempfile' Tempfile.open "diff" do |f1| f1 << io.string f1.close Tempfile.open "diff" do |f2| f2 << io2.string f2.close system "diff -u #{f1.path} #{f2.path}" end end end assert_equal io.string, io2.string # Go for a 3rd generation! scan2 = make_parser io2.string, g2 assert scan2.parse, "parsing the grammar" g3 = scan2.grammar unless g3.rules.empty? gr3 = KPeg::GrammarRenderer.new(g3) io3 = StringIO.new gr3.render(io3) assert_equal io2.string, io3.string # INCEPTION! 4! go for 4! scan3 = make_parser io3.string, g3 assert scan3.parse, "parsing the grammar" g4 = scan3.grammar gr4 = KPeg::GrammarRenderer.new(g4) io4 = StringIO.new gr4.render(io4) assert_equal io3.string, io4.string end end end kpeg-1.0.0/.gemtest0000644000175000017500000000000012321266363012543 0ustar domidomikpeg-1.0.0/.autotest0000644000175000017500000000033412321266363012755 0ustar domidomirequire 'autotest/restart' Autotest.add_hook :run_command do |at| system "#{Gem.ruby} -rubygems #{Gem.bin_path 'rake', 'rake'} parser" end Autotest.add_hook :initialize do |at| at.testlib = 'minitest/autorun' end kpeg-1.0.0/.travis.yml0000644000175000017500000000035012321266363013213 0ustar domidomi--- script: rake travis before_script: - gem install hoe-travis --no-rdoc --no-ri - rake travis:before -t language: ruby notifications: email: - evan@fallingsnow.net - drbrain@segment7.net rvm: - 1.8.7 - 1.9.2 - 1.9.3 - 2.0.0 kpeg-1.0.0/kpeg.gemspec0000644000175000017500000000652612321266363013410 0ustar domidomi# -*- encoding: utf-8 -*- # stub: kpeg 1.0.0.20140103162640 ruby lib Gem::Specification.new do |s| s.name = "kpeg" s.version = "1.0.0.20140103162640" s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Evan Phoenix"] s.date = "2014-01-04" s.description = "KPeg is a simple PEG library for Ruby. It provides an API as well as native\ngrammar to build the grammar.\n\nKPeg strives to provide a simple, powerful API without being too exotic.\n\nKPeg supports direct left recursion of rules via the\n{OMeta memoization}[http://www.vpri.org/pdf/tr2008003_experimenting.pdf] trick." s.email = ["evan@fallingsnow.net"] s.executables = ["kpeg"] s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.rdoc", "examples/phone_number/README.md", "examples/upper/README.md"] s.files = [".autotest", ".travis.yml", "History.txt", "LICENSE", "Manifest.txt", "README.rdoc", "Rakefile", "bin/kpeg", "examples/calculator/calculator.kpeg", "examples/calculator/calculator.rb", "examples/foreign_reference/literals.kpeg", "examples/foreign_reference/matcher.kpeg", "examples/foreign_reference/matcher.rb", "examples/lua_string/driver.rb", "examples/lua_string/lua_string.kpeg", "examples/lua_string/lua_string.kpeg.rb", "examples/phone_number/README.md", "examples/phone_number/phone_number.kpeg", "examples/phone_number/phone_number.rb", "examples/upper/README.md", "examples/upper/upper.kpeg", "examples/upper/upper.rb", "kpeg.gemspec", "lib/hoe/kpeg.rb", "lib/kpeg.rb", "lib/kpeg/code_generator.rb", "lib/kpeg/compiled_parser.rb", "lib/kpeg/format_parser.kpeg", "lib/kpeg/format_parser.rb", "lib/kpeg/grammar.rb", "lib/kpeg/grammar_renderer.rb", "lib/kpeg/match.rb", "lib/kpeg/parser.rb", "lib/kpeg/position.rb", "lib/kpeg/string_escape.kpeg", "lib/kpeg/string_escape.rb", "test/inputs/comments.kpeg", "test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_format_parser_round_trip.rb", "test/test_kpeg_grammar.rb", "test/test_kpeg_grammar_renderer.rb", "vim/syntax_kpeg/ftdetect/kpeg.vim", "vim/syntax_kpeg/syntax/kpeg.vim", "test/test_kpeg_string_escape.rb", ".gemtest"] s.homepage = "https://github.com/evanphx/kpeg" s.licenses = ["MIT"] s.rdoc_options = ["--main", "README.rdoc"] s.require_paths = ["lib"] s.rubyforge_project = "kpeg" s.rubygems_version = "2.1.10" s.summary = "KPeg is a simple PEG library for Ruby" s.test_files = ["test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_format_parser_round_trip.rb", "test/test_kpeg_grammar.rb", "test/test_kpeg_grammar_renderer.rb", "test/test_kpeg_string_escape.rb"] if s.respond_to? :specification_version then s.specification_version = 4 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then s.add_development_dependency(%q, ["~> 5.2"]) s.add_development_dependency(%q, ["~> 4.0"]) s.add_development_dependency(%q, ["~> 3.7"]) else s.add_dependency(%q, ["~> 5.2"]) s.add_dependency(%q, ["~> 4.0"]) s.add_dependency(%q, ["~> 3.7"]) end else s.add_dependency(%q, ["~> 5.2"]) s.add_dependency(%q, ["~> 4.0"]) s.add_dependency(%q, ["~> 3.7"]) end end kpeg-1.0.0/vim/0000755000175000017500000000000012321266363011677 5ustar domidomikpeg-1.0.0/vim/syntax_kpeg/0000755000175000017500000000000012321266363014233 5ustar domidomikpeg-1.0.0/vim/syntax_kpeg/syntax/0000755000175000017500000000000012321266363015561 5ustar domidomikpeg-1.0.0/vim/syntax_kpeg/syntax/kpeg.vim0000644000175000017500000000237012321266363017226 0ustar domidomi" Vim syntax file " Language: kpeg " Version: $Revision$ if version < 600 syntax clear elseif exists("b:current_syntax") finish endif syn case match " Misc syntax. syn match kpegOperator /[|*?+!\[\]]/ syn match kpegAssign "=" syn match kpegCapture /[<>]/ syn match kpegParen /[()]/ syn match kpegIdentifier /-|([a-zA-Z][-a-zA-Z0-9]*)/ syn match kpegComment /#.*$/ syn region kpegString start="\"" end="\"" skip="\\\\\|\\\"" syn region kpegRegexp start=/\// skip=/\\\// end=/\// syntax include @Ruby syntax/ruby.vim syn region kpegCode matchgroup=kpegCurly start=/{/ end=/}/ contains=@Ruby syn match kpegLabel /:[a-zA-Z][-a-zA-Z0-9]*/ if version >= 508 || !exists("did_c_syn_inits") if version < 508 let did_c_syn_inits = 1 command -nargs=+ HiLink hi link else command -nargs=+ HiLink hi def link endif HiLink kpegRegexp Special HiLink kpegNumber Number HiLink kpegComment Comment HiLink kpegString String HiLink kpegLabel Type HiLink kpegOperator Operator HiLink kpegAssign Define HiLink kpegCapture Keyword HiLink kpegFloat Float HiLink kpegIdentifier Identifier HiLink kpegParen Delimiter HiLink kpegCurly Delimiter delcommand HiLink endif let b:current_syntax = "kpeg" kpeg-1.0.0/vim/syntax_kpeg/ftdetect/0000755000175000017500000000000012321266363016035 5ustar domidomikpeg-1.0.0/vim/syntax_kpeg/ftdetect/kpeg.vim0000644000175000017500000000005712321266363017502 0ustar domidomiau BufNewFile,BufRead *.kpeg set filetype=kpeg kpeg-1.0.0/README.rdoc0000644000175000017500000001600412321266363012713 0ustar domidomi= kpeg home :: https://github.com/evanphx/kpeg bugs :: https://github.com/evanphx/kpeg/issues == Description KPeg is a simple PEG library for Ruby. It provides an API as well as native grammar to build the grammar. KPeg strives to provide a simple, powerful API without being too exotic. KPeg supports direct left recursion of rules via the {OMeta memoization}[http://www.vpri.org/pdf/tr2008003_experimenting.pdf] trick. == Writing your first grammar === Setting up your grammar All grammars start with with the class/module name that will be your parser %% name = Example::Parser After that a block of ruby code can be defined that will be added into the class body of your parser. Attributes that are defined in this block can be accessed within your parser as instance variables. Methods can also be defined in this block and used in action blocks as well. %% { attr_accessor :something_cool def something_awesome # do something awesome end } === Defining literals Literals are static declarations of characters or regular expressions designed for reuse in the grammar. These can be constants or variables. Literals can take strings, regular expressions or character ranges ALPHA = /[A-Za-z]/ DIGIT = /[0-9]/ period = "." string = "a string" regex = /(regexs?)+/ char_range = [b-t] Literals can also accept multiple definitions vowel = "a" | "e" | "i" | "o" | "u" alpha = /[A-Z]/ | /[a-z]/ === Defining Rules for Values Before you can start parsing a string you will need to define rules that you will use to accept or reject that string. There are many different types of rules available in kpeg The most basic of these rules is a string capture alpha = < /[A-Za-z]/ > { text } While this looks very much like the ALPHA literal defined above it differs in one important way, the text captured by the rule defined between the < and > symbols will be set as the text variable in block that follows. You can also explicitly define the variable that you would like but only with existing rules or literals. letter = alpha:a { a } Additionally blocks can return true or false values based upon an expression within the block. To return true if a test passes do the following: match_greater_than_10 = < num:n > &{ n > 10 } To test and return a false value if the test passes do the following: do_not_match_greater_than_10 = < num:n > !{ n > 10 } Rules can also act like functions and take parameters. An example of this is lifted from the {Email List Validator}[https://github.com/larb/email_address_validator], where an ascii value is passed in and the character is evaluated against it returning a true if it matches d(num) = <.> &{ text[0] == num } Rules support some regular expression syntax for matching * maybe ? * many + * kleene * * groupings () Examples: letters = alpha+ words = alpha+ space* period? sentence = (letters+ | space+)+ Kpeg also allows a rule to define the acceptable number of matches in the form of a range. In regular expressions this is often denoted with syntax like {0,3}. Kpeg uses this syntax to accomplish match ranges [min, max]. matches_3_to_5_times = letter[3,5] matches_3_to_any_times = letter[3,*] === Defining Actions Illustrated above in some of the examples, kpeg allows you to perform actions based upon a match that are described in block provided or in the rule definition itself. num = /[1-9][0-9]*/ sum = < num:n1 "+" num:n2 > { n1 + n2 } As of version 0.8 an alternate syntax has been added for calling defined methods as actions. %% { def add(n1, n2){ n1 + n2 } } num = /[1-9][0-9]*/ sum = < num:n1 "+" num:n2 > ~add(n1, n2) === Referencing an external grammar Kpeg allows you to run a rule that is defined in an external grammar. This is useful if there is a defined set of rules that you would like to reuse in another parser. To do this, create your grammar and generate a parser using the kpeg command line tool. kpeg literals.kpeg Once you have the generated parser, include that file into your new grammar %{ require "literals.kpeg.rb" } Then create a variable to hold to foreign interface and pass it the class name of your parser. In this case my parser class name is Literal %foreign_grammer = Literal You can then use rules defined in the foreign grammar in the local grammar file like so sentence = (%foreign_grammer.alpha %foreign_grammer.space*)+ %foreign_grammer.period === Comments Kpeg allows comments to be added to the grammar file by using the # symbol # This is a comment in my grammar === Variables A variable looks like this: %% name = value Kpeg allows the following variables that control the output parser: name:: The class name of the generated parser. custom_initialize:: When built as a standalone parser a default initialize method will not be included. === Directives A directive looks like this: %% header { ... } Kpeg allows the following directives: header:: Placed before any generated code pre-class:: Placed before the class definition to provide a class comment footer:: Placed after the end of the class (for requiring files dependent upon the parser's namespace == Generating and running your parser Before you can generate your parser you will need to define a root rule. This will be the first rule run against the string provided to the parser root = sentence To generate the parser run the kpeg command with the kpeg file(s) as an argument. This will generate a ruby file with the same name as your grammar file. kpeg example.kpeg Include your generated parser file into an application that you want to use the parser in and run it. Create a new instance of the parser and pass in the string you want to evaluate. When parse is called on the parser instance it will return a true if the sting is matched, or false if it doesn't. require "example.kpeg.rb" parser = Example::Parser.new(string_to_evaluate) parser.parse == Shortcuts and other techniques Per vito, you can get the current line or current column in the following way line = { current_line } column = { current_column } foo = line:line ... { # use line here } == AST Generation As of Kpeg 0.8 a parser can now generate an AST. To define an AST node use the following syntax %% assign = ast Assignment(name, value) Once you have a defined AST node, it can be used in your grammar like so assignment = identifier:i space* = space* value:v ~assign(i,v) This will create a new Assign node that you can add into your AST. For a good example of usage check out Talon[https://github.com/evanphx/talon] == Examples There are several examples available in the /examples directory. The upper parser has a readme with a step by step description of the grammar. == Projects using kpeg Dang[https://github.com/veganstraightedge/dang] {Email Address Validator}[https://github.com/larb/email_address_validator] Callisto[https://github.com/dwaite/Callisto] Doodle[https://github.com/vito/doodle] Kanbanpad[https://kanbanpad.com] (uses kpeg for parsing of the 'enter something' bar) kpeg-1.0.0/examples/0000755000175000017500000000000012321266363012722 5ustar domidomikpeg-1.0.0/examples/lua_string/0000755000175000017500000000000012321266363015071 5ustar domidomikpeg-1.0.0/examples/lua_string/driver.rb0000644000175000017500000000042612321266363016713 0ustar domidomirequire 'lua_string.kpeg.rb' ls = LuaString.new("[[blah]]") ls.parse p ls.result ls = LuaString.new("[==[blah2]==]") ls.parse p ls.result ls = LuaString.new("[==[embeded]stuff]==]") ls.parse p ls.result ls = LuaString.new("[==[embeded]=]stuff]==]") ls.parse p ls.result kpeg-1.0.0/examples/lua_string/lua_string.kpeg.rb0000644000175000017500000002264712321266363020525 0ustar domidomiclass LuaString # STANDALONE START def setup_parser(str, debug=false) @string = str @pos = 0 @memoizations = Hash.new { |h,k| h[k] = {} } @result = nil @failed_rule = nil @failing_rule_offset = -1 setup_foreign_grammar end # This is distinct from setup_parser so that a standalone parser # can redefine #initialize and still have access to the proper # parser setup code. # def initialize(str, debug=false) setup_parser(str, debug) end attr_reader :string attr_reader :result, :failing_rule_offset attr_accessor :pos # STANDALONE START def current_column(target=pos) if c = string.rindex("\n", target-1) return target - c - 1 end target + 1 end def current_line(target=pos) cur_offset = 0 cur_line = 0 string.each_line do |line| cur_line += 1 cur_offset += line.size return cur_line if cur_offset >= target end -1 end def lines lines = [] string.each_line { |l| lines << l } lines end # def get_text(start) @string[start..@pos-1] end def show_pos width = 10 if @pos < width "#{@pos} (\"#{@string[0,@pos]}\" @ \"#{@string[@pos,width]}\")" else "#{@pos} (\"... #{@string[@pos - width, width]}\" @ \"#{@string[@pos,width]}\")" end end def failure_info l = current_line @failing_rule_offset c = current_column @failing_rule_offset if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "line #{l}, column #{c}: failed rule '#{info.name}' = '#{info.rendered}'" else "line #{l}, column #{c}: failed rule '#{@failed_rule}'" end end def failure_caret l = current_line @failing_rule_offset c = current_column @failing_rule_offset line = lines[l-1] "#{line}\n#{' ' * (c - 1)}^" end def failure_character l = current_line @failing_rule_offset c = current_column @failing_rule_offset lines[l-1][c-1, 1] end def failure_oneline l = current_line @failing_rule_offset c = current_column @failing_rule_offset char = lines[l-1][c-1, 1] if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] "@#{l}:#{c} failed rule '#{info.name}', got '#{char}'" else "@#{l}:#{c} failed rule '#{@failed_rule}', got '#{char}'" end end class ParseError < RuntimeError end def raise_error raise ParseError, failure_oneline end def show_error(io=STDOUT) error_pos = @failing_rule_offset line_no = current_line(error_pos) col_no = current_column(error_pos) io.puts "On line #{line_no}, column #{col_no}:" if @failed_rule.kind_of? Symbol info = self.class::Rules[@failed_rule] io.puts "Failed to match '#{info.rendered}' (rule '#{info.name}')" else io.puts "Failed to match rule '#{@failed_rule}'" end io.puts "Got: #{string[error_pos,1].inspect}" line = lines[line_no-1] io.puts "=> #{line}" io.print(" " * (col_no + 3)) io.puts "^" end def set_failed_rule(name) if @pos > @failing_rule_offset @failed_rule = name @failing_rule_offset = @pos end end attr_reader :failed_rule def match_string(str) len = str.size if @string[pos,len] == str @pos += len return str end return nil end def scan(reg) if m = reg.match(@string[@pos..-1]) width = m.end(0) @pos += width return true end return nil end if "".respond_to? :getbyte def get_byte if @pos >= @string.size return nil end s = @string.getbyte @pos @pos += 1 s end else def get_byte if @pos >= @string.size return nil end s = @string[@pos] @pos += 1 s end end def parse(rule=nil) if !rule _root ? true : false else # This is not shared with code_generator.rb so this can be standalone method = rule.gsub("-","_hyphen_") __send__("_#{method}") ? true : false end end class LeftRecursive def initialize(detected=false) @detected = detected end attr_accessor :detected end class MemoEntry def initialize(ans, pos) @ans = ans @pos = pos @uses = 1 @result = nil end attr_reader :ans, :pos, :uses, :result def inc! @uses += 1 end def move!(ans, pos, result) @ans = ans @pos = pos @result = result end end def external_invoke(other, rule, *args) old_pos = @pos old_string = @string @pos = other.pos @string = other.string begin if val = __send__(rule, *args) other.pos = @pos else other.set_failed_rule "#{self.class}##{rule}" end val ensure @pos = old_pos @string = old_string end end def apply(rule) if m = @memoizations[rule][@pos] m.inc! prev = @pos @pos = m.pos if m.ans.kind_of? LeftRecursive m.ans.detected = true return nil end @result = m.result return m.ans else lr = LeftRecursive.new(false) m = MemoEntry.new(lr, @pos) @memoizations[rule][@pos] = m start_pos = @pos ans = __send__ rule m.move! ans, @pos, @result # Don't bother trying to grow the left recursion # if it's failing straight away (thus there is no seed) if ans and lr.detected return grow_lr(rule, start_pos, m) else return ans end return ans end end def grow_lr(rule, start_pos, m) while true @pos = start_pos @result = m.result ans = __send__ rule return nil unless ans break if @pos <= m.pos m.move! ans, @pos, @result end @result = m.result @pos = m.pos return m.ans end class RuleInfo def initialize(name, rendered) @name = name @rendered = rendered end attr_reader :name, :rendered end def self.rule_info(name, rendered) RuleInfo.new(name, rendered) end # attr_accessor :result def setup_foreign_grammar; end # equals = < "="* > { text } def _equals _save = self.pos while true # sequence _text_start = self.pos while true _tmp = match_string("=") break unless _tmp end _tmp = true if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end @result = begin; text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_equals unless _tmp return _tmp end # equal_ending = "]" equals:x &{ x == start } "]" def _equal_ending(start) _save = self.pos while true # sequence _tmp = match_string("]") unless _tmp self.pos = _save break end _tmp = apply(:_equals) x = @result unless _tmp self.pos = _save break end _save1 = self.pos _tmp = begin; x == start ; end self.pos = _save1 unless _tmp self.pos = _save break end _tmp = match_string("]") unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_equal_ending unless _tmp return _tmp end # root = "[" equals:e "[" < (!equal_ending(e) .)* > equal_ending(e) { @result = text } def _root _save = self.pos while true # sequence _tmp = match_string("[") unless _tmp self.pos = _save break end _tmp = apply(:_equals) e = @result unless _tmp self.pos = _save break end _tmp = match_string("[") unless _tmp self.pos = _save break end _text_start = self.pos while true _save2 = self.pos while true # sequence _save3 = self.pos _tmp = _equal_ending(e) _tmp = _tmp ? nil : true self.pos = _save3 unless _tmp self.pos = _save2 break end _tmp = get_byte unless _tmp self.pos = _save2 end break end # end sequence break unless _tmp end _tmp = true if _tmp text = get_text(_text_start) end unless _tmp self.pos = _save break end _tmp = _equal_ending(e) unless _tmp self.pos = _save break end @result = begin; @result = text ; end _tmp = true unless _tmp self.pos = _save end break end # end sequence set_failed_rule :_root unless _tmp return _tmp end Rules = {} Rules[:_equals] = rule_info("equals", "< \"=\"* > { text }") Rules[:_equal_ending] = rule_info("equal_ending", "\"]\" equals:x &{ x == start } \"]\"") Rules[:_root] = rule_info("root", "\"[\" equals:e \"[\" < (!equal_ending(e) .)* > equal_ending(e) { @result = text }") end kpeg-1.0.0/examples/lua_string/lua_string.kpeg0000644000175000017500000000035712321266363020115 0ustar domidomi%% name = LuaString %% { attr_accessor :result } equals = < "="* > { text } equal_ending(start) = "]" equals:x &{ x == start } "]" root = "[" equals:e "[" < (!equal_ending(e) .)* > equal_ending(e) { @result = text } kpeg-1.0.0/examples/upper/0000755000175000017500000000000012321266363014055 5ustar domidomikpeg-1.0.0/examples/upper/upper.kpeg0000644000175000017500000000071512321266363016063 0ustar domidomi%% name = Upper %% { attr_accessor :output } period = "." space = " " alpha = < /[A-Za-z]/ > { text.upcase } word = alpha:a word:w { "#{a}#{w}" } | alpha:a space { "#{a} "} | alpha:a { a } sentence = word:w sentence:s { "#{w}#{s}" } | word:w { w } document = sentence:s period space* document:d { "#{s}. #{d}" } | sentence:s period { "#{s}." } | sentence:s { "#{s}" } root = document:d { @output = d } kpeg-1.0.0/examples/upper/README.md0000644000175000017500000000550312321266363015337 0ustar domidomi# Upper Parser A parser that matches a string with alpha characters, spaces and a period and returns the string in upper case. ## Grammar Name of the class that will be used to do the parsing %% name = Upper A variable that I want to store the converted text for accessing later %% { attr_accessor :output } My literals period = "." space = " " A rule that states that all characters that match the regex [A-Za-z] should be returned uppercase alpha = < /[A-Za-z]/ > { text.upcase } My rules that defines a word, it consists of three different cases, first that a word is an alpha followed by another word. If this matches return the alpha and the word that follows. word = alpha:a word:w { "#{a}#{w}" } This rule states that a word can be an alpha followed by a space. If this matches return an alpha followed by a space | alpha:a space+ { "#{a} "} This rule states that a word can consist of just an alpha. If this matches just return the alpha | alpha:a { a } My rules that defines a sentence. The first states that a sentence consists of a word followed by a sentence. If this matches return the word followed by the sentence. sentence = < word:w sentence:s > { "#{w}#{s}" } This rule states that a sentence can just be a word. If this matches just return the word. | word:w { w } My rules that define a document. The first rule states that a document can be a sentence followed by a period that may have space followed by another document. If this matches return the sentence followed by a period with a space followed by the document. document = sentence:s period space* document:d { "#{s}. #{d}" } This rule states that a document can be a sentence followed by a period. If this matches return the sentence followed by a period. | sentence:s period { "#{s}." } This rule states that a document can just be a sentence. If this matches just return the sentence. | sentence:s { s } The root node it the first rule evaluated, is it essentially the starting point for your grammar. If the string provided can successfully be matched by the grammar provided store the returned document in the @output variable. root = document:d { @output = d } ## Generate the parser To generate the parser make sure you have kpeg installed and run the following command (you may have to remove upper.kpeg.rb if it was previously generated) kpeg upper.kpeg ## Run the parser To run the parser run the following ruby upper.rb ## Accepted Strings + a lower case string. Another lower case string. + A LOWER CASE STRING. ANOTHER LOWER CASE STRING. + a string with lots of spaces. ## Not accepted strings (there are tons) Anything that doesn't stick to spaces and periods, very brittle but it is a simple examplekpeg-1.0.0/examples/upper/upper.rb0000644000175000017500000000043512321266363015537 0ustar domidomi# Make sure you have the kpeg gem installed require 'rubygems' # To generate the upper.kpeg file run kpeg upper.kpeg require "./upper.kpeg.rb" # Require the generated parser parser = Upper.new("a lower case string. Another lower case string.") if parser.parse puts parser.output endkpeg-1.0.0/examples/phone_number/0000755000175000017500000000000012321266363015403 5ustar domidomikpeg-1.0.0/examples/phone_number/phone_number.rb0000644000175000017500000000020712321266363020410 0ustar domidomirequire 'rubygems' require "./phone_number.kpeg.rb" parser = PhoneNumber.new("8888888888") puts parser.parse puts parser.phone_number kpeg-1.0.0/examples/phone_number/README.md0000644000175000017500000000034212321266363016661 0ustar domidomi# Phone Number Parser A string is parsed to determine if it is a valid phone number. A phone number in the following format (888) 555-0000 will be available using the resolved_number method if the string successfully matches kpeg-1.0.0/examples/phone_number/phone_number.kpeg0000644000175000017500000000074112321266363020736 0ustar domidomi%% name = PhoneNumber %% { attr_accessor :phone_number } digit = [0-9] space = " " dash = "-" LP = "(" RP = ")" country_code = < digit > { text } area_code = < digit[3] > { text } prefix = < digit[3] > { text } suffix = < digit[4] > { text } phone_number = LP? area_code:ac RP? space* prefix:p space* dash? space* suffix:s space* { "(#{ac}) #{p}-#{s}" } root = phone_number:pn { @phone_number = pn } | country_code:c space* phone_number:pn { @phone_number = "+#{c} #{pn}" }kpeg-1.0.0/examples/foreign_reference/0000755000175000017500000000000012321266363016371 5ustar domidomikpeg-1.0.0/examples/foreign_reference/matcher.rb0000644000175000017500000000015312321266363020340 0ustar domidomirequire 'rubygems' require './matcher.kpeg.rb' parser = Matcher.new("this is a string.") puts parser.parsekpeg-1.0.0/examples/foreign_reference/matcher.kpeg0000644000175000017500000000020712321266363020663 0ustar domidomi%% name = Matcher %% { require "literals.kpeg.rb" } %grammer1 = Literal root = (%grammer1.alpha %grammer1.space*)+ %grammer1.periodkpeg-1.0.0/examples/foreign_reference/literals.kpeg0000644000175000017500000000007612321266363021063 0ustar domidomi%% name = Literal period = "." space = " " alpha = /[A-Za-z]/kpeg-1.0.0/examples/calculator/0000755000175000017500000000000012321266363015053 5ustar domidomikpeg-1.0.0/examples/calculator/calculator.rb0000644000175000017500000000017512321266363017534 0ustar domidomirequire 'rubygems' require "./calculator.kpeg" parser = Calculator.new("1 + 2 * 3") if parser.parse puts parser.result endkpeg-1.0.0/examples/calculator/calculator.kpeg0000644000175000017500000000052512321266363020056 0ustar domidomi%% name = Calculator %% { attr_accessor :result } space = " " - = space* num = < /[1-9][0-9]*/ > { text.to_i } term = term:t1 - "+" - term:t2 { t1 + t2 } | term:t1 - "-" - termLt2 { t1 - t2 } | fact fact = fact:f1 - "*" - fact:f2 { f1 * f2 } | fact:f1 - "/" - fact:f2 { f1 / f2 } | num root = term:t { @result = t } kpeg-1.0.0/History.txt0000644000175000017500000000160312321266363013306 0ustar domidomi=== 0.10 / 2012-04-16 * Minor enhancements * In standalone parsers generation of a default initialize method may be disabled with the custom_initialize variable: %% custom_initialize = true * Added a pre-class directive for adding class comments * Generated code is now surrounded by startdoc/stopdoc. * Bug fixes * Hoe plugin now overwrites generated files * Directives and variables now round-trip through KPeg::GrammarRenderer === 0.9 / 2012-04-06 * Minor enhancements * Added arbitrary directives to the kpeg grammar %% directive_name { ... } * Added header and footer directives to the kpeg code formatter. These appear above and below all other output, respectively: %% header { # coding: UTF-8 } [... your grammar ...] %% footer { require 'some/subclass' } * Switched to minitest * Switched to hoe kpeg-1.0.0/LICENSE0000644000175000017500000000273412321266363012117 0ustar domidomiCopyright (c) 2011, Evan Phoenix All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.