pax_global_header00006660000000000000000000000064141763174070014524gustar00rootroot0000000000000052 comment=cda4adb064262c661b230de737d01729f1608074 liquid-c-4.1.0/000077500000000000000000000000001417631740700132355ustar00rootroot00000000000000liquid-c-4.1.0/.github/000077500000000000000000000000001417631740700145755ustar00rootroot00000000000000liquid-c-4.1.0/.github/workflows/000077500000000000000000000000001417631740700166325ustar00rootroot00000000000000liquid-c-4.1.0/.github/workflows/liquid.yml000066400000000000000000000026551417631740700206540ustar00rootroot00000000000000name: Liquid on: [push, pull_request] jobs: test: runs-on: ubuntu-latest strategy: matrix: entry: - { ruby: '2.5', allowed-failure: false } - { ruby: '2.7', allowed-failure: false } - { ruby: '3.0', allowed-failure: false } - { ruby: ruby-head, allowed-failure: true } name: test (${{ matrix.entry.ruby }}) steps: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.entry.ruby }} - uses: actions/cache@v1 with: path: vendor/bundle key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile') }} restore-keys: ${{ runner.os }}-gems- - run: bundle install --jobs=3 --retry=3 --path=vendor/bundle - run: bundle exec rake continue-on-error: ${{ matrix.entry.allowed-failure }} env: LIQUID_C_PEDANTIC: 'true' - run: bundle exec rubocop valgrind: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: ruby-version: '3.0' - run: sudo apt-get install -y valgrind - uses: actions/cache@v1 with: path: vendor/bundle key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile') }} restore-keys: ${{ runner.os }}-gems- - run: bundle install --jobs=3 --retry=3 --path=vendor/bundle - run: bundle exec rake test:valgrind liquid-c-4.1.0/.gitignore000066400000000000000000000001531417631740700152240ustar00rootroot00000000000000*.gem *.rbc Gemfile.lock pkg tmp *.o *.bundle ext/*/Makefile *.so instruments*.trace *.cpu *.object *.dSYM liquid-c-4.1.0/.rubocop.yml000077500000000000000000000003421417631740700155110ustar00rootroot00000000000000inherit_gem: rubocop-shopify: rubocop.yml require: rubocop-performance AllCops: TargetRubyVersion: 2.5 Exclude: - 'vendor/bundle/**/*' - 'tmp/**/*' Style/GlobalVars: Exclude: - 'ext/liquid_c/extconf.rb' liquid-c-4.1.0/Gemfile000077500000000000000000000007451417631740700145410ustar00rootroot00000000000000# frozen_string_literal: true source "https://rubygems.org" git_source(:github) do |repo_name| "https://github.com/#{repo_name}.git" end gemspec gem "liquid", github: "Shopify/liquid", ref: "master" group :test do gem "rubocop", "~> 1.24.1", require: false gem "rubocop-performance", "~> 1.13.2", require: false gem "rubocop-shopify", "~> 2.4.0", require: false gem "spy", "0.4.1" gem "benchmark-ips" gem "ruby_memcheck" end group :development do gem "byebug" end liquid-c-4.1.0/LICENSE.txt000066400000000000000000000020501417631740700150550ustar00rootroot00000000000000Copyright (c) 2014 Shopify MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. liquid-c-4.1.0/README.md000066400000000000000000000046131417631740700145200ustar00rootroot00000000000000# Liquid::C [![Build Status](https://travis-ci.org/Shopify/liquid-c.svg?branch=master)](https://travis-ci.org/Shopify/liquid-c) Partial native implementation of the liquid ruby gem in C. ## Installation Add these lines to your application's Gemfile: gem 'liquid', github: 'Shopify/liquid', branch: 'master' gem 'liquid-c', github: 'Shopify/liquid-c', branch: 'master' And then execute: $ bundle ## Usage require 'liquid/c' then just use the documented API for the liquid Gem. ## Restrictions * Input strings are assumed to be UTF-8 encoded strings * Tag#parse(tokens) is given a Liquid::Tokenizer object, instead of an array of strings, which only implements the shift method to get the next token. ## Performance To compare Liquid-C's performance with plain Liquid run bundle exec rake compare:lax The latest benchmark results are shown below: ``` $ bundle exec rake compare:lax /home/spin/.rubies/ruby-3.0.2/bin/ruby ./performance.rb bare benchmark lax Running benchmark for 10 seconds (with 5 seconds warmup). Warming up -------------------------------------- parse: 2.000 i/100ms render: 8.000 i/100ms parse & render: 2.000 i/100ms Calculating ------------------------------------- parse: 29.527 (± 3.4%) i/s - 296.000 in 10.034520s render: 89.403 (± 6.7%) i/s - 896.000 in 10.072939s parse & render: 20.474 (± 4.9%) i/s - 206.000 in 10.072806s /home/spin/.rubies/ruby-3.0.2/bin/ruby ./performance.rb c benchmark lax Running benchmark for 10 seconds (with 5 seconds warmup). Warming up -------------------------------------- parse: 10.000 i/100ms render: 18.000 i/100ms parse & render: 5.000 i/100ms Calculating ------------------------------------- parse: 90.672 (± 3.3%) i/s - 910.000 in 10.051124s render: 163.871 (± 4.9%) i/s - 1.638k in 10.018105s parse & render: 50.165 (± 4.0%) i/s - 505.000 in 10.077377s ``` ## Developing bundle install # run tests bundle exec rake ## Contributing 1. Fork it ( http://github.com/Shopify/liquid-c/fork ) 2. Create your feature branch (`git checkout -b my-new-feature`) 3. Commit your changes (`git commit -am 'Add some feature'`) 4. Push to the branch (`git push origin my-new-feature`) 5. Create new Pull Request liquid-c-4.1.0/Rakefile000066400000000000000000000006461417631740700147100ustar00rootroot00000000000000# frozen_string_literal: true require "rake" require "rake/testtask" require "bundler/gem_tasks" require "rake/extensiontask" require "benchmark" require "ruby_memcheck" ENV["DEBUG"] ||= "true" RubyMemcheck.config(binary_name: "liquid_c") task default: [:test, :rubocop] task test: ["test:unit", "test:integration:all"] namespace :test do task valgrind: ["test:unit:valgrind", "test:integration:valgrind:all"] end liquid-c-4.1.0/ext/000077500000000000000000000000001417631740700140355ustar00rootroot00000000000000liquid-c-4.1.0/ext/liquid_c/000077500000000000000000000000001417631740700156265ustar00rootroot00000000000000liquid-c-4.1.0/ext/liquid_c/block.c000066400000000000000000000476761417631740700171100ustar00rootroot00000000000000#include "liquid.h" #include "block.h" #include "intutil.h" #include "tokenizer.h" #include "stringutil.h" #include "vm.h" #include "variable.h" #include "context.h" #include "parse_context.h" #include "vm_assembler.h" #include static ID intern_raise_missing_variable_terminator, intern_raise_missing_tag_terminator, intern_is_blank, intern_parse, intern_square_brackets, intern_unknown_tag_in_liquid_tag, intern_ivar_nodelist; static VALUE tag_registry; static VALUE variable_placeholder = Qnil; typedef struct tag_markup { VALUE name; VALUE markup; } tag_markup_t; typedef struct parse_context { tokenizer_t *tokenizer; VALUE tokenizer_obj; VALUE ruby_obj; } parse_context_t; static void ensure_body_compiled(const block_body_t *body) { if (!body->compiled) { rb_raise(rb_eRuntimeError, "Liquid::C::BlockBody has not been compiled"); } } static void block_body_mark(void *ptr) { block_body_t *body = ptr; if (body->compiled) { document_body_entry_mark(&body->as.compiled.document_body_entry); rb_gc_mark(body->as.compiled.nodelist); } else { rb_gc_mark(body->as.intermediate.parse_context); if (body->as.intermediate.vm_assembler_pool) vm_assembler_pool_gc_mark(body->as.intermediate.vm_assembler_pool); if (body->as.intermediate.code) vm_assembler_gc_mark(body->as.intermediate.code); } } static void block_body_free(void *ptr) { block_body_t *body = ptr; if (!body->compiled && body->as.intermediate.code) { // Free the assembler instead of recycling it because the vm_assembler_pool may have been GC'd vm_assembler_pool_free_assembler(body->as.intermediate.code); } xfree(body); } static size_t block_body_memsize(const void *ptr) { const block_body_t *body = ptr; if (!ptr) return 0; if (body->compiled) { return sizeof(block_body_t); } else { return sizeof(block_body_t) + vm_assembler_alloc_memsize(body->as.intermediate.code); } } const rb_data_type_t block_body_data_type = { "liquid_block_body", { block_body_mark, block_body_free, block_body_memsize, }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; #define BlockBody_Get_Struct(obj, sval) TypedData_Get_Struct(obj, block_body_t, &block_body_data_type, sval) static VALUE block_body_allocate(VALUE klass) { block_body_t *body; VALUE obj = TypedData_Make_Struct(klass, block_body_t, &block_body_data_type, body); body->compiled = false; body->obj = obj; body->as.intermediate.blank = true; body->as.intermediate.render_score = 0; body->as.intermediate.vm_assembler_pool = NULL; body->as.intermediate.code = NULL; return obj; } static VALUE block_body_initialize(VALUE self, VALUE parse_context) { block_body_t *body; BlockBody_Get_Struct(self, body); body->as.intermediate.parse_context = parse_context; body->as.intermediate.vm_assembler_pool = parse_context_get_vm_assembler_pool(parse_context); body->as.intermediate.code = vm_assembler_pool_alloc_assembler(body->as.intermediate.vm_assembler_pool); vm_assembler_add_leave(body->as.intermediate.code); return Qnil; } static int is_id(int c) { return rb_isalnum(c) || c == '_'; } static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_t *parse_context) { tokenizer_t *tokenizer = parse_context->tokenizer; token_t token; tag_markup_t unknown_tag = { Qnil, Qnil }; int render_score_increment = 0; while (true) { int token_start_line_number = tokenizer->line_number; if (token_start_line_number != 0) { rb_ivar_set(parse_context->ruby_obj, id_ivar_line_number, UINT2NUM(token_start_line_number)); } tokenizer_next(tokenizer, &token); switch (token.type) { case TOKENIZER_TOKEN_NONE: goto loop_break; case TOKEN_INVALID: { VALUE str = rb_enc_str_new(token.str_full, token.len_full, utf8_encoding); ID raise_method_id = intern_raise_missing_variable_terminator; if (token.str_full[1] == '%') raise_method_id = intern_raise_missing_tag_terminator; rb_funcall(cLiquidBlockBody, raise_method_id, 2, str, parse_context->ruby_obj); goto loop_break; } case TOKEN_RAW: { const char *start = token.str_full, *end = token.str_full + token.len_full; const char *token_start = start, *token_end = end; if (token.lstrip) token_start = read_while(start, end, rb_isspace); if (token.rstrip) { if (tokenizer->bug_compatible_whitespace_trimming) { token_end = read_while_reverse(token_start + 1, end, rb_isspace); } else { token_end = read_while_reverse(token_start, end, rb_isspace); } } // Skip token entirely if there is no data to be rendered. if (token_start == token_end) break; vm_assembler_add_write_raw(body->as.intermediate.code, token_start, token_end - token_start); render_score_increment += 1; if (body->as.intermediate.blank) { const char *end = token.str_full + token.len_full; if (read_while(token.str_full, end, rb_isspace) < end) body->as.intermediate.blank = false; } break; } case TOKEN_VARIABLE: { variable_parse_args_t parse_args = { .markup = token.str_trimmed, .markup_end = token.str_trimmed + token.len_trimmed, .code = body->as.intermediate.code, .code_obj = body->obj, .parse_context = parse_context->ruby_obj, }; internal_variable_compile(&parse_args, token_start_line_number); render_score_increment += 1; body->as.intermediate.blank = false; break; } case TOKEN_TAG: { const char *start = token.str_trimmed, *end = token.str_trimmed + token.len_trimmed; // Imitate \s*(\w+)\s*(.*)? regex const char *name_start = read_while(start, end, rb_isspace); const char *name_end = read_while(name_start, end, is_id); long name_len = name_end - name_start; if (name_len == 0) { VALUE str = rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding); unknown_tag = (tag_markup_t) { str, str }; goto loop_break; } if (name_len == 6 && strncmp(name_start, "liquid", 6) == 0) { const char *markup_start = read_while(name_end, end, rb_isspace); int line_number = token_start_line_number; if (line_number) { line_number += count_newlines(token.str_full, markup_start); } tokenizer_t saved_tokenizer = *tokenizer; tokenizer_setup_for_liquid_tag(tokenizer, markup_start, end, line_number); unknown_tag = internal_block_body_parse(body, parse_context); *tokenizer = saved_tokenizer; if (unknown_tag.name != Qnil) { rb_funcall(cLiquidBlockBody, intern_unknown_tag_in_liquid_tag, 2, unknown_tag.name, parse_context->ruby_obj); goto loop_break; } break; } VALUE tag_name = rb_enc_str_new(name_start, name_end - name_start, utf8_encoding); VALUE tag_class = rb_funcall(tag_registry, intern_square_brackets, 1, tag_name); const char *markup_start = read_while(name_end, end, rb_isspace); VALUE markup = rb_enc_str_new(markup_start, end - markup_start, utf8_encoding); if (tag_class == Qnil) { unknown_tag = (tag_markup_t) { tag_name, markup }; goto loop_break; } VALUE new_tag = rb_funcall(tag_class, intern_parse, 4, tag_name, markup, parse_context->tokenizer_obj, parse_context->ruby_obj); if (body->as.intermediate.blank && !RTEST(rb_funcall(new_tag, intern_is_blank, 0))) body->as.intermediate.blank = false; if (tokenizer->raw_tag_body) { if (tokenizer->raw_tag_body_len) { vm_assembler_add_write_raw(body->as.intermediate.code, tokenizer->raw_tag_body, tokenizer->raw_tag_body_len); } tokenizer->raw_tag_body = NULL; tokenizer->raw_tag_body_len = 0; } else { vm_assembler_add_write_node(body->as.intermediate.code, new_tag); } render_score_increment += 1; break; } case TOKEN_BLANK_LIQUID_TAG_LINE: break; } } loop_break: body->as.intermediate.render_score += render_score_increment; return unknown_tag; } static void ensure_intermediate(block_body_t *body) { if (body->compiled) { rb_raise(rb_eRuntimeError, "Liquid::C::BlockBody is already compiled"); } } static void ensure_intermediate_not_parsing(block_body_t *body) { ensure_intermediate(body); if (body->as.intermediate.code->parsing) { rb_raise(rb_eRuntimeError, "Liquid::C::BlockBody is in a incompletely parsed state"); } } static VALUE block_body_parse(VALUE self, VALUE tokenizer_obj, VALUE parse_context_obj) { parse_context_t parse_context = { .tokenizer_obj = tokenizer_obj, .ruby_obj = parse_context_obj, }; Tokenizer_Get_Struct(tokenizer_obj, parse_context.tokenizer); block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate_not_parsing(body); if (body->as.intermediate.parse_context != parse_context_obj) { rb_raise(rb_eArgError, "Liquid::C::BlockBody#parse called with different parse context"); } vm_assembler_remove_leave(body->as.intermediate.code); // to extend block tag_markup_t unknown_tag = internal_block_body_parse(body, &parse_context); vm_assembler_add_leave(body->as.intermediate.code); return rb_yield_values(2, unknown_tag.name, unknown_tag.markup); } static VALUE block_body_freeze(VALUE self) { block_body_t *body; BlockBody_Get_Struct(self, body); if (body->compiled) return Qnil; VALUE parse_context = body->as.intermediate.parse_context; VALUE document_body = parse_context_get_document_body(parse_context); rb_check_frozen(document_body); vm_assembler_pool_t *assembler_pool = body->as.intermediate.vm_assembler_pool; vm_assembler_t *assembler = body->as.intermediate.code; bool blank = body->as.intermediate.blank; uint32_t render_score = body->as.intermediate.render_score; vm_assembler_t *code = body->as.intermediate.code; body->as.compiled.document_body_entry = document_body_write_block_body(document_body, blank, render_score, code); body->as.compiled.nodelist = Qundef; body->compiled = true; vm_assembler_pool_recycle_assembler(assembler_pool, assembler); rb_call_super(0, NULL); return Qnil; } static VALUE block_body_render_to_output_buffer(VALUE self, VALUE context, VALUE output) { Check_Type(output, T_STRING); check_utf8_encoding(output, "output"); block_body_t *body; BlockBody_Get_Struct(self, body); ensure_body_compiled(body); document_body_entry_t *entry = &body->as.compiled.document_body_entry; document_body_ensure_compile_finished(entry->body); liquid_vm_render(document_body_get_block_body_header_ptr(entry), document_body_get_constants_ptr(entry), context, output); return output; } static VALUE block_body_blank_p(VALUE self) { block_body_t *body; BlockBody_Get_Struct(self, body); if (body->compiled) { block_body_header_t *body_header = document_body_get_block_body_header_ptr(&body->as.compiled.document_body_entry); return BLOCK_BODY_HEADER_BLANK_P(body_header) ? Qtrue : Qfalse; } else { return body->as.intermediate.blank ? Qtrue : Qfalse; } } static VALUE block_body_remove_blank_strings(VALUE self) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate_not_parsing(body); if (!body->as.intermediate.blank) { rb_raise(rb_eRuntimeError, "remove_blank_strings only support being called on a blank block body"); } uint8_t *ip = body->as.intermediate.code->instructions.data; while (*ip != OP_LEAVE) { if (*ip == OP_WRITE_RAW) { if (ip[1]) { // if (size != 0) ip[0] = OP_JUMP_FWD; // effectively a no-op body->as.intermediate.render_score--; } } else if (*ip == OP_WRITE_RAW_W) { if (ip[1] || ip[2] || ip[3]) { // if (size != 0) ip[0] = OP_JUMP_FWD_W; // effectively a no-op body->as.intermediate.render_score--; } } liquid_vm_next_instruction((const uint8_t **)&ip); } return Qnil; } static void memoize_variable_placeholder(void) { if (variable_placeholder == Qnil) { VALUE cLiquidCVariablePlaceholder = rb_const_get(mLiquidC, rb_intern("VariablePlaceholder")); variable_placeholder = rb_class_new_instance(0, NULL, cLiquidCVariablePlaceholder); } } // Deprecated: avoid using this for the love of performance static VALUE block_body_nodelist(VALUE self) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_body_compiled(body); document_body_entry_t *entry = &body->as.compiled.document_body_entry; block_body_header_t *body_header = document_body_get_block_body_header_ptr(entry); memoize_variable_placeholder(); if (body->as.compiled.nodelist != Qundef) return body->as.compiled.nodelist; VALUE nodelist = rb_ary_new_capa(body_header->render_score); const VALUE *constants = &entry->body->constants; const uint8_t *ip = block_body_instructions_ptr(body_header); while (true) { switch (*ip) { case OP_LEAVE: goto loop_break; case OP_WRITE_RAW_W: case OP_WRITE_RAW: { const char *text; size_t size; if (*ip == OP_WRITE_RAW_W) { size = bytes_to_uint24(&ip[1]); text = (const char *)&ip[4]; } else { size = ip[1]; text = (const char *)&ip[2]; } VALUE string = rb_enc_str_new(text, size, utf8_encoding); rb_ary_push(nodelist, string); break; } case OP_WRITE_NODE: { uint16_t constant_index = (ip[1] << 8) | ip[2]; VALUE node = RARRAY_AREF(*constants, constant_index); rb_ary_push(nodelist, node); break; } case OP_RENDER_VARIABLE_RESCUE: rb_ary_push(nodelist, variable_placeholder); break; } liquid_vm_next_instruction(&ip); } loop_break: rb_ary_freeze(nodelist); body->as.compiled.nodelist = nodelist; return nodelist; } static VALUE block_body_disassemble(VALUE self) { block_body_t *body; BlockBody_Get_Struct(self, body); document_body_entry_t *entry = &body->as.compiled.document_body_entry; block_body_header_t *header = document_body_get_block_body_header_ptr(entry); const uint8_t *start_ip = block_body_instructions_ptr(header); return vm_assembler_disassemble( start_ip, start_ip + header->instructions_bytes, &entry->body->constants ); } static VALUE block_body_add_evaluate_expression(VALUE self, VALUE expression) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_evaluate_expression_from_ruby(body->as.intermediate.code, self, expression); return self; } static VALUE block_body_add_find_variable(VALUE self, VALUE expression) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_find_variable_from_ruby(body->as.intermediate.code, self, expression); return self; } static VALUE block_body_add_lookup_command(VALUE self, VALUE name) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_lookup_command_from_ruby(body->as.intermediate.code, name); return self; } static VALUE block_body_add_lookup_key(VALUE self, VALUE expression) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_lookup_key_from_ruby(body->as.intermediate.code, self, expression); return self; } static VALUE block_body_add_new_int_range(VALUE self) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_new_int_range_from_ruby(body->as.intermediate.code); return self; } static VALUE block_body_add_hash_new(VALUE self, VALUE hash_size) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_hash_new_from_ruby(body->as.intermediate.code, hash_size); return self; } static VALUE block_body_add_filter(VALUE self, VALUE filter_name, VALUE num_args) { block_body_t *body; BlockBody_Get_Struct(self, body); ensure_intermediate(body); vm_assembler_add_filter_from_ruby(body->as.intermediate.code, filter_name, num_args); return self; } void liquid_define_block_body(void) { intern_raise_missing_variable_terminator = rb_intern("raise_missing_variable_terminator"); intern_raise_missing_tag_terminator = rb_intern("raise_missing_tag_terminator"); intern_is_blank = rb_intern("blank?"); intern_parse = rb_intern("parse"); intern_square_brackets = rb_intern("[]"); intern_unknown_tag_in_liquid_tag = rb_intern("unknown_tag_in_liquid_tag"); intern_ivar_nodelist = rb_intern("@nodelist"); tag_registry = rb_funcall(cLiquidTemplate, rb_intern("tags"), 0); rb_global_variable(&tag_registry); VALUE cLiquidCBlockBody = rb_define_class_under(mLiquidC, "BlockBody", rb_cObject); rb_define_alloc_func(cLiquidCBlockBody, block_body_allocate); rb_define_method(cLiquidCBlockBody, "initialize", block_body_initialize, 1); rb_define_method(cLiquidCBlockBody, "parse", block_body_parse, 2); rb_define_method(cLiquidCBlockBody, "freeze", block_body_freeze, 0); rb_define_method(cLiquidCBlockBody, "render_to_output_buffer", block_body_render_to_output_buffer, 2); rb_define_method(cLiquidCBlockBody, "remove_blank_strings", block_body_remove_blank_strings, 0); rb_define_method(cLiquidCBlockBody, "blank?", block_body_blank_p, 0); rb_define_method(cLiquidCBlockBody, "nodelist", block_body_nodelist, 0); rb_define_method(cLiquidCBlockBody, "disassemble", block_body_disassemble, 0); rb_define_method(cLiquidCBlockBody, "add_evaluate_expression", block_body_add_evaluate_expression, 1); rb_define_method(cLiquidCBlockBody, "add_find_variable", block_body_add_find_variable, 1); rb_define_method(cLiquidCBlockBody, "add_lookup_command", block_body_add_lookup_command, 1); rb_define_method(cLiquidCBlockBody, "add_lookup_key", block_body_add_lookup_key, 1); rb_define_method(cLiquidCBlockBody, "add_new_int_range", block_body_add_new_int_range, 0); rb_define_method(cLiquidCBlockBody, "add_hash_new", block_body_add_hash_new, 1); rb_define_method(cLiquidCBlockBody, "add_filter", block_body_add_filter, 2); rb_global_variable(&variable_placeholder); } liquid-c-4.1.0/ext/liquid_c/block.h000066400000000000000000000013471417631740700170760ustar00rootroot00000000000000#if !defined(LIQUID_BLOCK_H) #define LIQUID_BLOCK_H #include "document_body.h" #include "vm_assembler_pool.h" typedef struct block_body { bool compiled; VALUE obj; union { struct { document_body_entry_t document_body_entry; VALUE nodelist; } compiled; struct { VALUE parse_context; vm_assembler_pool_t *vm_assembler_pool; bool blank; unsigned int render_score; vm_assembler_t *code; } intermediate; } as; } block_body_t; void liquid_define_block_body(void); static inline uint8_t *block_body_instructions_ptr(block_body_header_t *body) { return ((uint8_t *)body) + body->instructions_offset; } #endif liquid-c-4.1.0/ext/liquid_c/c_buffer.c000066400000000000000000000024451417631740700175520ustar00rootroot00000000000000#include "c_buffer.h" static void c_buffer_expand_for_write(c_buffer_t *buffer, size_t write_size) { size_t capacity = c_buffer_capacity(buffer); size_t size = c_buffer_size(buffer); size_t required_capacity = size + write_size; if (capacity < 1) capacity = 1; do { capacity *= 2; } while (capacity < required_capacity); buffer->data = xrealloc(buffer->data, capacity); buffer->data_end = buffer->data + size; buffer->capacity_end = buffer->data + capacity; } void c_buffer_zero_pad_for_alignment(c_buffer_t *buffer, size_t alignment) { size_t unaligned_bytes = c_buffer_size(buffer) % alignment; if (unaligned_bytes) { size_t pad_size = alignment - unaligned_bytes; uint8_t *padding = c_buffer_extend_for_write(buffer, pad_size); memset(padding, 0, pad_size); } } void c_buffer_reserve_for_write(c_buffer_t *buffer, size_t write_size) { uint8_t *write_end = buffer->data_end + write_size; if (write_end > buffer->capacity_end) { c_buffer_expand_for_write(buffer, write_size); } } void c_buffer_write(c_buffer_t *buffer, void *write_data, size_t write_size) { c_buffer_reserve_for_write(buffer, write_size); memcpy(buffer->data_end, write_data, write_size); buffer->data_end += write_size; } liquid-c-4.1.0/ext/liquid_c/c_buffer.h000066400000000000000000000036061417631740700175570ustar00rootroot00000000000000#ifndef LIQUID_C_BUFFER_H #define LIQUID_C_BUFFER_H #include typedef struct c_buffer { uint8_t *data; uint8_t *data_end; uint8_t *capacity_end; } c_buffer_t; static inline c_buffer_t c_buffer_init(void) { return (c_buffer_t) { NULL, NULL, NULL }; } static inline c_buffer_t c_buffer_allocate(size_t capacity) { uint8_t *data = xmalloc(capacity); return (c_buffer_t) { data, data, data + capacity }; } static inline void c_buffer_free(c_buffer_t *buffer) { xfree(buffer->data); } static inline void c_buffer_reset(c_buffer_t *buffer) { buffer->data_end = buffer->data; } static inline size_t c_buffer_size(const c_buffer_t *buffer) { return buffer->data_end - buffer->data; } static inline size_t c_buffer_capacity(const c_buffer_t *buffer) { return buffer->capacity_end - buffer->data; } void c_buffer_zero_pad_for_alignment(c_buffer_t *buffer, size_t alignment); void c_buffer_reserve_for_write(c_buffer_t *buffer, size_t write_size); void c_buffer_write(c_buffer_t *buffer, void *data, size_t size); static inline void *c_buffer_extend_for_write(c_buffer_t *buffer, size_t write_size) { c_buffer_reserve_for_write(buffer, write_size); void *write_ptr = buffer->data_end; buffer->data_end += write_size; return write_ptr; } static inline void c_buffer_write_byte(c_buffer_t *buffer, uint8_t byte) { c_buffer_write(buffer, &byte, 1); } static inline void c_buffer_write_ruby_value(c_buffer_t *buffer, VALUE value) { c_buffer_write(buffer, &value, sizeof(VALUE)); } static inline void c_buffer_rb_gc_mark(c_buffer_t *buffer) { VALUE *buffer_end = (VALUE *)buffer->data_end; for (VALUE *obj_ptr = (VALUE *)buffer->data; obj_ptr < buffer_end; obj_ptr++) { rb_gc_mark(*obj_ptr); } } static inline void c_buffer_concat(c_buffer_t *dest, c_buffer_t *src) { c_buffer_write(dest, src->data, c_buffer_size(src)); } #endif liquid-c-4.1.0/ext/liquid_c/context.c000066400000000000000000000217601417631740700174640ustar00rootroot00000000000000#include "liquid.h" #include "context.h" #include "variable_lookup.h" #include "variable.h" #include "vm.h" #include "expression.h" #include "document_body.h" static VALUE cLiquidUndefinedVariable; ID id_aset, id_set_context; static ID id_has_key, id_aref, id_strainer, id_filter_methods_hash, id_strict_filters, id_global_filter; static ID id_ivar_scopes, id_ivar_environments, id_ivar_static_environments, id_ivar_strict_variables, id_ivar_interrupts, id_ivar_resource_limits, id_ivar_document_body; void context_internal_init(VALUE context_obj, context_t *context) { context->self = context_obj; context->environments = rb_ivar_get(context_obj, id_ivar_environments); Check_Type(context->environments, T_ARRAY); context->static_environments = rb_ivar_get(context_obj, id_ivar_static_environments); Check_Type(context->static_environments, T_ARRAY); context->scopes = rb_ivar_get(context_obj, id_ivar_scopes); Check_Type(context->scopes, T_ARRAY); context->strainer = rb_funcall(context->self, id_strainer, 0); Check_Type(context->strainer, T_OBJECT); context->filter_methods = rb_funcall(RBASIC_CLASS(context->strainer), id_filter_methods_hash, 0); Check_Type(context->filter_methods, T_HASH); context->interrupts = rb_ivar_get(context->self, id_ivar_interrupts); Check_Type(context->interrupts, T_ARRAY); context->resource_limits_obj = rb_ivar_get(context->self, id_ivar_resource_limits);; ResourceLimits_Get_Struct(context->resource_limits_obj, context->resource_limits); context->strict_variables = false; context->strict_filters = RTEST(rb_funcall(context->self, id_strict_filters, 0)); context->global_filter = rb_funcall(context->self, id_global_filter, 0); } void context_mark(context_t *context) { rb_gc_mark(context->self); rb_gc_mark(context->environments); rb_gc_mark(context->static_environments); rb_gc_mark(context->scopes); rb_gc_mark(context->strainer); rb_gc_mark(context->filter_methods); rb_gc_mark(context->interrupts); rb_gc_mark(context->resource_limits_obj); rb_gc_mark(context->global_filter); } static context_t *context_from_obj(VALUE self) { return &vm_from_context(self)->context; } static VALUE context_evaluate(VALUE self, VALUE expression) { // Scalar type stored directly in the VALUE, this needs to be checked anyways to use RB_BUILTIN_TYPE if (RB_SPECIAL_CONST_P(expression)) return expression; switch (RB_BUILTIN_TYPE(expression)) { case T_DATA: { if (RTYPEDDATA_P(expression) && RTYPEDDATA_TYPE(expression) == &expression_data_type) { if (RBASIC_CLASS(expression) == cLiquidCExpression) { return internal_expression_evaluate(DATA_PTR(expression), self); } else { assert(RBASIC_CLASS(expression) == cLiquidCVariableExpression); return internal_variable_expression_evaluate(DATA_PTR(expression), self); } } break; // e.g. BigDecimal } case T_OBJECT: // may be Liquid::VariableLookup or Liquid::RangeLookup { VALUE result = rb_check_funcall(expression, id_evaluate, 1, &self); return RB_LIKELY(result != Qundef) ? result : expression; } default: break; } return expression; } void context_maybe_raise_undefined_variable(VALUE self, VALUE key) { context_t *context = context_from_obj(self); if (context->strict_variables) { Check_Type(key, T_STRING); rb_enc_raise(utf8_encoding, cLiquidUndefinedVariable, "undefined variable %s", RSTRING_PTR(key)); } } static bool environments_find_variable(VALUE environments, VALUE key, bool strict_variables, VALUE raise_on_not_found, VALUE *scope_out, VALUE *variable_out) { VALUE variable = Qnil; Check_Type(environments, T_ARRAY); for (long i = 0; i < RARRAY_LEN(environments); i++) { VALUE this_environ = RARRAY_AREF(environments, i); if (RB_LIKELY(TYPE(this_environ) == T_HASH)) { // Does not invoke any default value proc, this is equivalent in // cost and semantics to #key? but loads the value as well variable = rb_hash_lookup2(this_environ, key, Qundef); if (variable != Qundef) { *variable_out = variable; *scope_out = this_environ; return true; } if (!(RTEST(raise_on_not_found) && strict_variables)) { // If we aren't running strictly, we need to invoke the default // value proc, rb_hash_aref does this variable = rb_hash_aref(this_environ, key); if (variable != Qnil) { *variable_out = variable; *scope_out = this_environ; return true; } } } else if (RTEST(rb_funcall(this_environ, id_has_key, 1, key))) { // Slow path: It is valid to pass a non-hash value to Liquid as an // environment if it supports #key? and #[] *variable_out = rb_funcall(this_environ, id_aref, 1, key); *scope_out = this_environ; return true; } } return false; } VALUE context_find_variable(context_t *context, VALUE key, VALUE raise_on_not_found) { VALUE self = context->self; VALUE scope = Qnil, variable = Qnil; VALUE scopes = context->scopes; for (long i = 0; i < RARRAY_LEN(scopes); i++) { VALUE this_scope = RARRAY_AREF(scopes, i); if (RB_LIKELY(TYPE(this_scope) == T_HASH)) { // Does not invoke any default value proc, this is equivalent in // cost and semantics to #key? but loads the value as well variable = rb_hash_lookup2(this_scope, key, Qundef); if (variable != Qundef) { scope = this_scope; goto variable_found; } } else if (RTEST(rb_funcall(this_scope, id_has_key, 1, key))) { // Slow path: It is valid to pass a non-hash value to Liquid as a // scope if it supports #key? and #[] variable = rb_funcall(this_scope, id_aref, 1, key); goto variable_found; } } if (environments_find_variable(context->environments, key, context->strict_variables, raise_on_not_found, &scope, &variable)) goto variable_found; if (environments_find_variable(context->static_environments, key, context->strict_variables, raise_on_not_found, &scope, &variable)) goto variable_found; if (RTEST(raise_on_not_found)) { context_maybe_raise_undefined_variable(self, key); } variable = Qnil; variable_found: variable = materialize_proc(self, scope, key, variable); variable = value_to_liquid_and_set_context(variable, self); return variable; } static VALUE context_find_variable_method(VALUE self, VALUE key, VALUE raise_on_not_found) { return context_find_variable(context_from_obj(self), key, raise_on_not_found); } static VALUE context_set_strict_variables(VALUE self, VALUE strict_variables) { context_t *context = context_from_obj(self); context->strict_variables = RTEST(strict_variables); rb_ivar_set(self, id_ivar_strict_variables, strict_variables); return Qnil; } // Shopify requires checking if we are filtering, so provide a // way to do that in liquid-c until we figure out how we want to // support that longer term. VALUE context_filtering_p(VALUE self) { return liquid_vm_filtering(self) ? Qtrue : Qfalse; } void liquid_define_context(void) { id_has_key = rb_intern("key?"); id_aset = rb_intern("[]="); id_aref = rb_intern("[]"); id_set_context = rb_intern("context="); id_strainer = rb_intern("strainer"); id_filter_methods_hash = rb_intern("filter_methods_hash"); id_strict_filters = rb_intern("strict_filters"); id_global_filter = rb_intern("global_filter"); id_ivar_scopes = rb_intern("@scopes"); id_ivar_environments = rb_intern("@environments"); id_ivar_static_environments = rb_intern("@static_environments"); id_ivar_strict_variables = rb_intern("@strict_variables"); id_ivar_interrupts = rb_intern("@interrupts"); id_ivar_resource_limits = rb_intern("@resource_limits"); id_ivar_document_body = rb_intern("@document_body"); cLiquidVariableLookup = rb_const_get(mLiquid, rb_intern("VariableLookup")); rb_global_variable(&cLiquidVariableLookup); cLiquidUndefinedVariable = rb_const_get(mLiquid, rb_intern("UndefinedVariable")); rb_global_variable(&cLiquidUndefinedVariable); VALUE cLiquidContext = rb_const_get(mLiquid, rb_intern("Context")); rb_define_method(cLiquidContext, "c_evaluate", context_evaluate, 1); rb_define_method(cLiquidContext, "c_find_variable", context_find_variable_method, 2); rb_define_method(cLiquidContext, "c_strict_variables=", context_set_strict_variables, 1); rb_define_private_method(cLiquidContext, "c_filtering?", context_filtering_p, 0); } liquid-c-4.1.0/ext/liquid_c/context.h000066400000000000000000000037041417631740700174670ustar00rootroot00000000000000#if !defined(LIQUID_CONTEXT_H) #define LIQUID_CONTEXT_H #include "resource_limits.h" typedef struct context { VALUE self; VALUE environments; VALUE static_environments; VALUE scopes; VALUE strainer; VALUE filter_methods; VALUE interrupts; VALUE resource_limits_obj; resource_limits_t *resource_limits; VALUE global_filter; bool strict_variables; bool strict_filters; } context_t; void liquid_define_context(void); void context_internal_init(VALUE context_obj, context_t *context); void context_mark(context_t *context); VALUE context_find_variable(context_t *context, VALUE key, VALUE raise_on_not_found); void context_maybe_raise_undefined_variable(VALUE self, VALUE key); extern ID id_aset, id_set_context; #ifndef RB_SPECIAL_CONST_P // RB_SPECIAL_CONST_P added in Ruby 2.3 #define RB_SPECIAL_CONST_P SPECIAL_CONST_P #endif inline static VALUE value_to_liquid_and_set_context(VALUE value, VALUE context_to_set) { // Scalar type stored directly in the VALUE, these all have a #to_liquid // that returns self, and should have no #context= method if (RB_SPECIAL_CONST_P(value)) return value; VALUE klass = RBASIC(value)->klass; // More basic types having #to_liquid of self and no #context= if (klass == rb_cString || klass == rb_cArray || klass == rb_cHash) return value; value = rb_funcall(value, id_to_liquid, 0); if (rb_respond_to(value, id_set_context)) rb_funcall(value, id_set_context, 1, context_to_set); return value; } inline static VALUE materialize_proc(VALUE context, VALUE scope, VALUE key, VALUE value) { if (scope != Qnil && rb_obj_is_proc(value) && rb_respond_to(scope, id_aset)) { if (rb_proc_arity(value) == 1) { value = rb_funcall(value, id_call, 1, context); } else { value = rb_funcall(value, id_call, 0); } rb_funcall(scope, id_aset, 2, key, value); } return value; } #endif liquid-c-4.1.0/ext/liquid_c/document_body.c000066400000000000000000000055141417631740700206320ustar00rootroot00000000000000#include #include #include "liquid.h" #include "vm_assembler.h" #include "document_body.h" static VALUE cLiquidCDocumentBody; static void document_body_mark(void *ptr) { document_body_t *body = ptr; rb_gc_mark(body->constants); } static void document_body_free(void *ptr) { document_body_t *body = ptr; c_buffer_free(&body->buffer); xfree(body); } static size_t document_body_memsize(const void *ptr) { const document_body_t *body = ptr; return sizeof(document_body_t) + c_buffer_size(&body->buffer); } const rb_data_type_t document_body_data_type = { "liquid_document_body", { document_body_mark, document_body_free, document_body_memsize }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; static VALUE document_body_allocate(VALUE klass) { document_body_t *body; VALUE obj = TypedData_Make_Struct(klass, document_body_t, &document_body_data_type, body); body->self = obj; body->constants = rb_ary_new(); body->buffer = c_buffer_init(); return obj; } #define DocumentBody_Get_Struct(obj, sval) TypedData_Get_Struct(obj, document_body_t, &document_body_data_type, sval) VALUE document_body_new_instance(void) { return rb_class_new_instance(0, NULL, cLiquidCDocumentBody); } document_body_entry_t document_body_write_block_body(VALUE self, bool blank, uint32_t render_score, vm_assembler_t *code) { assert(!RB_OBJ_FROZEN(self)); document_body_t *body; DocumentBody_Get_Struct(self, body); c_buffer_zero_pad_for_alignment(&body->buffer, alignof(block_body_header_t)); size_t buffer_offset = c_buffer_size(&body->buffer); assert(c_buffer_size(&code->constants) % sizeof(VALUE) == 0); uint32_t constants_len = (uint32_t)(c_buffer_size(&code->constants) / sizeof(VALUE)); block_body_header_t *buf_block_body = c_buffer_extend_for_write(&body->buffer, sizeof(block_body_header_t)); buf_block_body->instructions_offset = (uint32_t)sizeof(block_body_header_t); buf_block_body->instructions_bytes = (uint32_t)c_buffer_size(&code->instructions); buf_block_body->constants_offset = (uint32_t)RARRAY_LEN(body->constants); buf_block_body->constants_len = constants_len; buf_block_body->flags = 0; if (blank) buf_block_body->flags |= BLOCK_BODY_HEADER_FLAG_BLANK; buf_block_body->render_score = render_score; buf_block_body->max_stack_size = code->max_stack_size; c_buffer_concat(&body->buffer, &code->instructions); rb_ary_cat(body->constants, (VALUE *)code->constants.data, constants_len); return (document_body_entry_t) { .body = body, .buffer_offset = buffer_offset }; } void liquid_define_document_body(void) { cLiquidCDocumentBody = rb_define_class_under(mLiquidC, "DocumentBody", rb_cObject); rb_global_variable(&cLiquidCDocumentBody); rb_define_alloc_func(cLiquidCDocumentBody, document_body_allocate); } liquid-c-4.1.0/ext/liquid_c/document_body.h000066400000000000000000000033671417631740700206430ustar00rootroot00000000000000#ifndef LIQUID_DOCUMENT_BODY_H #define LIQUID_DOCUMENT_BODY_H #include "c_buffer.h" #include "vm_assembler.h" typedef struct block_body_header { uint32_t instructions_offset; uint32_t instructions_bytes; uint32_t constants_offset; uint32_t constants_len; uint32_t flags; uint32_t render_score; uint64_t max_stack_size; } block_body_header_t; #define BLOCK_BODY_HEADER_FLAG_BLANK (1 << 0) #define BLOCK_BODY_HEADER_BLANK_P(header) (header->flags & BLOCK_BODY_HEADER_FLAG_BLANK) typedef struct document_body { VALUE self; VALUE constants; c_buffer_t buffer; } document_body_t; typedef struct document_body_entry { document_body_t *body; size_t buffer_offset; } document_body_entry_t; void liquid_define_document_body(void); VALUE document_body_new_instance(void); document_body_entry_t document_body_write_block_body(VALUE self, bool blank, uint32_t render_score, vm_assembler_t *code); static inline void document_body_entry_mark(document_body_entry_t *entry) { rb_gc_mark(entry->body->self); rb_gc_mark(entry->body->constants); } static inline block_body_header_t *document_body_get_block_body_header_ptr(const document_body_entry_t *entry) { return (block_body_header_t *)(entry->body->buffer.data + entry->buffer_offset); } static inline const VALUE *document_body_get_constants_ptr(const document_body_entry_t *entry) { block_body_header_t *header = document_body_get_block_body_header_ptr(entry); return RARRAY_PTR(entry->body->constants) + header->constants_offset; } static inline void document_body_ensure_compile_finished(document_body_t *body) { if (RB_UNLIKELY(!RB_OBJ_FROZEN(body->self))) { rb_raise(rb_eRuntimeError, "Liquid document hasn't finished compilation"); } } #endif liquid-c-4.1.0/ext/liquid_c/expression.c000066400000000000000000000064401417631740700201750ustar00rootroot00000000000000#include "liquid.h" #include "vm_assembler.h" #include "parser.h" #include "vm.h" #include "expression.h" VALUE cLiquidCExpression; static void expression_mark(void *ptr) { expression_t *expression = ptr; vm_assembler_gc_mark(&expression->code); } static void expression_free(void *ptr) { expression_t *expression = ptr; vm_assembler_free(&expression->code); xfree(expression); } static size_t expression_memsize(const void *ptr) { const expression_t *expression = ptr; return sizeof(expression_t) + vm_assembler_alloc_memsize(&expression->code); } const rb_data_type_t expression_data_type = { "liquid_expression", { expression_mark, expression_free, expression_memsize, }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; VALUE expression_new(VALUE klass, expression_t **expression_ptr) { expression_t *expression; VALUE obj = TypedData_Make_Struct(klass, expression_t, &expression_data_type, expression); *expression_ptr = expression; vm_assembler_init(&expression->code); return obj; } static VALUE internal_expression_parse(parser_t *p) { if (p->cur.type == TOKEN_EOS) return Qnil; // Avoid allocating an expression object just to wrap a constant VALUE const_obj = try_parse_constant_expression(p); if (const_obj != Qundef) return const_obj; expression_t *expression; VALUE expr_obj = expression_new(cLiquidCExpression, &expression); parse_and_compile_expression(p, &expression->code); assert(expression->code.stack_size == 1); vm_assembler_add_leave(&expression->code); return expr_obj; } static VALUE expression_strict_parse(VALUE klass, VALUE markup) { if (NIL_P(markup)) return Qnil; StringValue(markup); char *start = RSTRING_PTR(markup); parser_t p; init_parser(&p, start, start + RSTRING_LEN(markup)); VALUE expr_obj = internal_expression_parse(&p); if (p.cur.type != TOKEN_EOS) rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "[:%s] is not a valid expression", symbol_names[p.cur.type]); return expr_obj; } VALUE expression_evaluate(VALUE self, VALUE context) { expression_t *expression; Expression_Get_Struct(self, expression); return liquid_vm_evaluate(context, &expression->code); } VALUE internal_expression_evaluate(expression_t *expression, VALUE context) { return liquid_vm_evaluate(context, &expression->code); } static VALUE expression_disassemble(VALUE self) { expression_t *expression; Expression_Get_Struct(self, expression); VALUE constants = rb_ary_new(); uint32_t constants_len = (uint32_t)(c_buffer_size(&expression->code.constants) / sizeof(VALUE)); rb_ary_cat(constants, (VALUE *)expression->code.constants.data, constants_len); return vm_assembler_disassemble( expression->code.instructions.data, expression->code.instructions.data_end, &constants ); } void liquid_define_expression(void) { cLiquidCExpression = rb_define_class_under(mLiquidC, "Expression", rb_cObject); rb_undef_alloc_func(cLiquidCExpression); rb_define_singleton_method(cLiquidCExpression, "strict_parse", expression_strict_parse, 1); rb_define_method(cLiquidCExpression, "evaluate", expression_evaluate, 1); rb_define_method(cLiquidCExpression, "disassemble", expression_disassemble, 0); } liquid-c-4.1.0/ext/liquid_c/expression.h000066400000000000000000000012351417631740700201770ustar00rootroot00000000000000#if !defined(LIQUID_EXPRESSION_H) #define LIQUID_EXPRESSION_H #include "vm_assembler.h" #include "parser.h" extern VALUE cLiquidCExpression; extern const rb_data_type_t expression_data_type; typedef struct expression { vm_assembler_t code; } expression_t; extern const rb_data_type_t expression_data_type; #define Expression_Get_Struct(obj, sval) TypedData_Get_Struct(obj, expression_t, &expression_data_type, sval) void liquid_define_expression(void); VALUE expression_new(VALUE klass, expression_t **expression_ptr); VALUE expression_evaluate(VALUE self, VALUE context); VALUE internal_expression_evaluate(expression_t *expression, VALUE context); #endif liquid-c-4.1.0/ext/liquid_c/extconf.rb000077500000000000000000000014531417631740700176270ustar00rootroot00000000000000# frozen_string_literal: true require "mkmf" $CFLAGS << " -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers" append_cflags("-fvisibility=hidden") # In Ruby 2.6 and earlier, the Ruby headers did not have struct timespec defined valid_headers = RbConfig::CONFIG["host_os"] !~ /linux/ || Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7") pedantic = !ENV["LIQUID_C_PEDANTIC"].to_s.empty? if pedantic && valid_headers $CFLAGS << " -Werror" end if ENV["DEBUG"] == "true" append_cflags("-fbounds-check") CONFIG["optflags"] = " -O0" else $CFLAGS << " -DNDEBUG" end if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7.0") # added in 2.7 $CFLAGS << " -DHAVE_RB_HASH_BULK_INSERT" end $warnflags&.gsub!(/-Wdeclaration-after-statement/, "") create_makefile("liquid_c") liquid-c-4.1.0/ext/liquid_c/intutil.h000066400000000000000000000006521417631740700174720ustar00rootroot00000000000000#ifndef LIQUID_INTUTIL_H #define LIQUID_INTUTIL_H #include static inline unsigned int bytes_to_uint24(const uint8_t *bytes) { return (bytes[0] << 16) | (bytes[1] << 8) | bytes[2]; } static inline void uint24_to_bytes(unsigned int num, uint8_t *bytes) { assert(num < (1 << 24)); bytes[0] = num >> 16; bytes[1] = num >> 8; bytes[2] = num; assert(bytes_to_uint24(bytes) == num); } #endif liquid-c-4.1.0/ext/liquid_c/lexer.c000066400000000000000000000103301417631740700171060ustar00rootroot00000000000000#include "liquid.h" #include "lexer.h" #include "usage.h" #include const char *symbol_names[TOKEN_END] = { [TOKEN_NONE] = "none", [TOKEN_COMPARISON] = "comparison", [TOKEN_STRING] = "string", [TOKEN_NUMBER] = "number", [TOKEN_IDENTIFIER] = "id", [TOKEN_DOTDOT] = "dotdot", [TOKEN_EOS] = "end_of_string", [TOKEN_PIPE] = "pipe", [TOKEN_DOT] = "dot", [TOKEN_COLON] = "colon", [TOKEN_COMMA] = "comma", [TOKEN_OPEN_SQUARE] = "open_square", [TOKEN_CLOSE_SQUARE] = "close_square", [TOKEN_OPEN_ROUND] = "open_round", [TOKEN_CLOSE_ROUND] = "close_round", [TOKEN_QUESTION] = "question", [TOKEN_DASH] = "dash" }; inline static int is_identifier(char c) { return ISALNUM(c) || c == '_' || c == '-'; } inline static int is_special(char c) { switch (c) { case '|': case '.': case ':': case ',': case '[': case ']': case '(': case ')': case '?': case '-': return 1; } return 0; } // Returns a pointer to the character after the end of the match. inline static const char *prefix_end(const char *cur, const char *end, const char *pattern) { size_t pattern_len = strlen(pattern); if (pattern_len > (size_t)(end - cur)) return NULL; if (memcmp(cur, pattern, pattern_len) != 0) return NULL; return cur + pattern_len; } inline static const char *scan_past(const char *cur, const char *end, char target) { const char *match = memchr(cur + 1, target, end - cur - 1); return match ? match + 1 : NULL; } #define RETURN_TOKEN(t, n) { \ const char *tok_end = str + (n); \ token->type = (t); \ token->val = str; \ return (token->val_end = tok_end); \ } // Reads one token from start, and fills it into the token argument. // Returns the start of the next token if any, otherwise the end of the string. const char *lex_one(const char *start, const char *end, lexer_token_t *token) { // str references the start of the token, after whitespace is skipped. // cur references the currently processing character during iterative lexing. const char *str = start, *cur; while (str < end && ISSPACE(*str)) ++str; token->val = token->val_end = NULL; token->flags = 0; if (str >= end) return str; char c = *str; // First character of the token. char cn = '\0'; // Second character if available, for lookahead. if (str + 1 < end) cn = str[1]; switch (c) { case '<': RETURN_TOKEN(TOKEN_COMPARISON, cn == '>' || cn == '=' ? 2 : 1); case '>': RETURN_TOKEN(TOKEN_COMPARISON, cn == '=' ? 2 : 1); case '=': case '!': if (cn == '=') RETURN_TOKEN(TOKEN_COMPARISON, 2); break; case '.': if (cn == '.') RETURN_TOKEN(TOKEN_DOTDOT, 2); break; } if ((cur = prefix_end(str, end, "contains"))) RETURN_TOKEN(TOKEN_COMPARISON, cur - str); if (c == '\'' || c == '"') { cur = scan_past(str, end, c); if (cur) { // Quote was properly terminated. RETURN_TOKEN(TOKEN_STRING, cur - str); } } // Instrument for bug: https://github.com/Shopify/liquid-c/pull/120 if (c == '-' && str + 1 < end && str[1] == '.') { usage_increment("liquid_c_negative_float_without_integer"); } if (ISDIGIT(c) || c == '-') { int has_dot = 0; cur = str; while (++cur < end) { if (!has_dot && *cur == '.') { has_dot = 1; } else if (!ISDIGIT(*cur)) { break; } } cur--; // Point to last digit (or dot). if (*cur == '.') { cur--; // Ignore any trailing dot. has_dot = 0; } if (*cur != '-') { if (has_dot) token->flags |= TOKEN_FLOAT_NUMBER; RETURN_TOKEN(TOKEN_NUMBER, cur + 1 - str); } } if (ISALPHA(c) || c == '_') { cur = str; while (++cur < end && is_identifier(*cur)) {} if (cur < end && *cur == '?') cur++; RETURN_TOKEN(TOKEN_IDENTIFIER, cur - str); } if (is_special(c)) RETURN_TOKEN(c, 1); rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %c", c); return NULL; } #undef RETURN_TOKEN liquid-c-4.1.0/ext/liquid_c/lexer.h000066400000000000000000000027221417631740700171210ustar00rootroot00000000000000#if !defined(LIQUID_LEXER_H) #define LIQUID_LEXER_H enum lexer_token_type { TOKEN_NONE, TOKEN_COMPARISON, TOKEN_STRING, TOKEN_NUMBER, TOKEN_IDENTIFIER, TOKEN_DOTDOT, TOKEN_EOS, TOKEN_PIPE = '|', TOKEN_DOT = '.', TOKEN_COLON = ':', TOKEN_COMMA = ',', TOKEN_OPEN_SQUARE = '[', TOKEN_CLOSE_SQUARE = ']', TOKEN_OPEN_ROUND = '(', TOKEN_CLOSE_ROUND = ')', TOKEN_QUESTION = '?', TOKEN_DASH = '-', TOKEN_END = 256 }; #define TOKEN_FLOAT_NUMBER 0x4 typedef struct lexer_token { unsigned char type, flags; const char *val, *val_end; } lexer_token_t; extern const char *symbol_names[TOKEN_END]; const char *lex_one(const char *str, const char *end, lexer_token_t *token); inline static VALUE token_to_rstr(lexer_token_t token) { return rb_enc_str_new(token.val, token.val_end - token.val, utf8_encoding); } inline static VALUE token_check_for_symbol(lexer_token_t token) { return rb_check_symbol_cstr(token.val, token.val_end - token.val, utf8_encoding); } inline static VALUE token_to_rstr_leveraging_existing_symbol(lexer_token_t token) { VALUE sym = token_check_for_symbol(token); if (RB_LIKELY(sym != Qnil)) return rb_sym2str(sym); return token_to_rstr(token); } inline static VALUE token_to_rsym(lexer_token_t token) { VALUE sym = token_check_for_symbol(token); if (RB_LIKELY(sym != Qnil)) return sym; return rb_str_intern(token_to_rstr(token)); } #endif liquid-c-4.1.0/ext/liquid_c/liquid.c000066400000000000000000000055611417631740700172700ustar00rootroot00000000000000#include "liquid.h" #include "tokenizer.h" #include "variable.h" #include "lexer.h" #include "parser.h" #include "raw.h" #include "resource_limits.h" #include "expression.h" #include "document_body.h" #include "block.h" #include "context.h" #include "parse_context.h" #include "variable_lookup.h" #include "vm_assembler_pool.h" #include "vm.h" #include "usage.h" ID id_evaluate; ID id_to_liquid; ID id_to_s; ID id_call; ID id_compile_evaluate; ID id_ivar_line_number; VALUE mLiquid, mLiquidC, cLiquidVariable, cLiquidTemplate, cLiquidBlockBody; VALUE cLiquidVariableLookup, cLiquidRangeLookup; VALUE cLiquidArgumentError, cLiquidSyntaxError, cMemoryError; rb_encoding *utf8_encoding; int utf8_encoding_index; __attribute__((noreturn)) void raise_non_utf8_encoding_error(VALUE string, const char *value_name) { rb_raise(rb_eEncCompatError, "non-UTF8 encoded %s (%"PRIsVALUE") not supported", value_name, rb_obj_encoding(string)); } RUBY_FUNC_EXPORTED void Init_liquid_c(void) { id_evaluate = rb_intern("evaluate"); id_to_liquid = rb_intern("to_liquid"); id_to_s = rb_intern("to_s"); id_call = rb_intern("call"); id_compile_evaluate = rb_intern("compile_evaluate"); id_ivar_line_number = rb_intern("@line_number"); utf8_encoding = rb_utf8_encoding(); utf8_encoding_index = rb_enc_to_index(utf8_encoding); mLiquid = rb_define_module("Liquid"); rb_global_variable(&mLiquid); mLiquidC = rb_define_module_under(mLiquid, "C"); rb_global_variable(&mLiquidC); cLiquidArgumentError = rb_const_get(mLiquid, rb_intern("ArgumentError")); rb_global_variable(&cLiquidArgumentError); cLiquidSyntaxError = rb_const_get(mLiquid, rb_intern("SyntaxError")); rb_global_variable(&cLiquidSyntaxError); cMemoryError = rb_const_get(mLiquid, rb_intern("MemoryError")); rb_global_variable(&cMemoryError); cLiquidVariable = rb_const_get(mLiquid, rb_intern("Variable")); rb_global_variable(&cLiquidVariable); cLiquidTemplate = rb_const_get(mLiquid, rb_intern("Template")); rb_global_variable(&cLiquidTemplate); cLiquidBlockBody = rb_const_get(mLiquid, rb_intern("BlockBody")); rb_global_variable(&cLiquidBlockBody); cLiquidVariableLookup = rb_const_get(mLiquid, rb_intern("VariableLookup")); rb_global_variable(&cLiquidVariableLookup); cLiquidRangeLookup = rb_const_get(mLiquid, rb_intern("RangeLookup")); rb_global_variable(&cLiquidRangeLookup); liquid_define_tokenizer(); liquid_define_parser(); liquid_define_raw(); liquid_define_resource_limits(); liquid_define_expression(); liquid_define_variable(); liquid_define_document_body(); liquid_define_block_body(); liquid_define_context(); liquid_define_parse_context(); liquid_define_variable_lookup(); liquid_define_vm_assembler_pool(); liquid_define_vm_assembler(); liquid_define_vm(); liquid_define_usage(); } liquid-c-4.1.0/ext/liquid_c/liquid.h000066400000000000000000000017121417631740700172670ustar00rootroot00000000000000#if !defined(LIQUID_H) #define LIQUID_H #include #include #include extern ID id_evaluate; extern ID id_to_liquid; extern ID id_to_s; extern ID id_call; extern ID id_compile_evaluate; extern ID id_ivar_line_number; extern VALUE mLiquid, mLiquidC, cLiquidVariable, cLiquidTemplate, cLiquidBlockBody; extern VALUE cLiquidVariableLookup, cLiquidRangeLookup; extern VALUE cLiquidArgumentError, cLiquidSyntaxError, cMemoryError; extern rb_encoding *utf8_encoding; extern int utf8_encoding_index; __attribute__((noreturn)) void raise_non_utf8_encoding_error(VALUE string, const char *string_name); static inline void check_utf8_encoding(VALUE string, const char *string_name) { if (RB_UNLIKELY(RB_ENCODING_GET_INLINED(string) != utf8_encoding_index)) raise_non_utf8_encoding_error(string, string_name); } #ifndef RB_LIKELY // RB_LIKELY added in Ruby 2.4 #define RB_LIKELY(x) (__builtin_expect(!!(x), 1)) #endif #endif liquid-c-4.1.0/ext/liquid_c/parse_context.c000066400000000000000000000046011417631740700206510ustar00rootroot00000000000000#include "parse_context.h" #include "document_body.h" static ID id_document_body, id_vm_assembler_pool; static bool parse_context_document_body_initialized_p(VALUE self) { return RTEST(rb_attr_get(self, id_document_body)); } static void parse_context_init_document_body(VALUE self) { VALUE document_body = document_body_new_instance(); rb_ivar_set(self, id_document_body, document_body); } VALUE parse_context_get_document_body(VALUE self) { assert(parse_context_document_body_initialized_p(self)); return rb_ivar_get(self, id_document_body); } vm_assembler_pool_t *parse_context_init_vm_assembler_pool(VALUE self) { assert(!RTEST(rb_attr_get(self, id_vm_assembler_pool))); VALUE vm_assembler_pool_obj = vm_assembler_pool_new(); rb_ivar_set(self, id_vm_assembler_pool, vm_assembler_pool_obj); vm_assembler_pool_t *vm_assembler_pool; VMAssemblerPool_Get_Struct(vm_assembler_pool_obj, vm_assembler_pool); return vm_assembler_pool; } vm_assembler_pool_t *parse_context_get_vm_assembler_pool(VALUE self) { VALUE obj = rb_ivar_get(self, id_vm_assembler_pool); if (obj == Qnil) { rb_raise(rb_eRuntimeError, "Liquid::ParseContext#start_liquid_c_parsing has not yet been called"); } vm_assembler_pool_t *vm_assembler_pool; VMAssemblerPool_Get_Struct(obj, vm_assembler_pool); return vm_assembler_pool; } static VALUE parse_context_start_liquid_c_parsing(VALUE self) { if (RB_UNLIKELY(parse_context_document_body_initialized_p(self))) { rb_raise(rb_eRuntimeError, "liquid-c parsing already started for this parse context"); } parse_context_init_document_body(self); parse_context_init_vm_assembler_pool(self); return Qnil; } static VALUE parse_context_cleanup_liquid_c_parsing(VALUE self) { rb_obj_freeze(rb_ivar_get(self, id_document_body)); rb_ivar_set(self, id_document_body, Qnil); rb_ivar_set(self, id_vm_assembler_pool, Qnil); return Qnil; } void liquid_define_parse_context(void) { id_document_body = rb_intern("document_body"); id_vm_assembler_pool = rb_intern("vm_assembler_pool"); VALUE cLiquidParseContext = rb_const_get(mLiquid, rb_intern("ParseContext")); rb_define_method(cLiquidParseContext, "start_liquid_c_parsing", parse_context_start_liquid_c_parsing, 0); rb_define_method(cLiquidParseContext, "cleanup_liquid_c_parsing", parse_context_cleanup_liquid_c_parsing, 0); } liquid-c-4.1.0/ext/liquid_c/parse_context.h000066400000000000000000000004601417631740700206550ustar00rootroot00000000000000#ifndef LIQUID_PARSE_CONTEXT_H #define LIQUID_PARSE_CONTEXT_H #include #include #include "vm_assembler_pool.h" void liquid_define_parse_context(void); VALUE parse_context_get_document_body(VALUE self); vm_assembler_pool_t *parse_context_get_vm_assembler_pool(VALUE self); #endif liquid-c-4.1.0/ext/liquid_c/parser.c000066400000000000000000000167131417631740700172760ustar00rootroot00000000000000#include "liquid.h" #include "parser.h" #include "lexer.h" static VALUE empty_string; static ID id_to_i, idEvaluate; void init_parser(parser_t *p, const char *str, const char *end) { p->str_end = end; p->cur.type = p->next.type = TOKEN_EOS; p->str = lex_one(str, end, &p->cur); p->str = lex_one(p->str, end, &p->next); } lexer_token_t parser_consume_any(parser_t *p) { lexer_token_t cur = p->cur; p->cur = p->next; p->next.type = TOKEN_EOS; p->str = lex_one(p->str, p->str_end, &p->next); return cur; } lexer_token_t parser_must_consume(parser_t *p, unsigned char type) { if (p->cur.type != type) { rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Expected %s but found %s", symbol_names[type], symbol_names[p->cur.type]); } return parser_consume_any(p); } lexer_token_t parser_consume(parser_t *p, unsigned char type) { if (p->cur.type != type) { lexer_token_t zero = {0}; return zero; } return parser_consume_any(p); } inline static int rstring_eq(VALUE rstr, const char *str) { size_t str_len = strlen(str); return TYPE(rstr) == T_STRING && str_len == (size_t)RSTRING_LEN(rstr) && memcmp(RSTRING_PTR(rstr), str, str_len) == 0; } static VALUE parse_number(parser_t *p) { VALUE out; lexer_token_t token = parser_must_consume(p, TOKEN_NUMBER); // Set up sentinel for rb_cstr operations. char tmp = *token.val_end; *(char *)token.val_end = '\0'; if (token.flags & TOKEN_FLOAT_NUMBER) { out = DBL2NUM(rb_cstr_to_dbl(token.val, 1)); } else { out = rb_cstr_to_inum(token.val, 10, 1); } *(char *)token.val_end = tmp; return out; } static VALUE try_parse_constant_range(parser_t *p) { parser_t saved_state = *p; parser_must_consume(p, TOKEN_OPEN_ROUND); VALUE begin = try_parse_constant_expression(p); if (begin == Qundef) { *p = saved_state; return Qundef; } parser_must_consume(p, TOKEN_DOTDOT); VALUE end = try_parse_constant_expression(p); if (end == Qundef) { *p = saved_state; return Qundef; } parser_must_consume(p, TOKEN_CLOSE_ROUND); begin = rb_funcall(begin, id_to_i, 0); end = rb_funcall(end, id_to_i, 0); bool exclude_end = false; return rb_range_new(begin, end, exclude_end); } static void parse_and_compile_range(parser_t *p, vm_assembler_t *code) { VALUE const_range = try_parse_constant_range(p); if (const_range != Qundef) { vm_assembler_add_push_const(code, const_range); return; } parser_must_consume(p, TOKEN_OPEN_ROUND); parse_and_compile_expression(p, code); parser_must_consume(p, TOKEN_DOTDOT); parse_and_compile_expression(p, code); parser_must_consume(p, TOKEN_CLOSE_ROUND); vm_assembler_add_new_int_range(code); } static void parse_and_compile_variable_lookup(parser_t *p, vm_assembler_t *code) { if (parser_consume(p, TOKEN_OPEN_SQUARE).type) { parse_and_compile_expression(p, code); parser_must_consume(p, TOKEN_CLOSE_SQUARE); vm_assembler_add_find_variable(code); } else { VALUE name = token_to_rstr_leveraging_existing_symbol(parser_must_consume(p, TOKEN_IDENTIFIER)); vm_assembler_add_find_static_variable(code, name); } while (true) { if (p->cur.type == TOKEN_OPEN_SQUARE) { parser_consume_any(p); parse_and_compile_expression(p, code); parser_must_consume(p, TOKEN_CLOSE_SQUARE); vm_assembler_add_lookup_key(code); } else if (p->cur.type == TOKEN_DOT) { parser_consume_any(p); VALUE key = token_to_rstr_leveraging_existing_symbol(parser_must_consume(p, TOKEN_IDENTIFIER)); if (rstring_eq(key, "size") || rstring_eq(key, "first") || rstring_eq(key, "last")) vm_assembler_add_lookup_command(code, key); else vm_assembler_add_lookup_const_key(code, key); } else { break; } } } static VALUE try_parse_literal(parser_t *p) { if (p->next.type == TOKEN_DOT || p->next.type == TOKEN_OPEN_SQUARE) return Qundef; const char *str = p->cur.val; long size = p->cur.val_end - str; VALUE result = Qundef; switch (size) { case 3: if (memcmp(str, "nil", size) == 0) result = Qnil; break; case 4: if (memcmp(str, "null", size) == 0) { result = Qnil; } else if (memcmp(str, "true", size) == 0) { result = Qtrue; } break; case 5: switch (*str) { case 'f': if (memcmp(str, "false", size) == 0) result = Qfalse; break; case 'b': if (memcmp(str, "blank", size) == 0) result = empty_string; break; case 'e': if (memcmp(str, "empty", size) == 0) result = empty_string; break; } break; } if (result != Qundef) parser_consume_any(p); return result; } VALUE try_parse_constant_expression(parser_t *p) { switch (p->cur.type) { case TOKEN_IDENTIFIER: return try_parse_literal(p); case TOKEN_NUMBER: return parse_number(p); case TOKEN_OPEN_ROUND: return try_parse_constant_range(p); case TOKEN_STRING: { lexer_token_t token = parser_consume_any(p); token.val++; token.val_end--; return token_to_rstr(token); } } return Qundef; } static void parse_and_compile_number(parser_t *p, vm_assembler_t *code) { VALUE num = parse_number(p); if (RB_FIXNUM_P(num)) vm_assembler_add_push_fixnum(code, num); else vm_assembler_add_push_const(code, num); return; } void parse_and_compile_expression(parser_t *p, vm_assembler_t *code) { switch (p->cur.type) { case TOKEN_IDENTIFIER: { VALUE literal = try_parse_literal(p); if (literal != Qundef) { vm_assembler_add_push_literal(code, literal); return; } __attribute__ ((fallthrough)); } case TOKEN_OPEN_SQUARE: parse_and_compile_variable_lookup(p, code); return; case TOKEN_NUMBER: parse_and_compile_number(p, code); return; case TOKEN_OPEN_ROUND: parse_and_compile_range(p, code); return; case TOKEN_STRING: { lexer_token_t token = parser_consume_any(p); token.val++; token.val_end--; VALUE str = token_to_rstr(token); vm_assembler_add_push_const(code, str); return; } } if (p->cur.type == TOKEN_EOS) { rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "[:%s] is not a valid expression", symbol_names[p->cur.type]); } else { rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "[:%s, \"%.*s\"] is not a valid expression", symbol_names[p->cur.type], (int)(p->cur.val_end - p->cur.val), p->cur.val); } } void liquid_define_parser(void) { id_to_i = rb_intern("to_i"); idEvaluate = rb_intern("evaluate"); empty_string = rb_utf8_str_new_literal(""); rb_global_variable(&empty_string); } liquid-c-4.1.0/ext/liquid_c/parser.h000066400000000000000000000011621417631740700172730ustar00rootroot00000000000000#if !defined(LIQUID_PARSER_H) #define LIQUID_PARSER_H #include "lexer.h" #include "vm_assembler.h" typedef struct parser { lexer_token_t cur, next; const char *str, *str_end; } parser_t; void init_parser(parser_t *parser, const char *str, const char *end); lexer_token_t parser_must_consume(parser_t *parser, unsigned char type); lexer_token_t parser_consume(parser_t *parser, unsigned char type); lexer_token_t parser_consume_any(parser_t *parser); void parse_and_compile_expression(parser_t *p, vm_assembler_t *code); VALUE try_parse_constant_expression(parser_t *p); void liquid_define_parser(void); #endif liquid-c-4.1.0/ext/liquid_c/raw.c000066400000000000000000000063131417631740700165660ustar00rootroot00000000000000#include "liquid.h" #include "raw.h" #include "stringutil.h" #include "tokenizer.h" static VALUE id_block_name, id_raise_tag_never_closed, id_block_delimiter, id_ivar_body; static VALUE cLiquidRaw; struct full_token_possibly_invalid_t { long body_len; const char *delimiter_start; long delimiter_len; }; static bool match_full_token_possibly_invalid(token_t *token, struct full_token_possibly_invalid_t *match) { const char *str = token->str_full; long len = token->len_full; match->body_len = 0; match->delimiter_start = NULL; match->delimiter_len = 0; if (len < 5) return false; // Must be at least 5 characters: \{%\w%\} if (str[len - 1] != '}' || str[len - 2] != '%') return false; const char *curr_delimiter_start; long curr_delimiter_len = 0; for (long i = len - 3; i >= 0; i--) { char c = str[i]; if (is_word_char(c)) { curr_delimiter_start = str + i; curr_delimiter_len++; } else { if (curr_delimiter_len > 0) { match->delimiter_start = curr_delimiter_start; match->delimiter_len = curr_delimiter_len; } curr_delimiter_start = NULL; curr_delimiter_len = 0; } if (c == '%' && match->delimiter_len > 0 && i - 1 >= 0 && str[i - 1] == '{') { match->body_len = i - 1; return true; } } return false; } static VALUE raw_parse_method(VALUE self, VALUE tokens) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(tokens, tokenizer); token_t token; struct full_token_possibly_invalid_t match; VALUE block_delimiter = rb_funcall(self, id_block_delimiter, 0); Check_Type(block_delimiter, T_STRING); char *block_delimiter_str = RSTRING_PTR(block_delimiter); long block_delimiter_len = RSTRING_LEN(block_delimiter); const char *body = NULL; long body_len = 0; while (true) { tokenizer_next(tokenizer, &token); if (!token.type) break; if (body == NULL) { body = token.str_full; } if (match_full_token_possibly_invalid(&token, &match) && match.delimiter_len == block_delimiter_len && memcmp(match.delimiter_start, block_delimiter_str, block_delimiter_len) == 0) { body_len += match.body_len; VALUE body_str = rb_enc_str_new(body, body_len, utf8_encoding); rb_ivar_set(self, id_ivar_body, body_str); if (RBASIC_CLASS(self) == cLiquidRaw) { tokenizer->raw_tag_body = RSTRING_PTR(body_str); tokenizer->raw_tag_body_len = (unsigned int)body_len; } return Qnil; } body_len += token.len_full; } rb_funcall(self, id_raise_tag_never_closed, 1, rb_funcall(self, id_block_name, 0)); return Qnil; } void liquid_define_raw(void) { id_block_name = rb_intern("block_name"); id_raise_tag_never_closed = rb_intern("raise_tag_never_closed"); id_block_delimiter = rb_intern("block_delimiter"); id_ivar_body = rb_intern("@body"); cLiquidRaw = rb_const_get(mLiquid, rb_intern("Raw")); rb_define_method(cLiquidRaw, "c_parse", raw_parse_method, 1); } liquid-c-4.1.0/ext/liquid_c/raw.h000066400000000000000000000001211417631740700165620ustar00rootroot00000000000000#ifndef LIQUID_RAW_H #define LIQUID_RAW_H void liquid_define_raw(void); #endif liquid-c-4.1.0/ext/liquid_c/resource_limits.c000066400000000000000000000223761417631740700212140ustar00rootroot00000000000000#include "liquid.h" #include "resource_limits.h" VALUE cLiquidResourceLimits; static void resource_limits_free(void *ptr) { resource_limits_t *resource_limits = ptr; xfree(resource_limits); } static size_t resource_limits_memsize(const void *ptr) { return sizeof(resource_limits_t); } const rb_data_type_t resource_limits_data_type = { "liquid_resource_limits", { NULL, resource_limits_free, resource_limits_memsize }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; static void resource_limits_reset(resource_limits_t *resource_limit) { resource_limit->reached_limit = true; resource_limit->last_capture_length = -1; resource_limit->render_score = 0; resource_limit->assign_score = 0; } static VALUE resource_limits_allocate(VALUE klass) { resource_limits_t *resource_limits; VALUE obj = TypedData_Make_Struct(klass, resource_limits_t, &resource_limits_data_type, resource_limits); resource_limits_reset(resource_limits); return obj; } static VALUE resource_limits_render_length_limit_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); return LONG2NUM(resource_limits->render_length_limit); } static VALUE resource_limits_set_render_length_limit_method(VALUE self, VALUE render_length_limit) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); if (render_length_limit == Qnil) { resource_limits->render_length_limit = LONG_MAX; } else { resource_limits->render_length_limit = NUM2LONG(render_length_limit); } return Qnil; } static VALUE resource_limits_render_score_limit_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); return LONG2NUM(resource_limits->render_score_limit); } static VALUE resource_limits_set_render_score_limit_method(VALUE self, VALUE render_score_limit) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); if (render_score_limit == Qnil) { resource_limits->render_score_limit = LONG_MAX; } else { resource_limits->render_score_limit = NUM2LONG(render_score_limit); } return Qnil; } static VALUE resource_limits_assign_score_limit_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); return LONG2NUM(resource_limits->assign_score_limit); } static VALUE resource_limits_set_assign_score_limit_method(VALUE self, VALUE assign_score_limit) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); if (assign_score_limit == Qnil) { resource_limits->assign_score_limit = LONG_MAX; } else { resource_limits->assign_score_limit = NUM2LONG(assign_score_limit); } return Qnil; } static VALUE resource_limits_render_score_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); return LONG2NUM(resource_limits->render_score); } static VALUE resource_limits_assign_score_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); return LONG2NUM(resource_limits->assign_score); } static VALUE resource_limits_initialize_method(VALUE self, VALUE render_length_limit, VALUE render_score_limit, VALUE assign_score_limit) { resource_limits_set_render_length_limit_method(self, render_length_limit); resource_limits_set_render_score_limit_method(self, render_score_limit); resource_limits_set_assign_score_limit_method(self, assign_score_limit); return Qnil; } __attribute__((noreturn)) void resource_limits_raise_limits_reached(resource_limits_t *resource_limit) { resource_limit->reached_limit = true; rb_raise(cMemoryError, "Memory limits exceeded"); } void resource_limits_increment_render_score(resource_limits_t *resource_limits, long amount) { resource_limits->render_score = resource_limits->render_score + amount; if (resource_limits->render_score > resource_limits->render_score_limit) { resource_limits_raise_limits_reached(resource_limits); } } static VALUE resource_limits_increment_render_score_method(VALUE self, VALUE amount) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); resource_limits_increment_render_score(resource_limits, NUM2LONG(amount)); return Qnil; } static void resource_limits_increment_assign_score(resource_limits_t *resource_limits, long amount) { resource_limits->assign_score = resource_limits->assign_score + amount; if (resource_limits->assign_score > resource_limits->assign_score_limit) { resource_limits_raise_limits_reached(resource_limits); } } static VALUE resource_limits_increment_assign_score_method(VALUE self, VALUE amount) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); resource_limits_increment_assign_score(resource_limits, NUM2LONG(amount)); return Qnil; } void resource_limits_increment_write_score(resource_limits_t *resource_limits, VALUE output) { long captured = RSTRING_LEN(output); if (resource_limits->last_capture_length >= 0) { long increment = captured - resource_limits->last_capture_length; resource_limits->last_capture_length = captured; resource_limits_increment_assign_score(resource_limits, increment); } else if (captured > resource_limits->render_length_limit) { resource_limits_raise_limits_reached(resource_limits); } } static VALUE resource_limits_increment_write_score_method(VALUE self, VALUE output) { Check_Type(output, T_STRING); resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); resource_limits_increment_write_score(resource_limits, output); return Qnil; } static VALUE resource_limits_raise_limits_reached_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); resource_limits_raise_limits_reached(resource_limits); } static VALUE resource_limits_reached_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); return resource_limits->reached_limit ? Qtrue : Qfalse; } struct capture_ensure_t { resource_limits_t *resource_limits; long old_capture_length; }; static VALUE capture_ensure(VALUE data) { struct capture_ensure_t *ensure_data = (struct capture_ensure_t *)data; ensure_data->resource_limits->last_capture_length = ensure_data->old_capture_length; return Qnil; } static VALUE resource_limits_with_capture_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); struct capture_ensure_t ensure_data = { .resource_limits = resource_limits, .old_capture_length = resource_limits->last_capture_length }; resource_limits->last_capture_length = 0; return rb_ensure(rb_yield, Qundef, capture_ensure, (VALUE)&ensure_data); } static VALUE resource_limits_reset_method(VALUE self) { resource_limits_t *resource_limits; ResourceLimits_Get_Struct(self, resource_limits); resource_limits_reset(resource_limits); return Qnil; } void liquid_define_resource_limits(void) { cLiquidResourceLimits = rb_define_class_under(mLiquidC, "ResourceLimits", rb_cObject); rb_global_variable(&cLiquidResourceLimits); rb_define_alloc_func(cLiquidResourceLimits, resource_limits_allocate); rb_define_method(cLiquidResourceLimits, "initialize", resource_limits_initialize_method, 3); rb_define_method(cLiquidResourceLimits, "render_length_limit", resource_limits_render_length_limit_method, 0); rb_define_method(cLiquidResourceLimits, "render_length_limit=", resource_limits_set_render_length_limit_method, 1); rb_define_method(cLiquidResourceLimits, "render_score_limit", resource_limits_render_score_limit_method, 0); rb_define_method(cLiquidResourceLimits, "render_score_limit=", resource_limits_set_render_score_limit_method, 1); rb_define_method(cLiquidResourceLimits, "assign_score_limit", resource_limits_assign_score_limit_method, 0); rb_define_method(cLiquidResourceLimits, "assign_score_limit=", resource_limits_set_assign_score_limit_method, 1); rb_define_method(cLiquidResourceLimits, "render_score", resource_limits_render_score_method, 0); rb_define_method(cLiquidResourceLimits, "assign_score", resource_limits_assign_score_method, 0); rb_define_method(cLiquidResourceLimits, "increment_render_score", resource_limits_increment_render_score_method, 1); rb_define_method(cLiquidResourceLimits, "increment_assign_score", resource_limits_increment_assign_score_method, 1); rb_define_method(cLiquidResourceLimits, "increment_write_score", resource_limits_increment_write_score_method, 1); rb_define_method(cLiquidResourceLimits, "raise_limits_reached", resource_limits_raise_limits_reached_method, 0); rb_define_method(cLiquidResourceLimits, "reached?", resource_limits_reached_method, 0); rb_define_method(cLiquidResourceLimits, "reset", resource_limits_reset_method, 0); rb_define_method(cLiquidResourceLimits, "with_capture", resource_limits_with_capture_method, 0); } liquid-c-4.1.0/ext/liquid_c/resource_limits.h000066400000000000000000000015061417631740700212110ustar00rootroot00000000000000#ifndef LIQUID_RESOURCE_LIMITS #define LIQUID_RESOURCE_LIMITS typedef struct resource_limits { long render_length_limit; long render_score_limit; long assign_score_limit; bool reached_limit; long last_capture_length; long render_score; long assign_score; } resource_limits_t; extern VALUE cLiquidResourceLimits; extern const rb_data_type_t resource_limits_data_type; #define ResourceLimits_Get_Struct(obj, sval) TypedData_Get_Struct(obj, resource_limits_t, &resource_limits_data_type, sval) void liquid_define_resource_limits(void); void resource_limits_raise_limits_reached(resource_limits_t *resource_limit); void resource_limits_increment_render_score(resource_limits_t *resource_limits, long amount); void resource_limits_increment_write_score(resource_limits_t *resource_limits, VALUE output); #endif liquid-c-4.1.0/ext/liquid_c/stringutil.h000066400000000000000000000015471417631740700202120ustar00rootroot00000000000000#if !defined(LIQUID_UTIL_H) #define LIQUID_UTIL_H inline static const char *read_while(const char *start, const char *end, int (func)(int)) { while (start < end && func((unsigned char) *start)) start++; return start; } inline static const char *read_while_reverse(const char *start, const char *end, int (func)(int)) { end--; while (start <= end && func((unsigned char) *end)) end--; end++; return end; } inline static int count_newlines(const char *start, const char *end) { int count = 0; while (start < end) { if (*start == '\n') count++; start++; } return count; } inline static int is_non_newline_space(int c) { return rb_isspace(c) && c != '\n'; } inline static int not_newline(int c) { return c != '\n'; } inline static bool is_word_char(char c) { return ISALNUM(c) || c == '_'; } #endif liquid-c-4.1.0/ext/liquid_c/tokenizer.c000066400000000000000000000222621417631740700200100ustar00rootroot00000000000000#include #include "liquid.h" #include "tokenizer.h" #include "stringutil.h" VALUE cLiquidTokenizer; static void tokenizer_mark(void *ptr) { tokenizer_t *tokenizer = ptr; rb_gc_mark(tokenizer->source); } static void tokenizer_free(void *ptr) { tokenizer_t *tokenizer = ptr; xfree(tokenizer); } static size_t tokenizer_memsize(const void *ptr) { return ptr ? sizeof(tokenizer_t) : 0; } const rb_data_type_t tokenizer_data_type = { "liquid_tokenizer", { tokenizer_mark, tokenizer_free, tokenizer_memsize, }, #if defined(RUBY_TYPED_FREE_IMMEDIATELY) NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY #endif }; static VALUE tokenizer_allocate(VALUE klass) { VALUE obj; tokenizer_t *tokenizer; obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer); tokenizer->source = Qnil; tokenizer->bug_compatible_whitespace_trimming = false; tokenizer->raw_tag_body = NULL; tokenizer->raw_tag_body_len = 0; return obj; } static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE start_line_number, VALUE for_liquid_tag) { tokenizer_t *tokenizer; Check_Type(source, T_STRING); check_utf8_encoding(source, "source"); #define MAX_SOURCE_CODE_BYTES ((1 << 24) - 1) if (RSTRING_LEN(source) > MAX_SOURCE_CODE_BYTES) { rb_enc_raise(utf8_encoding, rb_eArgError, "Source too large, max %d bytes", MAX_SOURCE_CODE_BYTES); } #undef MAX_SOURCE_CODE_BYTES Tokenizer_Get_Struct(self, tokenizer); source = rb_str_dup_frozen(source); tokenizer->source = source; tokenizer->cursor = RSTRING_PTR(source); tokenizer->cursor_end = tokenizer->cursor + RSTRING_LEN(source); tokenizer->lstrip_flag = false; // tokenizer->line_number keeps track of the current line number or it is 0 // to indicate that line numbers aren't being calculated tokenizer->line_number = FIX2UINT(start_line_number); tokenizer->for_liquid_tag = RTEST(for_liquid_tag); return Qnil; } // Internal function to setup an existing tokenizer from C for a liquid tag. // This overwrites the passed in tokenizer, so a copy of the struct should // be used to reset the tokenizer after parsing the liquid tag. void tokenizer_setup_for_liquid_tag(tokenizer_t *tokenizer, const char *cursor, const char *cursor_end, int line_number) { tokenizer->cursor = cursor; tokenizer->cursor_end = cursor_end; tokenizer->lstrip_flag = false; tokenizer->line_number = line_number; tokenizer->for_liquid_tag = true; } // Tokenizes contents of {% liquid ... %} static void tokenizer_next_for_liquid_tag(tokenizer_t *tokenizer, token_t *token) { const char *end = tokenizer->cursor_end; const char *start = tokenizer->cursor; const char *start_trimmed = read_while(start, end, is_non_newline_space); token->str_full = start; token->str_trimmed = start_trimmed; const char *end_full = read_while(start_trimmed, end, not_newline); if (end_full < end) { tokenizer->cursor = end_full + 1; if (tokenizer->line_number) tokenizer->line_number++; } else { tokenizer->cursor = end_full; } const char *end_trimmed = read_while_reverse(start_trimmed, end_full, rb_isspace); token->len_trimmed = end_trimmed - start_trimmed; token->len_full = end_full - token->str_full; if (token->len_trimmed == 0) { token->type = TOKEN_BLANK_LIQUID_TAG_LINE; } else { token->type = TOKEN_TAG; } } // Tokenizes contents of a full Liquid template static void tokenizer_next_for_template(tokenizer_t *tokenizer, token_t *token) { const char *cursor = tokenizer->cursor; const char *last = tokenizer->cursor_end - 1; token->str_full = cursor; token->type = TOKEN_RAW; while (cursor < last) { if (*cursor++ != '{') continue; char c = *cursor++; if (c != '%' && c != '{') continue; if (cursor <= last && *cursor == '-') { cursor++; token->rstrip = 1; } if (cursor - tokenizer->cursor > (ptrdiff_t)(2 + token->rstrip)) { token->type = TOKEN_RAW; cursor -= 2 + token->rstrip; token->lstrip = tokenizer->lstrip_flag; tokenizer->lstrip_flag = false; goto found; } tokenizer->lstrip_flag = false; token->type = TOKEN_INVALID; token->lstrip = token->rstrip; token->rstrip = 0; if (c == '%') { while (cursor < last) { if (*cursor++ != '%') continue; c = *cursor++; while (c == '%' && cursor <= last) c = *cursor++; if (c != '}') continue; token->type = TOKEN_TAG; if(cursor[-3] == '-') token->rstrip = tokenizer->lstrip_flag = true; goto found; } // unterminated tag cursor = tokenizer->cursor + 2; tokenizer->lstrip_flag = false; goto found; } else { while (cursor < last) { if (*cursor++ != '}') continue; if (*cursor++ != '}') { // variable incomplete end, used to end raw tags cursor--; goto found; } token->type = TOKEN_VARIABLE; if(cursor[-3] == '-') token->rstrip = tokenizer->lstrip_flag = true; goto found; } // unterminated variable cursor = tokenizer->cursor + 2; tokenizer->lstrip_flag = false; goto found; } } cursor = last + 1; token->lstrip = tokenizer->lstrip_flag; tokenizer->lstrip_flag = false; found: token->len_full = cursor - token->str_full; token->str_trimmed = token->str_full; token->len_trimmed = token->len_full; if (token->type == TOKEN_VARIABLE || token->type == TOKEN_TAG) { token->str_trimmed += 2 + token->lstrip; token->len_trimmed -= 2 + token->lstrip + 2; if (token->rstrip && token->len_trimmed) token->len_trimmed--; } assert(token->len_trimmed >= 0); tokenizer->cursor += token->len_full; if (tokenizer->line_number) { tokenizer->line_number += count_newlines(token->str_full, token->str_full + token->len_full); } } void tokenizer_next(tokenizer_t *tokenizer, token_t *token) { memset(token, 0, sizeof(*token)); if (tokenizer->cursor >= tokenizer->cursor_end) { return; } if (tokenizer->for_liquid_tag) { tokenizer_next_for_liquid_tag(tokenizer, token); } else { tokenizer_next_for_template(tokenizer, token); } } static VALUE tokenizer_shift_method(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); token_t token; tokenizer_next(tokenizer, &token); if (!token.type) return Qnil; // When sent back to Ruby, tokens are the raw string including whitespace // and tag delimiters. It should be possible to reconstruct the exact // template from the tokens. return rb_enc_str_new(token.str_full, token.len_full, utf8_encoding); } static VALUE tokenizer_shift_trimmed_method(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); token_t token; tokenizer_next(tokenizer, &token); if (!token.type) return Qnil; // This method doesn't include whitespace and tag delimiters. It allows for // testing the output of tokenizer_next as used by rb_block_parse. return rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding); } static VALUE tokenizer_line_number_method(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); if (tokenizer->line_number == 0) return Qnil; return UINT2NUM(tokenizer->line_number); } static VALUE tokenizer_for_liquid_tag_method(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); return tokenizer->for_liquid_tag ? Qtrue : Qfalse; } // Temporary to test rollout of the fix for this bug static VALUE tokenizer_bug_compatible_whitespace_trimming(VALUE self) { tokenizer_t *tokenizer; Tokenizer_Get_Struct(self, tokenizer); tokenizer->bug_compatible_whitespace_trimming = true; return Qnil; } void liquid_define_tokenizer(void) { cLiquidTokenizer = rb_define_class_under(mLiquidC, "Tokenizer", rb_cObject); rb_global_variable(&cLiquidTokenizer); rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate); rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 3); rb_define_method(cLiquidTokenizer, "line_number", tokenizer_line_number_method, 0); rb_define_method(cLiquidTokenizer, "for_liquid_tag", tokenizer_for_liquid_tag_method, 0); rb_define_method(cLiquidTokenizer, "bug_compatible_whitespace_trimming!", tokenizer_bug_compatible_whitespace_trimming, 0); // For testing the internal token representation. rb_define_private_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0); rb_define_private_method(cLiquidTokenizer, "shift_trimmed", tokenizer_shift_trimmed_method, 0); } liquid-c-4.1.0/ext/liquid_c/tokenizer.h000066400000000000000000000022271417631740700200140ustar00rootroot00000000000000#if !defined(LIQUID_TOKENIZER_H) #define LIQUID_TOKENIZER_H enum token_type { TOKENIZER_TOKEN_NONE = 0, TOKEN_INVALID, TOKEN_RAW, TOKEN_TAG, TOKEN_VARIABLE, TOKEN_BLANK_LIQUID_TAG_LINE }; typedef struct token { enum token_type type; // str_trimmed contains no tag delimiters const char *str_trimmed, *str_full; long len_trimmed, len_full; bool lstrip, rstrip; } token_t; typedef struct tokenizer { VALUE source; const char *cursor, *cursor_end; unsigned int line_number; bool lstrip_flag; bool for_liquid_tag; // Temporary to test rollout of the fix for this bug bool bug_compatible_whitespace_trimming; char *raw_tag_body; unsigned int raw_tag_body_len; } tokenizer_t; extern VALUE cLiquidTokenizer; extern const rb_data_type_t tokenizer_data_type; #define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval) void liquid_define_tokenizer(void); void tokenizer_next(tokenizer_t *tokenizer, token_t *token); void tokenizer_setup_for_liquid_tag(tokenizer_t *tokenizer, const char *cursor, const char *cursor_end, int line_number); #endif liquid-c-4.1.0/ext/liquid_c/usage.c000066400000000000000000000006151417631740700171000ustar00rootroot00000000000000#include "usage.h" static VALUE cLiquidUsage; static ID id_increment; void usage_increment(const char *name) { VALUE name_str = rb_str_new_cstr(name); rb_funcall(cLiquidUsage, id_increment, 1, name_str); } void liquid_define_usage(void) { cLiquidUsage = rb_const_get(mLiquid, rb_intern("Usage")); rb_global_variable(&cLiquidUsage); id_increment = rb_intern("increment"); } liquid-c-4.1.0/ext/liquid_c/usage.h000066400000000000000000000002241417631740700171010ustar00rootroot00000000000000#ifndef LIQUID_USAGE_H #define LIQUID_USAGE_H #include "liquid.h" void liquid_define_usage(void); void usage_increment(const char *name); #endif liquid-c-4.1.0/ext/liquid_c/variable.c000066400000000000000000000176041417631740700175670ustar00rootroot00000000000000#include "liquid.h" #include "variable.h" #include "parser.h" #include "expression.h" #include "vm.h" #include static ID id_rescue_strict_parse_syntax_error; static ID id_ivar_parse_context; static ID id_ivar_name; static ID id_ivar_filters; VALUE cLiquidCVariableExpression; static VALUE frozen_empty_array; static VALUE try_variable_strict_parse(VALUE uncast_args) { variable_parse_args_t *parse_args = (void *)uncast_args; parser_t p; init_parser(&p, parse_args->markup, parse_args->markup_end); vm_assembler_t *code = parse_args->code; if (p.cur.type == TOKEN_EOS) { vm_assembler_add_push_nil(code); return Qnil; } parse_and_compile_expression(&p, code); while (parser_consume(&p, TOKEN_PIPE).type) { lexer_token_t filter_name_token = parser_must_consume(&p, TOKEN_IDENTIFIER); VALUE filter_name = token_to_rsym(filter_name_token); size_t arg_count = 0; size_t keyword_arg_count = 0; VALUE push_keywords_obj = Qnil; vm_assembler_t *push_keywords_code = NULL; if (parser_consume(&p, TOKEN_COLON).type) { do { if (p.cur.type == TOKEN_IDENTIFIER && p.next.type == TOKEN_COLON) { VALUE key = token_to_rstr(parser_consume_any(&p)); parser_consume_any(&p); keyword_arg_count++; if (push_keywords_obj == Qnil) { expression_t *push_keywords_expr; // use an object to automatically free on an exception push_keywords_obj = expression_new(cLiquidCExpression, &push_keywords_expr); rb_obj_hide(push_keywords_obj); push_keywords_code = &push_keywords_expr->code; } vm_assembler_add_push_const(push_keywords_code, key); parse_and_compile_expression(&p, push_keywords_code); } else { parse_and_compile_expression(&p, code); arg_count++; } } while (parser_consume(&p, TOKEN_COMMA).type); } if (keyword_arg_count) { arg_count++; if (keyword_arg_count > 255) rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Too many filter keyword arguments"); vm_assembler_concat(code, push_keywords_code); vm_assembler_add_hash_new(code, keyword_arg_count); RB_GC_GUARD(push_keywords_obj); } vm_assembler_add_filter(code, filter_name, arg_count); } parser_must_consume(&p, TOKEN_EOS); return Qnil; } typedef struct variable_strict_parse_rescue { variable_parse_args_t *parse_args; size_t instructions_size; size_t constants_size; size_t stack_size; } variable_strict_parse_rescue_t; static VALUE variable_strict_parse_rescue(VALUE uncast_args, VALUE exception) { variable_strict_parse_rescue_t *rescue_args = (void *)uncast_args; variable_parse_args_t *parse_args = rescue_args->parse_args; vm_assembler_t *code = parse_args->code; // undo partial strict parse uint8_t *last_constants_data_end = (uint8_t *)code->constants.data + rescue_args->constants_size; VALUE *const_ptr = (VALUE *)last_constants_data_end; st_table *constants_table = code->constants_table; while((uint8_t *)const_ptr < code->constants.data_end) { st_data_t key = (st_data_t)const_ptr[0]; st_delete(constants_table, &key, 0); const_ptr++; } code->instructions.data_end = code->instructions.data + rescue_args->instructions_size; code->constants.data_end = last_constants_data_end; code->stack_size = rescue_args->stack_size; if (rb_obj_is_kind_of(exception, cLiquidSyntaxError) == Qfalse) rb_exc_raise(exception); VALUE markup_obj = rb_enc_str_new(parse_args->markup, parse_args->markup_end - parse_args->markup, utf8_encoding); VALUE variable_obj = rb_funcall( cLiquidVariable, id_rescue_strict_parse_syntax_error, 3, exception, markup_obj, parse_args->parse_context ); // lax parse code->protected_stack_size = code->stack_size; rb_funcall(variable_obj, id_compile_evaluate, 1, parse_args->code_obj); if (code->stack_size != code->protected_stack_size + 1) { rb_raise(rb_eRuntimeError, "Liquid::Variable#compile_evaluate didn't leave exactly 1 new element on the stack"); } return Qnil; } void internal_variable_compile_evaluate(variable_parse_args_t *parse_args) { vm_assembler_t *code = parse_args->code; variable_strict_parse_rescue_t rescue_args = { .parse_args = parse_args, .instructions_size = c_buffer_size(&code->instructions), .constants_size = c_buffer_size(&code->constants), .stack_size = code->stack_size, }; rb_rescue(try_variable_strict_parse, (VALUE)parse_args, variable_strict_parse_rescue, (VALUE)&rescue_args); } void internal_variable_compile(variable_parse_args_t *parse_args, unsigned int line_number) { vm_assembler_t *code = parse_args->code; vm_assembler_add_render_variable_rescue(code, line_number); internal_variable_compile_evaluate(parse_args); vm_assembler_add_pop_write(code); } static VALUE variable_strict_parse_method(VALUE self, VALUE markup) { StringValue(markup); check_utf8_encoding(markup, "markup"); VALUE parse_context = rb_ivar_get(self, id_ivar_parse_context); expression_t *expression; VALUE expression_obj = expression_new(cLiquidCVariableExpression, &expression); variable_parse_args_t parse_args = { .markup = RSTRING_PTR(markup), .markup_end = RSTRING_END(markup), .code = &expression->code, .code_obj = expression_obj, .parse_context = parse_context, }; try_variable_strict_parse((VALUE)&parse_args); RB_GC_GUARD(markup); assert(expression->code.stack_size == 1); vm_assembler_add_leave(&expression->code); rb_ivar_set(self, id_ivar_name, expression_obj); rb_ivar_set(self, id_ivar_filters, frozen_empty_array); return Qnil; } typedef struct { expression_t *expression; VALUE context; } variable_expression_evaluate_args_t; static VALUE try_variable_expression_evaluate(VALUE uncast_args) { variable_expression_evaluate_args_t *args = (void *)uncast_args; return liquid_vm_evaluate(args->context, &args->expression->code); } static VALUE rescue_variable_expression_evaluate(VALUE uncast_args, VALUE exception) { variable_expression_evaluate_args_t *args = (void *)uncast_args; vm_t *vm = vm_from_context(args->context); exception = vm_translate_if_filter_argument_error(vm, exception); rb_exc_raise(exception); } VALUE internal_variable_expression_evaluate(expression_t *expression, VALUE context) { variable_expression_evaluate_args_t args = { expression, context }; return rb_rescue(try_variable_expression_evaluate, (VALUE)&args, rescue_variable_expression_evaluate, (VALUE)&args); } static VALUE variable_expression_evaluate_method(VALUE self, VALUE context) { expression_t *expression; Expression_Get_Struct(self, expression); return internal_variable_expression_evaluate(expression, context); } void liquid_define_variable(void) { id_rescue_strict_parse_syntax_error = rb_intern("rescue_strict_parse_syntax_error"); id_ivar_parse_context = rb_intern("@parse_context"); id_ivar_name = rb_intern("@name"); id_ivar_filters = rb_intern("@filters"); frozen_empty_array = rb_ary_new(); rb_ary_freeze(frozen_empty_array); rb_global_variable(&frozen_empty_array); rb_define_method(cLiquidVariable, "c_strict_parse", variable_strict_parse_method, 1); cLiquidCVariableExpression = rb_define_class_under(mLiquidC, "VariableExpression", cLiquidCExpression); rb_global_variable(&cLiquidCVariableExpression); rb_define_method(cLiquidCVariableExpression, "evaluate", variable_expression_evaluate_method, 1); } liquid-c-4.1.0/ext/liquid_c/variable.h000066400000000000000000000012201417631740700175570ustar00rootroot00000000000000#if !defined(LIQUID_VARIABLE_H) #define LIQUID_VARIABLE_H #include "vm_assembler.h" #include "block.h" #include "expression.h" extern VALUE cLiquidCVariableExpression; typedef struct variable_parse_args { const char *markup; const char *markup_end; vm_assembler_t *code; VALUE code_obj; VALUE parse_context; } variable_parse_args_t; void liquid_define_variable(void); void internal_variable_compile(variable_parse_args_t *parse_args, unsigned int line_number); void internal_variable_compile_evaluate(variable_parse_args_t *parse_args); VALUE internal_variable_expression_evaluate(expression_t *expression, VALUE context); #endif liquid-c-4.1.0/ext/liquid_c/variable_lookup.c000066400000000000000000000026221417631740700211520ustar00rootroot00000000000000#include "liquid.h" #include "context.h" static ID id_has_key, id_aref, id_fetch, id_to_liquid_value; VALUE variable_lookup_key(VALUE context, VALUE object, VALUE key, bool is_command) { if (rb_obj_class(key) != rb_cString) { VALUE key_value = rb_check_funcall(key, id_to_liquid_value, 0, 0); if (key_value != Qundef) { key = key_value; } } if (rb_respond_to(object, id_aref) && ( (rb_respond_to(object, id_has_key) && rb_funcall(object, id_has_key, 1, key)) || (rb_obj_is_kind_of(key, rb_cInteger) && rb_respond_to(object, id_fetch)) )) { VALUE next_object = rb_funcall(object, id_aref, 1, key); next_object = materialize_proc(context, object, key, next_object); return value_to_liquid_and_set_context(next_object, context); } if (is_command) { Check_Type(key, T_STRING); ID intern_key = rb_intern(RSTRING_PTR(key)); if (rb_respond_to(object, intern_key)) { VALUE next_object = rb_funcall(object, intern_key, 0); return value_to_liquid_and_set_context(next_object, context); } } context_maybe_raise_undefined_variable(context, key); return Qnil; } void liquid_define_variable_lookup(void) { id_has_key = rb_intern("key?"); id_aref = rb_intern("[]"); id_fetch = rb_intern("fetch"); id_to_liquid_value = rb_intern("to_liquid_value"); } liquid-c-4.1.0/ext/liquid_c/variable_lookup.h000066400000000000000000000003201417631740700211500ustar00rootroot00000000000000#if !defined(LIQUID_VARIABLE_LOOKUP_H) #define LIQUID_VARIABLE_LOOKUP_H void liquid_define_variable_lookup(void); VALUE variable_lookup_key(VALUE context, VALUE object, VALUE key, bool is_command); #endif liquid-c-4.1.0/ext/liquid_c/vm.c000066400000000000000000000436061417631740700164250ustar00rootroot00000000000000#include #include #include "liquid.h" #include "vm.h" #include "variable_lookup.h" #include "intutil.h" #include "document_body.h" ID id_render_node; ID id_vm; static VALUE cLiquidCVM; static void vm_mark(void *ptr) { vm_t *vm = ptr; c_buffer_rb_gc_mark(&vm->stack); context_mark(&vm->context); } static void vm_free(void *ptr) { vm_t *vm = ptr; c_buffer_free(&vm->stack); xfree(vm); } static size_t vm_memsize(const void *ptr) { const vm_t *vm = ptr; return sizeof(vm_t) + c_buffer_capacity(&vm->stack); } const rb_data_type_t vm_data_type = { "liquid_vm", { vm_mark, vm_free, vm_memsize, }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; static VALUE vm_internal_new(VALUE context) { vm_t *vm; VALUE obj = TypedData_Make_Struct(cLiquidCVM, vm_t, &vm_data_type, vm); vm->stack = c_buffer_init(); vm->invoking_filter = false; context_internal_init(context, &vm->context); return obj; } vm_t *vm_from_context(VALUE context) { VALUE vm_obj = rb_attr_get(context, id_vm); if (vm_obj == Qnil) { vm_obj = vm_internal_new(context); rb_ivar_set(context, id_vm, vm_obj); } // instance variable is hidden from ruby so should be safe to unwrap it without type checking return DATA_PTR(vm_obj); } bool liquid_vm_filtering(VALUE context) { VALUE vm_obj = rb_attr_get(context, id_vm); if (vm_obj == Qnil) return false; vm_t *vm = DATA_PTR(vm_obj); return vm->invoking_filter; } static void write_fixnum(VALUE output, VALUE fixnum) { long long number = RB_NUM2LL(fixnum); int write_length = snprintf(NULL, 0, "%lld", number); long old_size = RSTRING_LEN(output); long new_size = old_size + write_length; long capacity = rb_str_capacity(output); if (new_size > capacity) { do { capacity *= 2; } while (new_size > capacity); rb_str_resize(output, capacity); } rb_str_set_len(output, new_size); snprintf(RSTRING_PTR(output) + old_size, write_length + 1, "%lld", number); } static VALUE obj_to_s(VALUE obj) { VALUE str = rb_funcall(obj, id_to_s, 0); if (RB_LIKELY(RB_TYPE_P(str, T_STRING))) return str; rb_raise(rb_eTypeError, "%"PRIsVALUE"#to_s returned a non-String convertible value of type %"PRIsVALUE, rb_obj_class(obj), rb_obj_class(str)); } static void write_obj(VALUE output, VALUE obj) { switch (TYPE(obj)) { default: obj = obj_to_s(obj); // fallthrough case T_STRING: rb_str_buf_append(output, obj); break; case T_FIXNUM: write_fixnum(output, obj); break; case T_ARRAY: for (long i = 0; i < RARRAY_LEN(obj); i++) { VALUE item = RARRAY_AREF(obj, i); if (RB_UNLIKELY(RB_TYPE_P(item, T_ARRAY))) { // Normally liquid arrays are flat, but for safety and simplicity we // leverage ruby's join that detects and raises on a recursion loop rb_str_buf_append(output, rb_ary_join(item, Qnil)); } else { write_obj(output, item); } } break; case T_NIL: break; } } static inline void vm_stack_push(vm_t *vm, VALUE value) { VALUE *stack_ptr = (VALUE *)vm->stack.data_end; assert(stack_ptr < (VALUE *)vm->stack.capacity_end); *stack_ptr++ = value; vm->stack.data_end = (uint8_t *)stack_ptr; } static inline VALUE vm_stack_pop(vm_t *vm) { VALUE *stack_ptr = (VALUE *)vm->stack.data_end; stack_ptr--; assert((VALUE *)vm->stack.data <= stack_ptr); vm->stack.data_end = (uint8_t *)stack_ptr; return *stack_ptr; } static inline VALUE *vm_stack_pop_n_use_in_place(vm_t *vm, size_t n) { VALUE *stack_ptr = (VALUE *)vm->stack.data_end; stack_ptr -= n; assert((VALUE *)vm->stack.data <= stack_ptr); vm->stack.data_end = (uint8_t *)stack_ptr; return stack_ptr; } static inline void vm_stack_reserve_for_write(vm_t *vm, size_t num_values) { c_buffer_reserve_for_write(&vm->stack, num_values * sizeof(VALUE)); } static VALUE vm_invoke_filter(vm_t *vm, VALUE filter_name, int num_args, VALUE *args) { bool not_invokable = rb_hash_lookup(vm->context.filter_methods, filter_name) != Qtrue; if (RB_UNLIKELY(not_invokable)) { if (vm->context.strict_filters) { VALUE error_class = rb_const_get(mLiquid, rb_intern("UndefinedFilter")); rb_raise(error_class, "undefined filter %"PRIsVALUE, rb_sym2str(filter_name)); } return args[0]; } vm->invoking_filter = true; VALUE result = rb_funcallv(vm->context.strainer, RB_SYM2ID(filter_name), num_args, args); vm->invoking_filter = false; return rb_funcall(result, id_to_liquid, 0); } typedef struct vm_render_until_error_args { vm_t *vm; const uint8_t *ip; // use for initial address and to save an address for rescuing const size_t *const_ptr; /* rendering fields */ VALUE output; const uint8_t *node_line_number; } vm_render_until_error_args_t; static VALUE raise_invalid_integer(VALUE unused_arg, VALUE exc) { rb_raise(cLiquidArgumentError, "invalid integer"); } // Equivalent to Integer(string) if string is an instance of String static VALUE try_string_to_integer(VALUE string) { return rb_str_to_inum(string, 0, true); } static VALUE range_value_to_integer(VALUE value) { if (RB_INTEGER_TYPE_P(value)) { return value; } else if (value == Qnil) { return INT2FIX(0); } else if (RB_TYPE_P(value, T_STRING)) { return rb_str_to_inum(value, 0, false); // equivalent to String#to_i } else { value = obj_to_s(value); return rb_rescue2(try_string_to_integer, value, raise_invalid_integer, Qnil, rb_eArgError, (VALUE)0); } } #ifdef HAVE_RB_HASH_BULK_INSERT #define hash_bulk_insert rb_hash_bulk_insert #else static void hash_bulk_insert(long argc, const VALUE *argv, VALUE hash) { for (long i = 0; i < argc; i += 2) { rb_hash_aset(hash, argv[i], argv[i + 1]); } } #endif // Actually returns a bool resume_rendering value static VALUE vm_render_until_error(VALUE uncast_args) { vm_render_until_error_args_t *args = (void *)uncast_args; const VALUE *constants = args->const_ptr; const uint8_t *ip = args->ip; vm_t *vm = args->vm; VALUE output = args->output; uint16_t constant_index; VALUE constant = Qnil; args->ip = NULL; // used by vm_render_rescue, NULL to indicate that it isn't in a rescue block while (true) { switch (*ip++) { case OP_LEAVE: return false; case OP_PUSH_NIL: vm_stack_push(vm, Qnil); break; case OP_PUSH_TRUE: vm_stack_push(vm, Qtrue); break; case OP_PUSH_FALSE: vm_stack_push(vm, Qfalse); break; case OP_PUSH_INT8: { int num = *(int8_t *)ip++; // signed vm_stack_push(vm, RB_INT2FIX(num)); break; } case OP_PUSH_INT16: { int num = *(int8_t *)ip++; // big endian encoding, so first byte has sign num = (num << 8) | *ip++; vm_stack_push(vm, RB_INT2FIX(num)); break; } case OP_FIND_STATIC_VAR: { constant_index = (ip[0] << 8) | ip[1]; constant = constants[constant_index]; ip += 2; VALUE value = context_find_variable(&vm->context, constant, Qtrue); vm_stack_push(vm, value); break; } case OP_FIND_VAR: { VALUE key = vm_stack_pop(vm); VALUE value = context_find_variable(&vm->context, key, Qtrue); vm_stack_push(vm, value); break; } case OP_LOOKUP_CONST_KEY: case OP_LOOKUP_COMMAND: { constant_index = (ip[0] << 8) | ip[1]; constant = constants[constant_index]; ip += 2; vm_stack_push(vm, constant); } /* fallthrough */ case OP_LOOKUP_KEY: { bool is_command = ip[-3] == OP_LOOKUP_COMMAND; VALUE key = vm_stack_pop(vm); VALUE object = vm_stack_pop(vm); VALUE result = variable_lookup_key(vm->context.self, object, key, is_command); vm_stack_push(vm, result); break; } case OP_NEW_INT_RANGE: { VALUE end = range_value_to_integer(vm_stack_pop(vm)); VALUE begin = range_value_to_integer(vm_stack_pop(vm)); bool exclude_end = false; vm_stack_push(vm, rb_range_new(begin, end, exclude_end)); break; } case OP_HASH_NEW: { size_t hash_size = *ip++; size_t num_keys_and_values = hash_size * 2; VALUE hash = rb_hash_new(); VALUE *args_ptr = vm_stack_pop_n_use_in_place(vm, num_keys_and_values); hash_bulk_insert(num_keys_and_values, args_ptr, hash); vm_stack_push(vm, hash); break; } case OP_FILTER: case OP_BUILTIN_FILTER: { VALUE filter_name; uint8_t num_args; if (ip[-1] == OP_FILTER) { constant_index = (ip[0] << 8) | ip[1]; constant = constants[constant_index]; filter_name = RARRAY_AREF(constant, 0); num_args = RARRAY_AREF(constant, 1); ip += 2; } else { assert(ip[-1] == OP_BUILTIN_FILTER); filter_name = builtin_filters[*ip++].sym; num_args = *ip++; // includes input argument } VALUE *args_ptr = vm_stack_pop_n_use_in_place(vm, num_args); VALUE result = vm_invoke_filter(vm, filter_name, num_args, args_ptr); vm_stack_push(vm, result); break; } // Rendering instructions case OP_WRITE_RAW_W: case OP_WRITE_RAW: { const char *text; size_t size; if (ip[-1] == OP_WRITE_RAW_W) { size = bytes_to_uint24(ip); text = (const char *)&ip[3]; ip += 3 + size; } else { size = *ip; text = (const char *)&ip[1]; ip += 1 + size; } rb_str_cat(output, text, size); resource_limits_increment_write_score(vm->context.resource_limits, output); break; } case OP_JUMP_FWD_W: { size_t size = bytes_to_uint24(ip); ip += 3 + size; break; } case OP_JUMP_FWD: { uint8_t size = *ip; ip += 1 + size; break; } case OP_PUSH_CONST: { constant_index = (ip[0] << 8) | ip[1]; constant = constants[constant_index]; ip += 2; vm_stack_push(vm, constant); break; } case OP_WRITE_NODE: { constant_index = (ip[0] << 8) | ip[1]; constant = constants[constant_index]; ip += 2; rb_funcall(cLiquidBlockBody, id_render_node, 3, vm->context.self, output, constant); if (RARRAY_LEN(vm->context.interrupts)) { return false; } resource_limits_increment_write_score(vm->context.resource_limits, output); break; } case OP_RENDER_VARIABLE_RESCUE: // Save state used by vm_render_rescue to rescue from a variable rendering exception args->node_line_number = ip; // vm_render_rescue will iterate from this instruction to the instruction // following OP_POP_WRITE_VARIABLE to resume rendering from ip += 3; args->ip = ip; break; case OP_POP_WRITE: { VALUE var_result = vm_stack_pop(vm); if (vm->context.global_filter != Qnil) var_result = rb_funcall(vm->context.global_filter, id_call, 1, var_result); write_obj(output, var_result); args->ip = NULL; // mark the end of a rescue block, used by vm_render_rescue resource_limits_increment_write_score(vm->context.resource_limits, output); break; } default: rb_bug("invalid opcode: %u", ip[-1]); } } } // Evaluate instructions that avoid using rendering instructions and leave with the result on // the top of the stack VALUE liquid_vm_evaluate(VALUE context, vm_assembler_t *code) { vm_t *vm = vm_from_context(context); vm_stack_reserve_for_write(vm, code->max_stack_size); #ifndef NDEBUG size_t old_stack_byte_size = c_buffer_size(&vm->stack); #endif vm_render_until_error_args_t args = { .vm = vm, .const_ptr = (const size_t *)code->constants.data, .ip = code->instructions.data }; vm_render_until_error((VALUE)&args); VALUE ret = vm_stack_pop(vm); assert(old_stack_byte_size == c_buffer_size(&vm->stack)); return ret; } void liquid_vm_next_instruction(const uint8_t **ip_ptr) { const uint8_t *ip = *ip_ptr; switch (*ip++) { case OP_LEAVE: case OP_POP_WRITE: case OP_PUSH_NIL: case OP_PUSH_TRUE: case OP_PUSH_FALSE: case OP_FIND_VAR: case OP_LOOKUP_KEY: case OP_NEW_INT_RANGE: break; case OP_HASH_NEW: case OP_PUSH_INT8: ip++; break; case OP_BUILTIN_FILTER: case OP_PUSH_INT16: case OP_PUSH_CONST: case OP_WRITE_NODE: case OP_FIND_STATIC_VAR: case OP_LOOKUP_CONST_KEY: case OP_LOOKUP_COMMAND: case OP_FILTER: ip += 2; break; case OP_RENDER_VARIABLE_RESCUE: ip += 3; break; case OP_WRITE_RAW_W: case OP_JUMP_FWD_W: { size_t size = bytes_to_uint24(ip); ip += 3 + size; break; } case OP_WRITE_RAW: case OP_JUMP_FWD: { uint8_t size = *ip; ip += 1 + size; break; } default: rb_bug("invalid opcode: %u", ip[-1]); } *ip_ptr = ip; } VALUE vm_translate_if_filter_argument_error(vm_t *vm, VALUE exception) { if (vm->invoking_filter) { if (rb_obj_is_kind_of(exception, rb_eArgError)) { VALUE cLiquidStrainerTemplate = rb_const_get(mLiquid, rb_intern("StrainerTemplate")); exception = rb_funcall(cLiquidStrainerTemplate, rb_intern("arg_exc_to_liquid_exc"), 1, exception); } vm->invoking_filter = false; } return exception; } typedef struct vm_render_rescue_args { vm_render_until_error_args_t *render_args; size_t old_stack_byte_size; } vm_render_rescue_args_t; // Actually returns a bool resume_rendering value static VALUE vm_render_rescue(VALUE uncast_args, VALUE exception) { vm_render_rescue_args_t *args = (void *)uncast_args; VALUE blank_tag = Qfalse; // tags are still rendered using Liquid::BlockBody.render_node vm_render_until_error_args_t *render_args = args->render_args; vm_t *vm = render_args->vm; exception = vm_translate_if_filter_argument_error(vm, exception); const uint8_t *ip = render_args->ip; if (!ip) rb_exc_raise(exception); // rescue for variable render, where ip is at the start of the render and we need to // skip to the end of the variable render to resume rendering if the error is handled enum opcode last_op; do { last_op = *ip; liquid_vm_next_instruction(&ip); } while (last_op != OP_POP_WRITE); render_args->ip = ip; // remove temporary stack values from variable evaluation vm->stack.data_end = vm->stack.data + args->old_stack_byte_size; assert(render_args->node_line_number); unsigned int node_line_number = bytes_to_uint24(render_args->node_line_number); VALUE line_number = node_line_number != 0 ? UINT2NUM(node_line_number) : Qnil; rb_funcall(cLiquidBlockBody, rb_intern("c_rescue_render_node"), 5, vm->context.self, render_args->output, line_number, exception, blank_tag); return true; } void liquid_vm_render(block_body_header_t *body, const VALUE *const_ptr, VALUE context, VALUE output) { vm_t *vm = vm_from_context(context); vm_stack_reserve_for_write(vm, body->max_stack_size); resource_limits_increment_render_score(vm->context.resource_limits, body->render_score); vm_render_until_error_args_t render_args = { .vm = vm, .const_ptr = const_ptr, .ip = block_body_instructions_ptr(body), .output = output, }; vm_render_rescue_args_t rescue_args = { .render_args = &render_args, .old_stack_byte_size = c_buffer_size(&vm->stack), }; while (rb_rescue(vm_render_until_error, (VALUE)&render_args, vm_render_rescue, (VALUE)&rescue_args)) { } assert(rescue_args.old_stack_byte_size == c_buffer_size(&vm->stack)); } void liquid_define_vm(void) { id_render_node = rb_intern("render_node"); id_vm = rb_intern("vm"); cLiquidCVM = rb_define_class_under(mLiquidC, "VM", rb_cObject); rb_undef_alloc_func(cLiquidCVM); rb_global_variable(&cLiquidCVM); } liquid-c-4.1.0/ext/liquid_c/vm.h000066400000000000000000000012271417631740700164230ustar00rootroot00000000000000#ifndef VM_H #define VM_H #include #include "block.h" #include "vm_assembler.h" #include "context.h" typedef struct vm { c_buffer_t stack; bool invoking_filter; context_t context; } vm_t; void liquid_define_vm(void); vm_t *vm_from_context(VALUE context); void liquid_vm_render(block_body_header_t *block, const VALUE *const_ptr, VALUE context, VALUE output); void liquid_vm_next_instruction(const uint8_t **ip_ptr); bool liquid_vm_filtering(VALUE context); VALUE liquid_vm_evaluate(VALUE context, vm_assembler_t *code); vm_t *vm_from_context(VALUE context); VALUE vm_translate_if_filter_argument_error(vm_t *vm, VALUE exception); #endif liquid-c-4.1.0/ext/liquid_c/vm_assembler.c000066400000000000000000000353771417631740700204700ustar00rootroot00000000000000#include "liquid.h" #include "vm_assembler.h" #include "expression.h" #include "vm.h" #define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) static st_table *builtin_filter_table; // methods from Liquid::StandardFilters filter_desc_t builtin_filters[] = { { .name = "size" }, { .name = "downcase" }, { .name = "upcase" }, { .name = "capitalize" }, { .name = "h" }, { .name = "escape" }, { .name = "escape_once" }, { .name = "url_encode" }, { .name = "url_decode" }, { .name = "slice" }, { .name = "truncate" }, { .name = "truncatewords" }, { .name = "split" }, { .name = "strip" }, { .name = "lstrip" }, { .name = "rstrip" }, { .name = "strip_html" }, { .name = "strip_newlines" }, { .name = "join" }, { .name = "sort" }, { .name = "sort_natural" }, { .name = "where" }, { .name = "uniq" }, { .name = "reverse" }, { .name = "map" }, { .name = "compact" }, { .name = "replace" }, { .name = "replace_first" }, { .name = "remove" }, { .name = "remove_first" }, { .name = "append" }, { .name = "concat" }, { .name = "prepend" }, { .name = "newline_to_br" }, { .name = "date" }, { .name = "first" }, { .name = "last" }, { .name = "abs" }, { .name = "plus" }, { .name = "minus" }, { .name = "times" }, { .name = "divided_by" }, { .name = "modulo" }, { .name = "round" }, { .name = "ceil" }, { .name = "floor" }, { .name = "at_least" }, { .name = "at_most" }, { .name = "default" }, }; static_assert(ARRAY_LENGTH(builtin_filters) < 256, "support for larger than byte sized indexing of filters has not yet been implemented"); static void vm_assembler_common_init(vm_assembler_t *code) { code->max_stack_size = 0; code->stack_size = 0; code->protected_stack_size = 0; code->parsing = true; } void vm_assembler_init(vm_assembler_t *code) { code->instructions = c_buffer_allocate(8); code->constants = c_buffer_allocate(8 * sizeof(VALUE)); code->constants_table = st_init_numtable(); vm_assembler_common_init(code); } void vm_assembler_reset(vm_assembler_t *code) { c_buffer_reset(&code->instructions); c_buffer_reset(&code->constants); st_clear(code->constants_table); vm_assembler_common_init(code); } void vm_assembler_free(vm_assembler_t *code) { c_buffer_free(&code->instructions); c_buffer_free(&code->constants); st_free_table(code->constants_table); } void vm_assembler_gc_mark(vm_assembler_t *code) { c_buffer_rb_gc_mark(&code->constants); } VALUE vm_assembler_disassemble(const uint8_t *start_ip, const uint8_t *end_ip, const VALUE *constants) { const uint8_t *ip = start_ip; VALUE output = rb_str_buf_new(32); VALUE constant = Qnil; while (ip < end_ip) { rb_str_catf(output, "0x%04lx: ", ip - start_ip); if (vm_assembler_opcode_has_constant(*ip)) { uint16_t constant_index = (ip[1] << 8) | ip[2]; constant = RARRAY_AREF(*constants, constant_index); } switch (*ip) { case OP_LEAVE: rb_str_catf(output, "leave\n"); break; case OP_POP_WRITE: rb_str_catf(output, "pop_write\n"); break; case OP_PUSH_NIL: rb_str_catf(output, "push_nil\n"); break; case OP_PUSH_TRUE: rb_str_catf(output, "push_true\n"); break; case OP_PUSH_FALSE: rb_str_catf(output, "push_false\n"); break; case OP_FIND_VAR: rb_str_catf(output, "find_var\n"); break; case OP_LOOKUP_KEY: rb_str_catf(output, "lookup_key\n"); break; case OP_NEW_INT_RANGE: rb_str_catf(output, "new_int_range\n"); break; case OP_HASH_NEW: rb_str_catf(output, "hash_new(%u)\n", ip[1]); break; case OP_PUSH_INT8: rb_str_catf(output, "push_int8(%u)\n", ip[1]); break; case OP_PUSH_INT16: { int num = (ip[1] << 8) | ip[2]; rb_str_catf(output, "push_int16(%u)\n", num); break; } case OP_RENDER_VARIABLE_RESCUE: { unsigned int line_number = bytes_to_uint24(ip + 1); rb_str_catf(output, "render_variable_rescue(line_number: %u)\n", line_number); break; } case OP_WRITE_RAW_W: case OP_WRITE_RAW: { const char *text; size_t size; const char *name; if (*ip == OP_WRITE_RAW_W) { name = "write_raw_w"; size = bytes_to_uint24(&ip[1]); text = (const char *)&ip[4]; } else { name = "write_raw"; size = ip[1]; text = (const char *)&ip[2]; } VALUE string = rb_enc_str_new(text, size, utf8_encoding); rb_str_catf(output, "%s(%+"PRIsVALUE")\n", name, string); break; } case OP_WRITE_NODE: rb_str_catf(output, "write_node(%+"PRIsVALUE")\n", constant); break; case OP_PUSH_CONST: rb_str_catf(output, "push_const(%+"PRIsVALUE")\n", constant); break; case OP_FIND_STATIC_VAR: rb_str_catf(output, "find_static_var(%+"PRIsVALUE")\n", constant); break; case OP_LOOKUP_CONST_KEY: rb_str_catf(output, "lookup_const_key(%+"PRIsVALUE")\n", constant); break; case OP_LOOKUP_COMMAND: rb_str_catf(output, "lookup_command(%+"PRIsVALUE")\n", constant); break; case OP_FILTER: { VALUE filter_name = RARRAY_AREF(constant, 0); uint8_t num_args = RARRAY_AREF(constant, 1); rb_str_catf(output, "filter(name: %+"PRIsVALUE", num_args: %u)\n", filter_name, num_args); break; } case OP_BUILTIN_FILTER: rb_str_catf(output, "builtin_filter(name: :%s, num_args: %u)\n", builtin_filters[ip[1]].name, ip[2]); break; default: rb_str_catf(output, "\n", ip[0]); break; } liquid_vm_next_instruction(&ip); } return output; } struct merge_constants_table_func_args { st_table *hash; size_t increment_amount; }; static int merge_constants_table(st_data_t key, st_data_t value, VALUE _arg) { struct merge_constants_table_func_args *arg = (struct merge_constants_table_func_args *)_arg; st_table *dest_hash = arg->hash; uint16_t new_value = value + arg->increment_amount; st_insert(dest_hash, key, new_value); return ST_CONTINUE; } void update_instructions_constants_table_index_ref(c_buffer_t *instructions, size_t increment_amount, c_buffer_t *constants) { uint8_t *ip = instructions->data; while (ip < instructions->data_end) { if (vm_assembler_opcode_has_constant(*ip)) { uint16_t constant_index = (ip[1] << 8) | ip[2]; uint16_t new_constant_index = constant_index + increment_amount; ip[1] = new_constant_index >> 8; ip[2] = (uint8_t)new_constant_index; } liquid_vm_next_instruction((const uint8_t **)&ip); } } void vm_assembler_concat(vm_assembler_t *dest, vm_assembler_t *src) { size_t dest_element_count = c_buffer_size(&dest->constants) / sizeof(VALUE); // merge src constants table into dest constants table with new index struct merge_constants_table_func_args arg; arg.hash = dest->constants_table; arg.increment_amount = dest_element_count; st_foreach(src->constants_table, merge_constants_table, (VALUE)&arg); // merge constants array c_buffer_concat(&dest->constants, &src->constants); update_instructions_constants_table_index_ref(&src->instructions, dest_element_count, &dest->constants); c_buffer_concat(&dest->instructions, &src->instructions); size_t max_src_stack_size = dest->stack_size + src->max_stack_size; if (max_src_stack_size > dest->max_stack_size) dest->max_stack_size = max_src_stack_size; dest->stack_size += src->stack_size; } void vm_assembler_require_stack_args(vm_assembler_t *code, unsigned int count) { if (code->stack_size < code->protected_stack_size + count) { rb_raise(rb_eRuntimeError, "insufficient number of values on the stack"); } } void vm_assembler_add_write_raw(vm_assembler_t *code, const char *string, size_t size) { if (size > UINT8_MAX) { uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 4); instructions[0] = OP_WRITE_RAW_W; uint24_to_bytes((unsigned int)size, &instructions[1]); } else { uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 2); instructions[0] = OP_WRITE_RAW; instructions[1] = size; } c_buffer_write(&code->instructions, (char *)string, size); } void vm_assembler_add_write_node(vm_assembler_t *code, VALUE node) { vm_assembler_add_op_with_constant(code, node, OP_WRITE_NODE); } void vm_assembler_add_push_fixnum(vm_assembler_t *code, VALUE num) { long x = FIX2LONG(num); if (x >= INT8_MIN && x <= INT8_MAX) { vm_assembler_add_push_int8(code, x); } else if (x >= INT16_MIN && x <= INT16_MAX) { vm_assembler_add_push_int16(code, x); } else { vm_assembler_add_push_const(code, num); } } void vm_assembler_add_push_literal(vm_assembler_t *code, VALUE literal) { switch (literal) { case Qnil: vm_assembler_add_push_nil(code); break; case Qtrue: vm_assembler_add_push_true(code); break; case Qfalse: vm_assembler_add_push_false(code); break; default: if (RB_FIXNUM_P(literal)) { vm_assembler_add_push_fixnum(code, literal); } else { vm_assembler_add_push_const(code, literal); } break; } } void vm_assembler_add_filter(vm_assembler_t *code, VALUE filter_name, size_t arg_count) { if (arg_count > 254) { rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Too many filter arguments"); } code->stack_size -= arg_count; // pop arg_count + 1, push 1 st_data_t builtin_index; bool is_builtin = st_lookup(builtin_filter_table, filter_name, &builtin_index); if (is_builtin) { uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 3); *instructions++ = OP_BUILTIN_FILTER; *instructions++ = builtin_index; *instructions++ = arg_count + 1; // include input } else { VALUE filter_args = rb_ary_new_capa(2); rb_ary_push(filter_args, filter_name); rb_ary_push(filter_args, arg_count + 1); vm_assembler_add_op_with_constant(code, filter_args, OP_FILTER); } } static void ensure_parsing(vm_assembler_t *code) { if (!code->parsing) rb_raise(rb_eRuntimeError, "cannot extend code after it has finished being compiled"); } void vm_assembler_add_evaluate_expression_from_ruby(vm_assembler_t *code, VALUE code_obj, VALUE expression) { ensure_parsing(code); if (RB_SPECIAL_CONST_P(expression)) { vm_assembler_add_push_literal(code, expression); return; } switch (RB_BUILTIN_TYPE(expression)) { case T_DATA: if (RBASIC_CLASS(expression) == cLiquidCExpression) { vm_assembler_concat(code, &((expression_t *)DATA_PTR(expression))->code); vm_assembler_remove_leave(code); return; } break; case T_OBJECT: { VALUE klass = RBASIC_CLASS(expression); if (klass == cLiquidVariableLookup || klass == cLiquidRangeLookup) { rb_funcall(expression, id_compile_evaluate, 1, code_obj); return; } break; } default: break; } vm_assembler_add_push_const(code, expression); } void vm_assembler_add_find_variable_from_ruby(vm_assembler_t *code, VALUE code_obj, VALUE expression) { ensure_parsing(code); if (RB_TYPE_P(expression, T_STRING)) { vm_assembler_add_find_static_variable(code, expression); } else { vm_assembler_add_evaluate_expression_from_ruby(code, code_obj, expression); vm_assembler_add_find_variable(code); } } void vm_assembler_add_lookup_command_from_ruby(vm_assembler_t *code, VALUE command) { StringValue(command); ensure_parsing(code); vm_assembler_require_stack_args(code, 1); vm_assembler_add_lookup_command(code, command); } void vm_assembler_add_lookup_key_from_ruby(vm_assembler_t *code, VALUE code_obj, VALUE expression) { ensure_parsing(code); vm_assembler_require_stack_args(code, 1); if (RB_TYPE_P(expression, T_STRING)) { vm_assembler_add_lookup_const_key(code, expression); } else { vm_assembler_add_evaluate_expression_from_ruby(code, code_obj, expression); vm_assembler_add_lookup_key(code); } } void vm_assembler_add_new_int_range_from_ruby(vm_assembler_t *code) { ensure_parsing(code); vm_assembler_require_stack_args(code, 2); vm_assembler_add_new_int_range(code); } void vm_assembler_add_hash_new_from_ruby(vm_assembler_t *code, VALUE hash_size_obj) { ensure_parsing(code); unsigned int hash_size = NUM2USHORT(hash_size_obj); vm_assembler_require_stack_args(code, hash_size * 2); vm_assembler_add_hash_new(code, hash_size); } void vm_assembler_add_filter_from_ruby(vm_assembler_t *code, VALUE filter_name, VALUE arg_count_obj) { ensure_parsing(code); unsigned int arg_count = NUM2USHORT(arg_count_obj); vm_assembler_require_stack_args(code, arg_count + 1); filter_name = rb_str_intern(filter_name); vm_assembler_add_filter(code, filter_name, arg_count); } bool vm_assembler_opcode_has_constant(uint8_t ip) { if ( ip == OP_PUSH_CONST || ip == OP_WRITE_NODE || ip == OP_FIND_STATIC_VAR || ip == OP_LOOKUP_CONST_KEY || ip == OP_LOOKUP_COMMAND || ip == OP_FILTER ) { return true; } return false; } void liquid_define_vm_assembler(void) { builtin_filter_table = st_init_numtable_with_size(ARRAY_LENGTH(builtin_filters)); for (unsigned int i = 0; i < ARRAY_LENGTH(builtin_filters); i++) { filter_desc_t *filter = &builtin_filters[i]; filter->sym = ID2SYM(rb_intern(filter->name)); st_insert(builtin_filter_table, filter->sym, i); } } liquid-c-4.1.0/ext/liquid_c/vm_assembler.h000066400000000000000000000175351417631740700204710ustar00rootroot00000000000000#ifndef VM_ASSEMBLER_H #define VM_ASSEMBLER_H #include #include "liquid.h" #include "c_buffer.h" #include "intutil.h" enum opcode { OP_LEAVE = 0, OP_WRITE_RAW_W = 1, OP_WRITE_NODE = 2, OP_POP_WRITE, OP_WRITE_RAW_SKIP, OP_PUSH_CONST, OP_PUSH_NIL, OP_PUSH_TRUE, OP_PUSH_FALSE, OP_PUSH_INT8, OP_PUSH_INT16, OP_FIND_STATIC_VAR, OP_FIND_VAR, OP_LOOKUP_CONST_KEY, OP_LOOKUP_KEY, OP_LOOKUP_COMMAND, OP_NEW_INT_RANGE, OP_HASH_NEW, // rb_hash_new & rb_hash_bulk_insert OP_FILTER, OP_BUILTIN_FILTER, OP_RENDER_VARIABLE_RESCUE, // setup state to rescue variable rendering OP_WRITE_RAW, OP_JUMP_FWD_W, OP_JUMP_FWD, }; typedef struct { const char *name; VALUE sym; } filter_desc_t; extern filter_desc_t builtin_filters[]; typedef struct vm_assembler { c_buffer_t instructions; c_buffer_t constants; st_table *constants_table; size_t max_stack_size; size_t stack_size; size_t protected_stack_size; bool parsing; // prevent executing when incomplete or extending when complete } vm_assembler_t; void liquid_define_vm_assembler(void); void vm_assembler_init(vm_assembler_t *code); void vm_assembler_reset(vm_assembler_t *code); void vm_assembler_free(vm_assembler_t *code); void vm_assembler_gc_mark(vm_assembler_t *code); VALUE vm_assembler_disassemble(const uint8_t *start_ip, const uint8_t *end_ip, const VALUE *constants); void vm_assembler_concat(vm_assembler_t *dest, vm_assembler_t *src); void vm_assembler_require_stack_args(vm_assembler_t *code, unsigned int count); void vm_assembler_add_write_raw(vm_assembler_t *code, const char *string, size_t size); void vm_assembler_add_write_node(vm_assembler_t *code, VALUE node); void vm_assembler_add_push_fixnum(vm_assembler_t *code, VALUE num); void vm_assembler_add_push_literal(vm_assembler_t *code, VALUE literal); void vm_assembler_add_filter(vm_assembler_t *code, VALUE filter_name, size_t arg_count); void vm_assembler_add_evaluate_expression_from_ruby(vm_assembler_t *code, VALUE code_obj, VALUE expression); void vm_assembler_add_find_variable_from_ruby(vm_assembler_t *code, VALUE code_obj, VALUE expression); void vm_assembler_add_lookup_command_from_ruby(vm_assembler_t *code, VALUE command); void vm_assembler_add_lookup_key_from_ruby(vm_assembler_t *code, VALUE code_obj, VALUE expression); void vm_assembler_add_new_int_range_from_ruby(vm_assembler_t *code); void vm_assembler_add_hash_new_from_ruby(vm_assembler_t *code, VALUE hash_size_obj); void vm_assembler_add_filter_from_ruby(vm_assembler_t *code, VALUE filter_name, VALUE arg_count_obj); bool vm_assembler_opcode_has_constant(uint8_t ip); static inline size_t vm_assembler_alloc_memsize(const vm_assembler_t *code) { return c_buffer_capacity(&code->instructions) + c_buffer_capacity(&code->constants) + sizeof(st_table); } static inline void vm_assembler_write_opcode(vm_assembler_t *code, enum opcode op) { c_buffer_write_byte(&code->instructions, op); } static inline uint16_t vm_assembler_write_ruby_constant(vm_assembler_t *code, VALUE constant) { st_table *constants_table = code->constants_table; st_data_t index_value; if (st_lookup(constants_table, constant, &index_value)) { return (uint16_t)index_value; } else { uint16_t index = c_buffer_size(&code->constants) / sizeof(VALUE); st_insert(constants_table, constant, index); c_buffer_write(&code->constants, &constant, sizeof(VALUE)); return index; } } static inline void vm_assembler_increment_stack_size(vm_assembler_t *code, size_t amount) { code->stack_size += amount; if (code->stack_size > code->max_stack_size) code->max_stack_size = code->stack_size; } static inline void vm_assembler_reserve_stack_size(vm_assembler_t *code, size_t amount) { vm_assembler_increment_stack_size(code, amount); code->stack_size -= amount; } static inline void vm_assembler_add_op_with_constant(vm_assembler_t *code, VALUE constant, uint8_t opcode) { uint16_t index = vm_assembler_write_ruby_constant(code, constant); uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 3); instructions[0] = opcode; instructions[1] = index >> 8; instructions[2] = (uint8_t)index; } static inline void vm_assembler_add_leave(vm_assembler_t *code) { vm_assembler_write_opcode(code, OP_LEAVE); code->parsing = false; } static inline void vm_assembler_remove_leave(vm_assembler_t *code) { code->parsing = true; code->instructions.data_end--; assert(*code->instructions.data_end == OP_LEAVE); } static inline void vm_assembler_add_pop_write(vm_assembler_t *code) { code->stack_size -= 1; vm_assembler_write_opcode(code, OP_POP_WRITE); } static inline void vm_assembler_add_hash_new(vm_assembler_t *code, size_t hash_size) { if (hash_size > 255) rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Hash literal has too many keys"); code->stack_size -= hash_size * 2; code->stack_size++; uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 2); instructions[0] = OP_HASH_NEW; instructions[1] = hash_size; } static inline void vm_assembler_add_push_nil(vm_assembler_t *code) { vm_assembler_increment_stack_size(code, 1); vm_assembler_write_opcode(code, OP_PUSH_NIL); } static inline void vm_assembler_add_push_true(vm_assembler_t *code) { vm_assembler_increment_stack_size(code, 1); vm_assembler_write_opcode(code, OP_PUSH_TRUE); } static inline void vm_assembler_add_push_false(vm_assembler_t *code) { vm_assembler_increment_stack_size(code, 1); vm_assembler_write_opcode(code, OP_PUSH_FALSE); } static inline void vm_assembler_add_push_int8(vm_assembler_t *code, int8_t value) { vm_assembler_increment_stack_size(code, 1); uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 2); instructions[0] = OP_PUSH_INT8; instructions[1] = value; } static inline void vm_assembler_add_push_int16(vm_assembler_t *code, int16_t value) { vm_assembler_increment_stack_size(code, 1); uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 3); instructions[0] = OP_PUSH_INT16; instructions[1] = value >> 8; instructions[2] = (uint8_t)value; } static inline void vm_assembler_add_push_const(vm_assembler_t *code, VALUE constant) { vm_assembler_increment_stack_size(code, 1); vm_assembler_add_op_with_constant(code, constant, OP_PUSH_CONST); } static inline void vm_assembler_add_find_static_variable(vm_assembler_t *code, VALUE key) { vm_assembler_increment_stack_size(code, 1); vm_assembler_add_op_with_constant(code, key, OP_FIND_STATIC_VAR); } static inline void vm_assembler_add_find_variable(vm_assembler_t *code) { // pop 1, push 1 vm_assembler_write_opcode(code, OP_FIND_VAR); } static inline void vm_assembler_add_lookup_const_key(vm_assembler_t *code, VALUE key) { vm_assembler_reserve_stack_size(code, 1); // push 1, pop 2, push 1 vm_assembler_add_op_with_constant(code, key, OP_LOOKUP_CONST_KEY); } static inline void vm_assembler_add_lookup_key(vm_assembler_t *code) { code->stack_size--; // pop 2, push 1 vm_assembler_write_opcode(code, OP_LOOKUP_KEY); } static inline void vm_assembler_add_lookup_command(vm_assembler_t *code, VALUE command) { vm_assembler_reserve_stack_size(code, 1); // push 1, pop 2, push 1 vm_assembler_add_op_with_constant(code, command, OP_LOOKUP_COMMAND); } static inline void vm_assembler_add_new_int_range(vm_assembler_t *code) { code->stack_size--; // pop 2, push 1 vm_assembler_write_opcode(code, OP_NEW_INT_RANGE); } static inline void vm_assembler_add_render_variable_rescue(vm_assembler_t *code, size_t node_line_number) { uint8_t *instructions = c_buffer_extend_for_write(&code->instructions, 4); instructions[0] = OP_RENDER_VARIABLE_RESCUE; uint24_to_bytes((unsigned int)node_line_number, &instructions[1]); } #endif liquid-c-4.1.0/ext/liquid_c/vm_assembler_pool.c000066400000000000000000000053141417631740700215050ustar00rootroot00000000000000#include "vm_assembler_pool.h" #include "parse_context.h" static VALUE cLiquidCVMAssemblerPool; void vm_assembler_pool_gc_mark(vm_assembler_pool_t *pool) { rb_gc_mark(pool->self); } static void vm_assembler_pool_free(void *ptr) { vm_assembler_pool_t *pool = ptr; vm_assembler_element_t *element = pool->freelist; while (element) { vm_assembler_free(&element->vm_assembler); vm_assembler_element_t *next = element->next; xfree(element); element = next; } xfree(pool); } static size_t vm_assembler_pool_memsize(const void *ptr) { const vm_assembler_pool_t *pool = ptr; size_t elements_size = 0; vm_assembler_element_t *element = pool->freelist; while (element) { elements_size += sizeof(vm_assembler_element_t) + vm_assembler_alloc_memsize(&element->vm_assembler); element = element->next; } return sizeof(vm_assembler_pool_t) + elements_size; } const rb_data_type_t vm_assembler_pool_data_type = { "liquid_vm_assembler_pool", { NULL, vm_assembler_pool_free, vm_assembler_pool_memsize, }, NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY }; VALUE vm_assembler_pool_new(void) { vm_assembler_pool_t *pool; VALUE obj = TypedData_Make_Struct(cLiquidCVMAssemblerPool, vm_assembler_pool_t, &vm_assembler_pool_data_type, pool); pool->self = obj; pool->freelist = NULL; return obj; } vm_assembler_t *vm_assembler_pool_alloc_assembler(vm_assembler_pool_t *pool) { vm_assembler_element_t *element; if (!pool->freelist) { element = xmalloc(sizeof(vm_assembler_element_t)); element->next = NULL; vm_assembler_init(&element->vm_assembler); } else { element = pool->freelist; pool->freelist = element->next; } return &element->vm_assembler; } static vm_assembler_element_t *get_element_from_assembler(vm_assembler_t *assembler) { return (vm_assembler_element_t *)((char *)assembler - offsetof(vm_assembler_element_t, vm_assembler)); } void vm_assembler_pool_free_assembler(vm_assembler_t *assembler) { vm_assembler_element_t *element = get_element_from_assembler(assembler); vm_assembler_free(&element->vm_assembler); xfree(element); } void vm_assembler_pool_recycle_assembler(vm_assembler_pool_t *pool, vm_assembler_t *assembler) { vm_assembler_element_t *element = get_element_from_assembler(assembler); vm_assembler_reset(&element->vm_assembler); element->next = pool->freelist; pool->freelist = element; } void liquid_define_vm_assembler_pool(void) { cLiquidCVMAssemblerPool = rb_define_class_under(mLiquidC, "VMAssemblerPool", rb_cObject); rb_global_variable(&cLiquidCVMAssemblerPool); rb_undef_alloc_func(cLiquidCVMAssemblerPool); } liquid-c-4.1.0/ext/liquid_c/vm_assembler_pool.h000066400000000000000000000016551417631740700215160ustar00rootroot00000000000000#ifndef LIQUID_VM_ASSEMBLER_POOL_H #define LIQUID_VM_ASSEMBLER_POOL_H #include "liquid.h" #include "vm_assembler.h" typedef struct vm_assembler_element { struct vm_assembler_element *next; vm_assembler_t vm_assembler; } vm_assembler_element_t; typedef struct vm_assembler_pool { VALUE self; vm_assembler_element_t *freelist; } vm_assembler_pool_t; extern const rb_data_type_t vm_assembler_pool_data_type; #define VMAssemblerPool_Get_Struct(obj, sval) TypedData_Get_Struct(obj, vm_assembler_pool_t, &vm_assembler_pool_data_type, sval) void liquid_define_vm_assembler_pool(void); void vm_assembler_pool_gc_mark(vm_assembler_pool_t *pool); VALUE vm_assembler_pool_new(void); vm_assembler_t *vm_assembler_pool_alloc_assembler(vm_assembler_pool_t *pool); void vm_assembler_pool_free_assembler(vm_assembler_t *assembler); void vm_assembler_pool_recycle_assembler(vm_assembler_pool_t *pool, vm_assembler_t *assembler); #endif liquid-c-4.1.0/lib/000077500000000000000000000000001417631740700140035ustar00rootroot00000000000000liquid-c-4.1.0/lib/liquid/000077500000000000000000000000001417631740700152725ustar00rootroot00000000000000liquid-c-4.1.0/lib/liquid/c.rb000066400000000000000000000167641417631740700160570ustar00rootroot00000000000000# frozen_string_literal: true require "liquid/c/version" require "liquid" require "liquid_c" require "liquid/c/compile_ext" Liquid::C::BlockBody.class_eval do def render(context) render_to_output_buffer(context, +"") end end module Liquid BlockBody.class_eval do def self.c_rescue_render_node(context, output, line_number, exc, blank_tag) # There seems to be a MRI ruby bug with how the rb_rescue C function, # where $! gets set for its rescue callback, but it doesn't stay set # after a nested exception is raised and handled as is the case in # Liquid::Context#internal_error. This isn't a problem for plain ruby code, # so use a ruby rescue block to have setup $! properly. raise(exc) rescue => exc rescue_render_node(context, output, line_number, exc, blank_tag) end end end module Liquid module C # Placeholder for variables in the Liquid::C::BlockBody#nodelist. class VariablePlaceholder class << self private :new end end class Tokenizer MAX_SOURCE_BYTE_SIZE = (1 << 24) - 1 end end end Liquid::Raw.class_eval do alias_method :ruby_parse, :parse def parse(tokenizer) if parse_context.liquid_c_nodes_disabled? ruby_parse(tokenizer) else c_parse(tokenizer) end end end Liquid::ParseContext.class_eval do class << self attr_accessor :liquid_c_nodes_disabled end self.liquid_c_nodes_disabled = false alias_method :ruby_new_block_body, :new_block_body def new_block_body if liquid_c_nodes_disabled? ruby_new_block_body else Liquid::C::BlockBody.new(self) end end alias_method :ruby_new_tokenizer, :new_tokenizer def new_tokenizer(source, start_line_number: nil, for_liquid_tag: false) unless liquid_c_nodes_disabled? source = source.to_s.to_str if source.bytesize <= Liquid::C::Tokenizer::MAX_SOURCE_BYTE_SIZE source = source.encode(Encoding::UTF_8) return Liquid::C::Tokenizer.new(source, start_line_number || 0, for_liquid_tag) else @liquid_c_nodes_disabled = true end end ruby_new_tokenizer(source, start_line_number: start_line_number, for_liquid_tag: for_liquid_tag) end def parse_expression(markup) if liquid_c_nodes_disabled? Liquid::Expression.ruby_parse(markup) else Liquid::C::Expression.lax_parse(markup) end end # @api private def liquid_c_nodes_disabled? # Liquid::Profiler exposes the internal parse tree that we don't want to build when # parsing with liquid-c, so consider liquid-c to be disabled when using it. # Also, some templates are parsed before the profiler is running, on which case we # provide the `disable_liquid_c_nodes` option to enable the Ruby AST to be produced # so the profiler can use it on future runs. return @liquid_c_nodes_disabled if defined?(@liquid_c_nodes_disabled) @liquid_c_nodes_disabled = !Liquid::C.enabled || @template_options[:profile] || @template_options[:disable_liquid_c_nodes] || self.class.liquid_c_nodes_disabled end end module Liquid module C module DocumentClassPatch def parse(tokenizer, parse_context) if tokenizer.is_a?(Liquid::C::Tokenizer) if parse_context[:bug_compatible_whitespace_trimming] # Temporary to test rollout of the fix for this bug tokenizer.bug_compatible_whitespace_trimming! end begin parse_context.start_liquid_c_parsing super ensure parse_context.cleanup_liquid_c_parsing end else super end end end Liquid::Document.singleton_class.prepend(DocumentClassPatch) end end Liquid::Variable.class_eval do class << self # @api private def call_variable_fallback_stats_callback(parse_context) callbacks = parse_context[:stats_callbacks] callbacks[:variable_fallback]&.call if callbacks end private # helper method for C code def rescue_strict_parse_syntax_error(error, markup, parse_context) error.line_number = parse_context.line_number error.markup_context = "in \"{{#{markup}}}\"" case parse_context.error_mode when :strict raise when :warn parse_context.warnings << error end call_variable_fallback_stats_callback(parse_context) lax_parse(markup, parse_context) # Avoid redundant strict parse end def lax_parse(markup, parse_context) old_error_mode = parse_context.error_mode begin set_error_mode(parse_context, :lax) new(markup, parse_context) ensure set_error_mode(parse_context, old_error_mode) end end def set_error_mode(parse_context, mode) parse_context.instance_variable_set(:@error_mode, mode) end end alias_method :ruby_strict_parse, :strict_parse def strict_parse(markup) if parse_context.liquid_c_nodes_disabled? ruby_strict_parse(markup) else c_strict_parse(markup) end end end Liquid::StrainerTemplate.class_eval do class << self private def filter_methods_hash @filter_methods_hash ||= {}.tap do |hash| filter_methods.each do |method_name| hash[method_name.to_sym] = true end end end # Convert wrong number of argument error into a liquid exception to # treat it as an error in the template, not an internal error. def arg_exc_to_liquid_exc(argument_error) raise Liquid::ArgumentError, argument_error.message, argument_error.backtrace rescue Liquid::ArgumentError => liquid_exc liquid_exc end end end Liquid::C::Expression.class_eval do class << self def lax_parse(markup) strict_parse(markup) rescue Liquid::SyntaxError Liquid::Expression.ruby_parse(markup) end end end Liquid::Expression.class_eval do class << self alias_method :ruby_parse, :parse def c_parse(markup) Liquid::C::Expression.lax_parse(markup) end end end Liquid::Context.class_eval do alias_method :ruby_evaluate, :evaluate alias_method :ruby_find_variable, :find_variable alias_method :ruby_strict_variables=, :strict_variables= # This isn't entered often by Ruby (most calls stay in C via VariableLookup#evaluate) # so the wrapper method isn't costly. def c_find_variable_kwarg(key, raise_on_not_found: true) c_find_variable(key, raise_on_not_found) end end Liquid::ResourceLimits.class_eval do def self.new(limits) if Liquid::C.enabled Liquid::C::ResourceLimits.new( limits[:render_length_limit], limits[:render_score_limit], limits[:assign_score_limit] ) else super end end end module Liquid module C class << self attr_reader :enabled def enabled=(value) @enabled = value if value Liquid::Context.send(:alias_method, :evaluate, :c_evaluate) Liquid::Context.send(:alias_method, :find_variable, :c_find_variable_kwarg) Liquid::Context.send(:alias_method, :strict_variables=, :c_strict_variables=) Liquid::Expression.singleton_class.send(:alias_method, :parse, :c_parse) else Liquid::Context.send(:alias_method, :evaluate, :ruby_evaluate) Liquid::Context.send(:alias_method, :find_variable, :ruby_find_variable) Liquid::Context.send(:alias_method, :strict_variables=, :ruby_strict_variables=) Liquid::Expression.singleton_class.send(:alias_method, :parse, :ruby_parse) end end end self.enabled = true end end liquid-c-4.1.0/lib/liquid/c/000077500000000000000000000000001417631740700155145ustar00rootroot00000000000000liquid-c-4.1.0/lib/liquid/c/compile_ext.rb000066400000000000000000000021551417631740700203540ustar00rootroot00000000000000# frozen_string_literal: true Liquid::Variable.class_eval do def compile_evaluate(code) code.add_evaluate_expression(@name) filters.each do |filter_name, filter_args, keyword_args| filter_args.each do |arg| code.add_evaluate_expression(arg) end num_args = filter_args.size if keyword_args keyword_args.each do |key, value| code.add_evaluate_expression(key) code.add_evaluate_expression(value) end num_args += 1 code.add_hash_new(keyword_args.size) end code.add_filter(filter_name, num_args) end end end Liquid::VariableLookup.class_eval do def compile_evaluate(code) code.add_find_variable(name) lookups.each_with_index do |lookup, i| is_command = @command_flags & (1 << i) != 0 if is_command code.add_lookup_command(lookup) else code.add_lookup_key(lookup) end end end end Liquid::RangeLookup.class_eval do def compile_evaluate(code) code.add_evaluate_expression(@start_obj) code.add_evaluate_expression(@end_obj) code.add_new_int_range end end liquid-c-4.1.0/lib/liquid/c/version.rb000066400000000000000000000001301417631740700175200ustar00rootroot00000000000000# frozen_string_literal: true module Liquid module C VERSION = "4.0.0" end end liquid-c-4.1.0/liquid-c.gemspec000077500000000000000000000024261417631740700163200ustar00rootroot00000000000000# coding: utf-8 # frozen_string_literal: true # rubocop:disable Gemspec/RubyVersionGlobalsUsage lib = File.expand_path("../lib", __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require "liquid/c/version" Gem::Specification.new do |spec| spec.name = "liquid-c" spec.version = Liquid::C::VERSION spec.authors = ["Justin Li", "Dylan Thacker-Smith"] spec.email = ["gems@shopify.com"] spec.summary = "Liquid performance extension in C" spec.homepage = "https://github.com/shopify/liquid-c" spec.license = "MIT" spec.extensions = ["ext/liquid_c/extconf.rb"] spec.files = %x(git ls-files -z).split("\x0") spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] spec.required_ruby_version = ">= 2.5.0" spec.add_dependency("liquid", ">= 5.0.1") spec.add_development_dependency("bundler", ">= 1.5") # has bugfix for segfaulting deploys spec.add_development_dependency("minitest") spec.add_development_dependency("rake") spec.add_development_dependency("rake-compiler") spec.add_development_dependency("stackprof") if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0") end liquid-c-4.1.0/performance.rb000066400000000000000000000004651417631740700160700ustar00rootroot00000000000000# frozen_string_literal: true require "liquid" require "liquid/c" if ARGV.shift == "c" liquid_lib_dir = $LOAD_PATH.detect { |p| File.exist?(File.join(p, "liquid.rb")) } (script = ARGV.shift) || abort("unspecified performance script") require File.join(File.dirname(liquid_lib_dir), "performance/#{script}") liquid-c-4.1.0/performance/000077500000000000000000000000001417631740700155365ustar00rootroot00000000000000liquid-c-4.1.0/performance/c_profile.rb000066400000000000000000000012461417631740700200300ustar00rootroot00000000000000# frozen_string_literal: true require "liquid" require "liquid/c" liquid_lib_dir = $LOAD_PATH.detect { |p| File.exist?(File.join(p, "liquid.rb")) } require File.join(File.dirname(liquid_lib_dir), "performance/theme_runner") TASK_NAMES = ["run", "compile", "render"] task_name = ARGV.first || "run" unless TASK_NAMES.include?(task_name) raise "Unsupported task '#{task_name}' (must be one of #{TASK_NAMES})" end task = ThemeRunner.new.method(task_name) runner_id = fork do end_time = Time.now + 5.0 task.call until Time.now >= end_time end profiler_pid = spawn("instruments -t 'Time Profiler' -p #{runner_id}") Process.waitpid(runner_id) Process.waitpid(profiler_pid) liquid-c-4.1.0/rakelib/000077500000000000000000000000001417631740700146465ustar00rootroot00000000000000liquid-c-4.1.0/rakelib/compile.rake000066400000000000000000000006311417631740700171420ustar00rootroot00000000000000# frozen_string_literal: true ext_task = Rake::ExtensionTask.new("liquid_c") # For MacOS, generate debug information that ruby can read dsymutil = RbConfig::CONFIG["dsymutil"] unless dsymutil.to_s.empty? ext_lib_path = "lib/#{ext_task.binary}" dsym_path = "#{ext_lib_path}.dSYM" file dsym_path => [ext_lib_path] do sh dsymutil, ext_lib_path end Rake::Task["compile"].enhance([dsym_path]) end liquid-c-4.1.0/rakelib/integration_test.rake000066400000000000000000000023551417631740700211010ustar00rootroot00000000000000# frozen_string_literal: true namespace :test do integration_test_config = lambda do |t| t.libs << "lib" t.test_files = ["test/integration_test.rb"] end desc "run test suite with default parser" Rake::TestTask.new(integration: :compile, &integration_test_config) namespace :integration do define_env_integration_tests = lambda do |task_name| rake_task = Rake::Task[task_name] [ [:lax, { "LIQUID_PARSER_MODE" => "lax" }], [:strict, { "LIQUID_PARSER_MODE" => "strict" }], [:without_vm, { "LIQUID_C_DISABLE_VM" => "true" }], ].each do |name, env_vars| task(name) do old_env_values = ENV.to_hash.slice(*env_vars.keys) begin env_vars.each { |key, value| ENV[key] = value } rake_task.invoke ensure old_env_values.each { |key, value| ENV[key] = value } rake_task.reenable end end end task(all: [:lax, :strict, :without_vm]) end define_env_integration_tests.call("test:integration") RubyMemcheck::TestTask.new(valgrind: :compile, &integration_test_config) namespace :valgrind do define_env_integration_tests.call("test:integration:valgrind") end end end liquid-c-4.1.0/rakelib/performance.rake000066400000000000000000000017731417631740700200230ustar00rootroot00000000000000# frozen_string_literal: true namespace :benchmark do desc "Run the liquid benchmark with lax parsing" task :run do ruby "./performance.rb c benchmark lax" end desc "Run the liquid benchmark with strict parsing" task :strict do ruby "./performance.rb c benchmark strict" end end namespace :c_profile do [:run, :compile, :render].each do |task_name| task(task_name) do ruby "./performance/c_profile.rb #{task_name}" end end end namespace :profile do desc "Run the liquid profile/performance coverage" task :run do ruby "./performance.rb c profile lax" end desc "Run the liquid profile/performance coverage with strict parsing" task :strict do ruby "./performance.rb c profile strict" end end namespace :compare do ["lax", "warn", "strict"].each do |type| desc "Compare Liquid to Liquid-C in #{type} mode" task type.to_sym do ruby "./performance.rb bare benchmark #{type}" ruby "./performance.rb c benchmark #{type}" end end end liquid-c-4.1.0/rakelib/rubocop.rake000066400000000000000000000001521417631740700171610ustar00rootroot00000000000000# frozen_string_literal: true task :rubocop do require "rubocop/rake_task" RuboCop::RakeTask.new end liquid-c-4.1.0/rakelib/unit_test.rake000066400000000000000000000005161417631740700175320ustar00rootroot00000000000000# frozen_string_literal: true namespace :test do unit_test_config = lambda do |t| t.libs << "lib" << "test" t.test_files = FileList["test/unit/**/*_test.rb"] end Rake::TestTask.new(unit: :compile, &unit_test_config) namespace :unit do RubyMemcheck::TestTask.new(valgrind: :compile, &unit_test_config) end end liquid-c-4.1.0/test/000077500000000000000000000000001417631740700142145ustar00rootroot00000000000000liquid-c-4.1.0/test/integration_test.rb000066400000000000000000000004441417631740700201250ustar00rootroot00000000000000# frozen_string_literal: true at_exit { GC.start } require_relative "liquid_test_helper" test_files = Dir[File.join(LIQUID_TEST_DIR, "integration/**/*_test.rb")] test_files << File.join(LIQUID_TEST_DIR, "unit/tokenizer_unit_test.rb") test_files.each do |test_file| require test_file end liquid-c-4.1.0/test/liquid_test_helper.rb000066400000000000000000000013071417631740700204270ustar00rootroot00000000000000# frozen_string_literal: true # This can be used to setup for running tests from the liquid test suite. # For example, you could run a single liquid test as follows: # $ ruby -r./test/liquid_test_helper `bundle info liquid --path`/test/integration/template_test.rb require "bundler/setup" liquid_lib_dir = $LOAD_PATH.detect { |p| File.exist?(File.join(p, "liquid.rb")) } LIQUID_TEST_DIR = File.join(File.dirname(liquid_lib_dir), "test") $LOAD_PATH << LIQUID_TEST_DIR << File.expand_path("../lib", __dir__) require "test_helper" require "liquid/c" if ENV["LIQUID_C_DISABLE_VM"] puts "-- Liquid-C VM Disabled" Liquid::ParseContext.liquid_c_nodes_disabled = true end GC.stress = true if ENV["GC_STRESS"] liquid-c-4.1.0/test/test_helper.rb000066400000000000000000000006171417631740700170630ustar00rootroot00000000000000# frozen_string_literal: true at_exit { GC.start } require "minitest/autorun" require "liquid/c" if GC.respond_to?(:verify_compaction_references) # This method was added in Ruby 3.0.0. Calling it this way asks the GC to # move objects around, helping to find object movement bugs. GC.verify_compaction_references(double_heap: true, toward: :empty) end GC.stress = true if ENV["GC_STRESS"] liquid-c-4.1.0/test/unit/000077500000000000000000000000001417631740700151735ustar00rootroot00000000000000liquid-c-4.1.0/test/unit/block_test.rb000066400000000000000000000077501417631740700176620ustar00rootroot00000000000000# frozen_string_literal: true require "test_helper" class BlockTest < MiniTest::Test def test_no_allocation_of_trimmed_strings template = Liquid::Template.parse("{{ a -}} {{- b }}") assert_equal(2, template.root.nodelist.size) template = Liquid::Template.parse("{{ a -}} foo {{- b }}") assert_equal(3, template.root.nodelist.size) end def test_raise_on_output_with_non_utf8_encoding output = String.new(encoding: Encoding::ASCII) template = Liquid::Template.parse("ascii text") exc = assert_raises(Encoding::CompatibilityError) do template.render!({}, output: output) end assert_equal("non-UTF8 encoded output (US-ASCII) not supported", exc.message) end def test_write_unicode_characters output = String.new(encoding: Encoding::UTF_8) template = Liquid::Template.parse("ü{{ unicode_char }}") assert_equal("üñ", template.render!({ "unicode_char" => "ñ" }, output: output)) end def test_op_write_raw_w source = "a" * 2**8 template = Liquid::Template.parse(source) assert_equal(source, template.render!) end def test_raise_for_non_c_parse_context parse_context = Liquid::ParseContext.new assert_raises(RuntimeError) do Liquid::C::BlockBody.new(parse_context) end end # Test for bug: https://github.com/Shopify/liquid-c/pull/120 def test_bug_120_instrument calls = [] Liquid::Usage.stub(:increment, ->(name) { calls << name }) do Liquid::Template.parse("{{ -.1 }}") end assert_equal(["liquid_c_negative_float_without_integer"], calls) calls = [] Liquid::Usage.stub(:increment, ->(name) { calls << name }) do Liquid::Template.parse("{{ .1 }}") end assert_equal([], calls) end def test_disassemble_raw_w source = "a" * 2**8 template = Liquid::Template.parse(source) block_body = template.root.body assert_equal(<<~ASM, block_body.disassemble) 0x0000: write_raw_w("#{source}") 0x0104: leave ASM end def test_disassemble source = <<~LIQUID raw {{- var | default: "none", allow_false: true -}} {%- increment counter -%} LIQUID template = Liquid::Template.parse(source, line_numbers: true) block_body = template.root.body increment_node = block_body.nodelist[2] assert_instance_of(Liquid::Increment, increment_node) assert_equal(<<~ASM, block_body.disassemble) 0x0000: write_raw("raw") 0x0005: render_variable_rescue(line_number: 2) 0x0009: find_static_var("var") 0x000c: push_const(\"none\") 0x000f: push_const(\"allow_false\") 0x0012: push_true 0x0013: hash_new(1) 0x0015: builtin_filter(name: :default, num_args: 3) 0x0018: pop_write 0x0019: write_node(#{increment_node.inspect}) 0x001c: leave ASM end def test_exception_renderer_exception original_error = Liquid::Error.new("original") handler_error = RuntimeError.new("exception handler error") context = Liquid::Context.new("raise_error" => ->(_ctx) { raise(original_error) }) context.exception_renderer = lambda do |exc| if exc == original_error raise(handler_error) end exc end template = Liquid::Template.parse("{% assign x = raise_error %}") exc = assert_raises(RuntimeError) do template.render(context) end assert_equal(handler_error, exc) end StubFileSystem = Struct.new(:partials) do def read_template_file(template_path) partials.fetch(template_path) end end def test_include_partial_with_syntax_error old_file_system = Liquid::Template.file_system begin Liquid::Template.file_system = StubFileSystem.new({ "invalid" => "{% foo %}", "valid" => '{% include "nested" %}', "nested" => "valid", }) template = Liquid::Template.parse("{% include 'invalid' %},{% include 'valid' %}") assert_equal("Liquid syntax error: Unknown tag 'foo',valid", template.render) ensure Liquid::Template.file_system = old_file_system end end end liquid-c-4.1.0/test/unit/context_test.rb000066400000000000000000000045521417631740700202510ustar00rootroot00000000000000# frozen_string_literal: true require "test_helper" require "bigdecimal" class ContextTest < Minitest::Test def test_evaluate_works_with_normal_values context = Liquid::Context.new ["abc", 123, false, 1.21, BigDecimal(42)].each do |value| assert_equal(value, context.evaluate(value)) end assert_nil(context.evaluate(nil)) end def test_evaluate_works_with_classes_that_have_an_evaluate_method class_with_evaluate = Class.new do def evaluate(_context) 42 end end assert_equal(42, Liquid::Context.new.evaluate(class_with_evaluate.new)) end def test_evaluate_works_with_variable_lookup assert_equal(42, Liquid::Context.new({ "var" => 42 }).evaluate(Liquid::C::Expression.strict_parse("var"))) end def test_evaluating_a_variable_entirely_within_c context = Liquid::Context.new({ "var" => 42 }) lookup = Liquid::C::Expression.strict_parse("var") context.evaluate(lookup) # memoize vm_internal_new calls called_ruby_method_count = 0 called_c_method_count = 0 test_thread = Thread.current begin call_trace = TracePoint.trace(:call) do |t| unless t.self == TracePoint || t.self.is_a?(TracePoint) || Thread.current != test_thread called_ruby_method_count += 1 end end c_call_trace = TracePoint.trace(:c_call) do |t| unless t.self == TracePoint || t.self.is_a?(TracePoint) || Thread.current != test_thread called_c_method_count += 1 end end context.evaluate(lookup) ensure call_trace&.disable c_call_trace&.disable end assert_equal(0, called_ruby_method_count) assert_equal(1, called_c_method_count) # context.evaluate call end class TestDrop < Liquid::Drop def is_filtering # rubocop:disable Naming/PredicateName @context.send(:c_filtering?) end end def test_c_filtering_predicate context = Liquid::Context.new({ "test" => [TestDrop.new] }) template = Liquid::Template.parse('{{ test[0].is_filtering }},{{ test | map: "is_filtering" }}') assert_equal("false,true", template.render!(context)) assert_equal(false, context.send(:c_filtering?)) end def test_strict_variables= context = Liquid::Context.new assert_equal(false, context.strict_variables) context.strict_variables = true assert_equal(true, context.strict_variables) end end liquid-c-4.1.0/test/unit/expression_test.rb000066400000000000000000000131741417631740700207640ustar00rootroot00000000000000# frozen_string_literal: true require "test_helper" class ExpressionTest < MiniTest::Test def test_constant_literals assert_equal(true, Liquid::C::Expression.strict_parse("true")) assert_equal(false, Liquid::C::Expression.strict_parse("false")) assert_nil(Liquid::C::Expression.strict_parse("nil")) assert_nil(Liquid::C::Expression.strict_parse("null")) empty = Liquid::C::Expression.strict_parse("empty") assert_equal("", empty) assert_same(empty, Liquid::C::Expression.strict_parse("blank")) end def test_push_literals assert_nil(compile_and_eval("nil")) assert_equal(true, compile_and_eval("true")) assert_equal(false, compile_and_eval("false")) end def test_constant_integer assert_equal(42, Liquid::C::Expression.strict_parse("42")) end def test_push_int8 assert_equal(127, compile_and_eval("127")) assert_equal(-128, compile_and_eval("-128")) end def test_push_int16 assert_equal(128, compile_and_eval("128")) assert_equal(-129, compile_and_eval("-129")) assert_equal(32767, compile_and_eval("32767")) assert_equal(-32768, compile_and_eval("-32768")) end def test_push_large_fixnum assert_equal(32768, compile_and_eval("32768")) assert_equal(-2147483648, compile_and_eval("-2147483648")) assert_equal(2147483648, compile_and_eval("2147483648")) assert_equal(4611686018427387903, compile_and_eval("4611686018427387903")) end def test_push_big_int num = 1 << 128 assert_equal(num, compile_and_eval(num.to_s)) end def test_float assert_equal(123.4, Liquid::C::Expression.strict_parse("123.4")) assert_equal(-1.5, compile_and_eval("-1.5")) end def test_string assert_equal("hello", Liquid::C::Expression.strict_parse('"hello"')) assert_equal("world", compile_and_eval("'world'")) end def test_find_static_variable context = Liquid::Context.new({ "x" => 123 }) expr = Liquid::C::Expression.strict_parse("x") assert_instance_of(Liquid::C::Expression, expr) assert_equal(123, context.evaluate(expr)) end def test_find_dynamic_variable context = Liquid::Context.new({ "x" => "y", "y" => 42 }) expr = Liquid::C::Expression.strict_parse("[x]") assert_equal(42, context.evaluate(expr)) end def test_find_missing_variable context = Liquid::Context.new({}) expr = Liquid::C::Expression.strict_parse("missing") assert_nil(context.evaluate(expr)) context.strict_variables = true assert_raises(Liquid::UndefinedVariable) do context.evaluate(expr) end end def test_lookup_const_key context = Liquid::Context.new({ "obj" => { "prop" => "some value" } }) expr = Liquid::C::Expression.strict_parse("obj.prop") assert_equal("some value", context.evaluate(expr)) expr = Liquid::C::Expression.strict_parse('obj["prop"]') assert_equal("some value", context.evaluate(expr)) end def test_lookup_variable_key context = Liquid::Context.new({ "field_name" => "prop", "obj" => { "prop" => "another value" } }) expr = Liquid::C::Expression.strict_parse("obj[field_name]") assert_equal("another value", context.evaluate(expr)) end def test_lookup_command context = Liquid::Context.new({ "ary" => ["a", "b", "c"] }) assert_equal(3, context.evaluate(Liquid::C::Expression.strict_parse("ary.size"))) assert_equal("a", context.evaluate(Liquid::C::Expression.strict_parse("ary.first"))) assert_equal("c", context.evaluate(Liquid::C::Expression.strict_parse("ary.last"))) end def test_lookup_missing_key context = Liquid::Context.new({ "obj" => {} }) expr = Liquid::C::Expression.strict_parse("obj.missing") assert_nil(context.evaluate(expr)) context.strict_variables = true assert_raises(Liquid::UndefinedVariable) do context.evaluate(expr) end end def test_lookup_on_var_with_literal_name context = Liquid::Context.new({ "blank" => { "x" => "result" } }) assert_equal("result", context.evaluate(Liquid::C::Expression.strict_parse("blank.x"))) assert_equal("result", context.evaluate(Liquid::C::Expression.strict_parse('blank["x"]'))) end def test_const_range assert_equal((1..2), Liquid::C::Expression.strict_parse("(1..2)")) end def test_dynamic_range context = Liquid::Context.new({ "var" => 42 }) expr = Liquid::C::Expression.strict_parse("(1..var)") assert_instance_of(Liquid::C::Expression, expr) assert_equal((1..42), context.evaluate(expr)) end def test_disassemble expression = Liquid::C::Expression.strict_parse("foo.bar[123]") assert_equal(<<~ASM, expression.disassemble) 0x0000: find_static_var("foo") 0x0003: lookup_const_key("bar") 0x0006: push_int8(123) 0x0008: lookup_key 0x0009: leave ASM end def test_disassemble_int16 assert_equal(<<~ASM, Liquid::C::Expression.strict_parse("[12345]").disassemble) 0x0000: push_int16(12345) 0x0003: find_var 0x0004: leave ASM end def test_disable_c_nodes context = Liquid::Context.new({ "x" => 123 }) expr = Liquid::ParseContext.new.parse_expression("x") assert_instance_of(Liquid::C::Expression, expr) assert_equal(123, context.evaluate(expr)) expr = Liquid::ParseContext.new(disable_liquid_c_nodes: true).parse_expression("x") assert_instance_of(Liquid::VariableLookup, expr) assert_equal(123, context.evaluate(expr)) end private class ReturnKeyDrop < Liquid::Drop def liquid_method_missing(key) key end end def compile_and_eval(source) context = Liquid::Context.new({ "ret_key" => ReturnKeyDrop.new }) expr = Liquid::C::Expression.strict_parse("ret_key[#{source}]") context.evaluate(expr) end end liquid-c-4.1.0/test/unit/gc_stress_test.rb000066400000000000000000000010621417631740700205520ustar00rootroot00000000000000# encoding: utf-8 # frozen_string_literal: true require "test_helper" # Help catch bugs from objects not being marked at all # GC opportunities. class GCStressTest < Minitest::Test def test_compile_and_render source = "{% assign x = 1 %}{% if x -%} x: {{ x | plus: 2 }}{% endif %}" result = gc_stress do Liquid::Template.parse(source).render! end assert_equal("x: 3", result) end private def gc_stress old_value = GC.stress GC.stress = true begin yield ensure GC.stress = old_value end end end liquid-c-4.1.0/test/unit/raw_test.rb000066400000000000000000000007051417631740700173520ustar00rootroot00000000000000# frozen_string_literal: true require "test_helper" class RawTest < Minitest::Test class RawWrapper < Liquid::Raw def render_to_output_buffer(_context, output) output << "<" super output << ">" end end Liquid::Template.register_tag("raw_wrapper", RawWrapper) def test_derived_class output = Liquid::Template.parse("{% raw_wrapper %}body{% endraw_wrapper %}").render! assert_equal("", output) end end liquid-c-4.1.0/test/unit/resource_limits_test.rb000066400000000000000000000026421417631740700217730ustar00rootroot00000000000000# frozen_string_literal: true require "test_helper" class ResourceLimitsTest < Minitest::Test def test_increment_render_score resource_limits = Liquid::ResourceLimits.new(render_score_limit: 5) resource_limits.increment_render_score(4) assert_raises(Liquid::MemoryError) do resource_limits.increment_render_score(2) end assert_equal(6, resource_limits.render_score) end def test_increment_assign_score resource_limits = Liquid::ResourceLimits.new(assign_score_limit: 5) resource_limits.increment_assign_score(5) assert_raises(Liquid::MemoryError) do resource_limits.increment_assign_score(1) end assert_equal(6, resource_limits.assign_score) end def test_increment_write_score resource_limits = Liquid::ResourceLimits.new(render_length_limit: 5) output = "a" * 10 assert_raises(Liquid::MemoryError) do resource_limits.increment_write_score(output) end end def test_raise_limits_reached resource_limits = Liquid::ResourceLimits.new({}) assert_raises(Liquid::MemoryError) do resource_limits.raise_limits_reached end assert(resource_limits.reached?) end def test_with_capture resource_limits = Liquid::ResourceLimits.new(assign_score_limit: 5) output = "foo" resource_limits.with_capture do resource_limits.increment_write_score(output) end assert_equal(3, resource_limits.assign_score) end end liquid-c-4.1.0/test/unit/tokenizer_test.rb000077500000000000000000000074621417631740700206050ustar00rootroot00000000000000# encoding: utf-8 # frozen_string_literal: true require "test_helper" class TokenizerTest < Minitest::Test def test_tokenizer_nil tokenizer = new_tokenizer(nil) assert_nil(tokenizer.send(:shift)) end def test_tokenize_strings assert_equal([" "], tokenize(" ")) assert_equal(["hello world"], tokenize("hello world")) end def test_tokenize_variables assert_equal(["{{funk}}"], tokenize("{{funk}}")) assert_equal([" ", "{{funk}}", " "], tokenize(" {{funk}} ")) assert_equal([" ", "{{funk}}", " ", "{{so}}", " ", "{{brother}}", " "], tokenize(" {{funk}} {{so}} {{brother}} ")) assert_equal([" ", "{{ funk }}", " "], tokenize(" {{ funk }} ")) # Doesn't strip whitespace assert_equal([" ", " funk ", " "], tokenize(" {{ funk }} ", trimmed: true)) end def test_tokenize_blocks assert_equal(["{%comment%}"], tokenize("{%comment%}")) assert_equal([" ", "{%comment%}", " "], tokenize(" {%comment%} ")) assert_equal([" ", "{%comment%}", " ", "{%endcomment%}", " "], tokenize(" {%comment%} {%endcomment%} ")) assert_equal([" ", "{% comment %}", " ", "{% endcomment %}", " "], tokenize(" {% comment %} {% endcomment %} ")) # Doesn't strip whitespace assert_equal([" ", " comment ", " "], tokenize(" {% comment %} ", trimmed: true)) end def test_tokenize_for_liquid_tag source = "\nfunk\n\n so | brother \n" assert_equal(["", "funk", "", " so | brother "], tokenize(source, for_liquid_tag: true)) # Strips whitespace assert_equal(["", "funk", "", "so | brother"], tokenize(source, for_liquid_tag: true, trimmed: true)) end def test_invalid_tags assert_equal([""], tokenize("{%-%}", trimmed: true)) assert_equal([""], tokenize("{{-}}", trimmed: true)) end def test_utf8_encoded_source source = "auswählen" assert_equal(Encoding::UTF_8, source.encoding) output = tokenize(source) assert_equal([Encoding::UTF_8], output.map(&:encoding)) assert_equal([source], output) end def test_utf8_compatible_source source = String.new("ascii", encoding: Encoding::ASCII) tokenizer = new_tokenizer(source) output = tokenizer.send(:shift) assert_equal(Encoding::UTF_8, output.encoding) assert_equal(source, output) assert_nil(tokenizer.send(:shift)) end def test_non_utf8_compatible_source source = "üñicode".dup.force_encoding(Encoding::BINARY) # rubocop:disable Performance/UnfreezeString exc = assert_raises(Encoding::CompatibilityError) do Liquid::C::Tokenizer.new(source, 1, false) end assert_equal("non-UTF8 encoded source (ASCII-8BIT) not supported", exc.message) end def test_source_too_large too_large_source = "a" * 2**24 max_length_source = too_large_source.chop # C safety check err = assert_raises(ArgumentError) do Liquid::C::Tokenizer.new(too_large_source, 1, false) end assert_match(/Source too large, max \d+ bytes/, err.message) # ruby patch fallback parse_context = Liquid::ParseContext.new liquid_c_tokenizer = parse_context.new_tokenizer(max_length_source) assert_instance_of(Liquid::C::Tokenizer, liquid_c_tokenizer) refute(parse_context.liquid_c_nodes_disabled?) parse_context = Liquid::ParseContext.new fallback_tokenizer = parse_context.new_tokenizer(too_large_source) assert_instance_of(Liquid::Tokenizer, fallback_tokenizer) assert_equal(true, parse_context.liquid_c_nodes_disabled?) end private def new_tokenizer(source, parse_context: Liquid::ParseContext.new) parse_context.new_tokenizer(source) end def tokenize(source, for_liquid_tag: false, trimmed: false) tokenizer = Liquid::C::Tokenizer.new(source, 1, for_liquid_tag) tokens = [] while (t = trimmed ? tokenizer.send(:shift_trimmed) : tokenizer.send(:shift)) tokens << t end tokens end end liquid-c-4.1.0/test/unit/variable_test.rb000066400000000000000000000220231417631740700203430ustar00rootroot00000000000000# encoding: utf-8 # frozen_string_literal: true require "test_helper" class VariableTest < Minitest::Test def test_variable_parse assert_equal("world", variable_strict_parse("hello").render!({ "hello" => "world" })) assert_equal("world", variable_strict_parse('"world"').render!) assert_equal("answer", variable_strict_parse('hello["world"]').render!({ "hello" => { "world" => "answer" } })) assert_equal("answer", variable_strict_parse("question?").render!({ "question?" => "answer" })) assert_equal("value", variable_strict_parse("[meta]").render!({ "meta" => "key", "key" => "value" })) assert_equal("result", variable_strict_parse("a-b").render!({ "a-b" => "result" })) assert_equal("result", variable_strict_parse("a-2").render!({ "a-2" => "result" })) end def test_strictness assert_raises(Liquid::SyntaxError) { variable_strict_parse(' hello["world\']" ') } assert_raises(Liquid::SyntaxError) { variable_strict_parse(" -..") } assert_raises(Liquid::SyntaxError) { variable_strict_parse("question?mark") } assert_raises(Liquid::SyntaxError) { variable_strict_parse("123.foo") } assert_raises(Liquid::SyntaxError) { variable_strict_parse(" | nothing") } ["a -b", "a- b", "a - b"].each do |var| assert_raises(Liquid::SyntaxError) { variable_strict_parse(var) } end end def test_literals assert_equal("", variable_strict_parse("").render!) assert_equal("true", variable_strict_parse("true").render!) assert_equal("", variable_strict_parse("nil").render!) assert_equal("123.4", variable_strict_parse("123.4").render!) assert_equal("blank_value", variable_strict_parse("[blank]").render!({ "" => "blank_value" })) assert_equal("result", variable_strict_parse("[true][blank]").render!({ true => { "" => "result" } })) assert_equal("result", variable_strict_parse('x["size"]').render!({ "x" => { "size" => "result" } })) assert_equal("result", variable_strict_parse("blank.x").render!({ "blank" => { "x" => "result" } })) assert_equal("result", variable_strict_parse('blank["x"]').render!({ "blank" => { "x" => "result" } })) end module InspectCallFilters def filter1(input, *args) inspect_call(__method__, input, args) end def filter2(input, *args) inspect_call(__method__, input, args) end private def inspect_call(filter_name, input, args) "{ filter: #{filter_name.inspect}, input: #{input.inspect}, args: #{args.inspect} }" end end def test_variable_filter context = { "name" => "Bob" } filter1_output = variable_strict_parse("name | filter1").render!(context, filters: [InspectCallFilters]) assert_equal('{ filter: :filter1, input: "Bob", args: [] }', filter1_output) filter2_output = variable_strict_parse("name | filter1 | filter2").render!(context, filters: [InspectCallFilters]) assert_equal("{ filter: :filter2, input: #{filter1_output.inspect}, args: [] }", filter2_output) end def test_variable_filter_args context = { "name" => "Bob", "abc" => "xyz" } render_opts = { filters: [InspectCallFilters] } filter1_output = variable_strict_parse("name | filter1: abc").render!(context, render_opts) assert_equal('{ filter: :filter1, input: "Bob", args: ["xyz"] }', filter1_output) filter2_output = variable_strict_parse("name | filter1: abc | filter2: abc").render!(context, render_opts) assert_equal("{ filter: :filter2, input: #{filter1_output.inspect}, args: [\"xyz\"] }", filter2_output) context = { "name" => "Bob", "a" => 1, "c" => 3, "e" => 5 } output = variable_strict_parse("name | filter1 : a , b : c , d : e").render!(context, render_opts) assert_equal('{ filter: :filter1, input: "Bob", args: [1, {"b"=>3, "d"=>5}] }', output) assert_raises(Liquid::SyntaxError) do variable_strict_parse("name | filter : a : b : c : d : e") end end def test_unicode_strings string_content = "å߀êùidhtлsԁѵ߀ráƙìstɦeƅêstpcmáѕterrãcêcհèrr" assert_equal(string_content, variable_strict_parse("\"#{string_content}\"").render!) end def test_broken_unicode_errors err = assert_raises(Liquid::SyntaxError) do Liquid::Template.parse("test {{ \xC2\xA0 test }}", error_mode: :strict) end assert(err.message) end def test_callbacks variable_fallbacks = 0 callbacks = { variable_fallback: lambda { variable_fallbacks += 1 }, } Liquid::Template.parse("{{abc}}", error_mode: :lax, stats_callbacks: callbacks) assert_equal(0, variable_fallbacks) Liquid::Template.parse("{{@!#}}", error_mode: :lax, stats_callbacks: callbacks) assert_equal(1, variable_fallbacks) end def test_write_string output = Liquid::Template.parse("{{ str }}").render({ "str" => "foo" }) assert_equal("foo", output) end def test_write_fixnum output = Liquid::Template.parse("{{ num }}").render({ "num" => 123456 }) assert_equal("123456", output) end def test_write_array output = Liquid::Template.parse("{{ ary }}").render({ "ary" => ["foo", 123, ["nested", "ary"], nil, 0.5] }) assert_equal("foo123nestedary0.5", output) end def test_write_nil output = Liquid::Template.parse("{{ obj }}").render({ "obj" => nil }) assert_equal("", output) end class StringConvertible def initialize(as_string) @as_string = as_string end def to_s @as_string end def to_liquid self end end def test_write_to_s_convertible_object output = Liquid::Template.parse("{{ obj }}").render!({ "obj" => StringConvertible.new("foo") }) assert_equal("foo", output) end def test_write_object_with_broken_to_s template = Liquid::Template.parse("{{ obj }}") exc = assert_raises(TypeError) do template.render!({ "obj" => StringConvertible.new(123) }) end assert_equal( "VariableTest::StringConvertible#to_s returned a non-String convertible value of type Integer", exc.message ) end class DerivedString < String def to_s self end end def test_write_derived_string output = Liquid::Template.parse("{{ obj }}").render!({ "obj" => DerivedString.new("bar") }) assert_equal("bar", output) end def test_filter_without_args output = Liquid::Template.parse("{{ var | upcase }}").render({ "var" => "Hello" }) assert_equal("HELLO", output) end def test_filter_with_const_arg output = Liquid::Template.parse("{{ x | plus: 2 }}").render({ "x" => 3 }) assert_equal("5", output) end def test_filter_with_variable_arg output = Liquid::Template.parse("{{ x | plus: y }}").render({ "x" => 10, "y" => 123 }) assert_equal("133", output) end def test_filter_with_variable_arg_after_const_arg output = Liquid::Template.parse("{{ ary | slice: 1, 2 }}").render({ "ary" => [1, 2, 3, 4] }) assert_equal("23", output) end def test_filter_with_const_keyword_arg output = Liquid::Template.parse("{{ value | default: 'None' }}").render({ "value" => false }) assert_equal("None", output) output = Liquid::Template.parse("{{ value | default: 'None', allow_false: true }}").render({ "value" => false }) assert_equal("false", output) end def test_filter_with_variable_keyword_arg template = Liquid::Template.parse("{{ value | default: 'None', allow_false: false_allowed }}") assert_equal("None", template.render({ "value" => false, "false_allowed" => false })) assert_equal("false", template.render({ "value" => false, "false_allowed" => true })) end def test_filter_error output = Liquid::Template.parse("before ({{ ary | concat: 2 }}) after").render({ "ary" => [1] }) assert_equal("before (Liquid error: concat filter requires an array argument) after", output) end def test_render_variable_object variable = Liquid::Variable.new("ary | concat: ary2", Liquid::ParseContext.new) assert_instance_of(Liquid::C::VariableExpression, variable.name) context = Liquid::Context.new("ary" => [1], "ary2" => [2]) assert_equal([1, 2], variable.render(context)) context["ary2"] = 2 exc = assert_raises(Liquid::ArgumentError) do variable.render(context) end assert_equal("Liquid error: concat filter requires an array argument", exc.message) end def test_filter_argument_error_translation variable = Liquid::Variable.new("'some words' | split", Liquid::ParseContext.new) context = Liquid::Context.new exc = assert_raises(Liquid::ArgumentError) { variable.render(context) } assert_equal("Liquid error: wrong number of arguments (given 1, expected 2)", exc.message) end class IntegerDrop < Liquid::Drop def initialize(value) super() @value = value.to_i end def to_liquid_value @value end end def test_to_liquid_value_on_variable_lookup context = { "number" => IntegerDrop.new("1"), "list" => [1, 2, 3, 4, 5], } output = variable_strict_parse("list[number]").render!(context) assert_equal("2", output) end private def variable_strict_parse(markup) Liquid::Template.parse("{{#{markup}}}", error_mode: :strict) end end