pax_global_header00006660000000000000000000000064150070531760014516gustar00rootroot0000000000000052 comment=742a3809b2081c7b8e9ca4d2270ab3c511a3470d xgrammar-0.1.19/000077500000000000000000000000001500705317600134245ustar00rootroot00000000000000xgrammar-0.1.19/.clang-format000066400000000000000000000002571500705317600160030ustar00rootroot00000000000000BasedOnStyle: Google DerivePointerAlignment: false ColumnLimit: 100 PointerAlignment: Left AlignAfterOpenBracket: BlockIndent BinPackArguments: false BinPackParameters: false xgrammar-0.1.19/.cmake-format.yaml000066400000000000000000000001731500705317600167350ustar00rootroot00000000000000format: line_width: 100 tab_size: 2 dangle_parens: true command_case: lower keyword_case: upper autosort: true xgrammar-0.1.19/.github/000077500000000000000000000000001500705317600147645ustar00rootroot00000000000000xgrammar-0.1.19/.github/workflows/000077500000000000000000000000001500705317600170215ustar00rootroot00000000000000xgrammar-0.1.19/.github/workflows/benchmark.yaml000066400000000000000000000022151500705317600216370ustar00rootroot00000000000000name: XGrammar Benchmark on: workflow_dispatch: schedule: - cron: '0 0 * * *' jobs: run_benchmark: name: Run XGrammar Benchmark if: github.ref == 'refs/heads/main'&& github.repository_owner == 'mlc-ai' runs-on: [self-hosted, Linux, X64] steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: 3.11 - name: Build xgrammar from source run: | python -m pip install --upgrade pip pip install . - name: Install dependencies run: | pip install torch transformers datasets tqdm requests - name: Run benchmark id: benchmark env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | python examples/benchmark/cibench_grammar_compile_mask_gen.py --num_iters 3 --num_warmup 2 --datasets all | tee benchmark_output.txt - name: Upload benchmark results uses: actions/upload-artifact@v4 with: name: benchmark-results path: benchmark_output.txt xgrammar-0.1.19/.github/workflows/build_and_release.yaml000066400000000000000000000053731500705317600233360ustar00rootroot00000000000000name: Build and upload to PyPI on: workflow_dispatch: pull_request: push: branches: - main release: types: - published jobs: build_wheels: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - {os: ubuntu-latest, arch: x86_64, build: 'cp*-manylinux*'} - {os: ubuntu-24.04-arm, arch: aarch64, build: 'cp*-manylinux*'} - {os: windows-latest, arch: AMD64, build: 'cp*'} - {os: macos-14, arch: arm64, build: 'cp*'} - {os: macos-13, arch: x86_64, build: 'cp*',} steps: - uses: astral-sh/setup-uv@v4 - uses: actions/checkout@v4 with: submodules: recursive - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: CIBW_ARCHS_MACOS: ${{ matrix.arch }} CIBW_ARCHS_LINUX: ${{ matrix.arch }} CIBW_ARCHS_WINDOWS: ${{ matrix.arch }} CIBW_BUILD: ${{ matrix.build }} CIBW_TEST_SKIP: '*' CIBW_BUILD_VERBOSITY: 1 - uses: actions/upload-artifact@v4 with: name: cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: recursive - uses: astral-sh/setup-uv@v4 - name: Build sdist run: pipx run build --sdist - name: Check metadata run: pipx run twine check dist/* - uses: actions/upload-artifact@v4 with: name: cibw-sdist path: dist/*.tar.gz upload_pypi: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest environment: pypi permissions: id-token: write attestations: write if: github.event_name == 'release' && github.event.action == 'published' # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/download-artifact@v4 with: # unpacks all CIBW artifacts into dist/ pattern: cibw-* path: dist merge-multiple: true - name: Generate artifact attestation for sdist and wheels uses: actions/attest-build-provenance@v1 with: subject-path: dist/* - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: attestations: true verbose: true # repository-url: https://test.pypi.org/legacy/ # To test: repository-url: https://test.pypi.org/legacy/ xgrammar-0.1.19/.github/workflows/close_issues.yaml000066400000000000000000000017741500705317600224160ustar00rootroot00000000000000name: Close inactive issues on: workflow_dispatch: schedule: - cron: "0 0 * * *" jobs: close-issues: runs-on: ubuntu-latest permissions: issues: write pull-requests: write steps: - uses: actions/stale@v9 with: days-before-issue-stale: 60 days-before-issue-close: 14 stale-issue-label: "stale" exempt-issue-labels: "pinned,important" stale-issue-message: > This issue has been inactive for 60 days and is marked as stale. Please confirm if this issue is still relevant by commenting or removing the 'stale' label within 14 days, otherwise it will be closed automatically. close-issue-message: > Closing this issue due to inactivity for 14 days since it was marked as stale. Feel free to reopen if you believe it's still relevant. days-before-pr-stale: -1 days-before-pr-close: -1 repo-token: ${{ secrets.GITHUB_TOKEN }} xgrammar-0.1.19/.github/workflows/documentation.yaml000066400000000000000000000021051500705317600225540ustar00rootroot00000000000000name: Build Docs on: workflow_dispatch: pull_request: push: branches: - main jobs: deploy_docs: name: Deploy Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Configuring build Environment run: | sudo apt-get update python -m pip install -U pip wheel setuptools - name: Setup Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '3.0' - name: Installing dependencies run: | python -m pip install -r docs/requirements.txt gem install jekyll jekyll-remote-theme - name: Deploying on GitHub Pages if: github.ref == 'refs/heads/main'&& github.repository_owner == 'mlc-ai' run: | git remote set-url origin https://x-access-token:${{ secrets.MLC_GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git config --global user.email "mlc-gh-actions-bot@nomail" git config --global user.name "mlc-gh-actions-bot" ./scripts/gh_deploy_site.sh xgrammar-0.1.19/.github/workflows/tmate.yaml000066400000000000000000000016711500705317600210240ustar00rootroot00000000000000on: workflow_dispatch: jobs: run_tmate_session: name: Run tmate session on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14, macos-13] python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] exclude: - os: macos-13 python: '3.13' # The reason for the exclusion is that pytorch distribution # can't be found by pip on macos-13 with python 3.13. steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Setup tmate session uses: mxschmitt/action-tmate@v3 timeout-minutes: 30 with: limit-access-to-actor: true xgrammar-0.1.19/.github/workflows/unit_test.yaml000066400000000000000000000041621500705317600217260ustar00rootroot00000000000000on: workflow_dispatch: pull_request: push: branches: - main jobs: pre_check: name: Pre-check on Ubuntu-latest runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python uses: actions/setup-python@v5 - name: Pre-commit uses: pre-commit/action@v3.0.1 - name: Ruff check uses: astral-sh/ruff-action@v3 run_unit_test: needs: [pre_check] name: Run unit tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-14, macos-13] python: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] exclude: - os: macos-13 python: '3.13' # The reason for the exclusion is that pytorch distribution # can't be found by pip on macos-13 with python 3.13. steps: - name: Checkout uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Cache huggingface uses: actions/cache@v4 with: path: | ~/.cache/huggingface key: huggingface-${{ matrix.os }}-python-${{ matrix.python }} - name: Build xgrammar from source run: | echo "set(XGRAMMAR_BUILD_CXX_TESTS ON)" >> cmake/config.cmake python -m pip install --upgrade pip pip install -v ".[test]" - name: Run C++ tests run: | ctest --test-dir build -V --timeout 60 --stop-on-failure - name: Run Python tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_HUB_DOWNLOAD_TIMEOUT: 60 if: env.HF_TOKEN != '' run: | pytest - name: Run Python tests without HF_TOKEN env: HF_TOKEN: ${{ secrets.HF_TOKEN }} if: env.HF_TOKEN == '' run: | pytest -m "not hf_token_required" xgrammar-0.1.19/.gitignore000066400000000000000000000063321500705317600154200ustar00rootroot00000000000000/tmp/ *.bak # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class .DS_Store *.S # C extensions *.so build/ *.ll .npm # Distribution / packaging .Python env/ build/ build-*/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST .conda/ # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Generated by python/gen_requirements.py python/requirements/*.txt # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ /Testing/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ docs/_staging/ # PyBuilder target/ /target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject *~ *.pyc *~ config.mk /config.cmake Win32 *.dir perf *.wasm .emscripten ## IOS DerivedData/ ## Java *.class *.worksheet *.idea *.iml *.classpath *.project *.settings */node_modules/ ## Various settings *.pbxuser !default.pbxuser *.mode1v3 !default.mode1v3 *.mode2v3 !default.mode2v3 *.perspectivev3 !default.perspectivev3 xcuserdata/ .pkl_memoize_* .emscripten* .m2 # Compiled Dynamic libraries *.so *.dylib *.dll # Compiled Object files *.slo *.lo *.o *.obj # Precompiled Headers *.gch *.pch # Compiled Static libraries *.lai *.la *.a *.lib # Executables *.exe *.out *.app ## Other *.moved-aside *.xccheckout *.xcscmblueprint .DS_Store tags cscope* *.lock # vim temporary files *.swp *.swo .bash_history # *.json *.params *.ro *.onnx *.h5 # Mac OS X .DS_Store # Jetbrain .idea .ipython .jupyter .nv .pylint.d .python_history .pytest_cache .local cmake-build-debug # Visual Studio .vs # Visual Studio Code .vscode # tmp file .nfs* # keys *.pem *.p12 *.pfx *.cer *.crt *.der # patch sentinel patched.txt # Python type checking .mypy_cache/ .pyre/ # pipenv files Pipfile Pipfile.lock # conda package artifacts conda/Dockerfile.cuda* conda/pkg .node_repl_history # nix files .envrc *.nix # Docker files .sudo_as_admin_successful # Local docs build _docs/ .config/configstore/ .ci-py-scripts/ # Used in CI to communicate between Python and Jenkins .docker-image-names/ # GDB history file .gdb_history xgrammar-0.1.19/.gitmodules000066400000000000000000000004751500705317600156070ustar00rootroot00000000000000[submodule "3rdparty/dlpack"] path = 3rdparty/dlpack url = https://github.com/dmlc/dlpack.git [submodule "3rdparty/googletest"] path = 3rdparty/googletest url = https://github.com/google/googletest.git [submodule "3rdparty/cpptrace"] path = 3rdparty/cpptrace url = https://github.com/jeremy-rifkin/cpptrace.git xgrammar-0.1.19/.pre-commit-config.yaml000066400000000000000000000033361500705317600177120ustar00rootroot00000000000000# To run for staged files: # # pre-commit run # # To run for all files: # # pre-commit run -a # # To run every time you commit in git: # # pre-commit install # # To update this file: # # pre-commit autoupdate # # See https://github.com/pre-commit/pre-commit # Note the pre-commit hooks should only be used for formatting, but not for linting. # For linting consider using CI. repos: # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-merge-conflict - id: check-symlinks - id: end-of-file-fixer - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.5.5 hooks: - id: remove-tabs - id: remove-crlf # Formatters - repo: https://github.com/psf/black-pre-commit-mirror rev: 24.1.0 hooks: - id: black - repo: https://github.com/pycqa/isort rev: 6.0.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-clang-format rev: v19.1.7 hooks: - id: clang-format types_or: [c++, c, cuda] exclude: | (?x)^(.*cubin.cpp$ | .*fmha_cubin.h | 3rdparty/.*)$ - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.13 hooks: - id: cmake-format additional_dependencies: [pyyaml>=5.1] - repo: https://github.com/google/yamlfmt rev: v0.16.0 hooks: - id: yamlfmt - repo: https://github.com/ComPWA/taplo-pre-commit rev: v0.9.3 hooks: - id: taplo-format args: ["--option", "column_width=100"] xgrammar-0.1.19/.yamlfmt000066400000000000000000000002441500705317600150760ustar00rootroot00000000000000formatter: indent: 2 retain_line_breaks_single: true max_line_length: 100 # avoid replacing newline with #magic___^_^___line scan_folded_as_literal: true xgrammar-0.1.19/3rdparty/000077500000000000000000000000001500705317600151745ustar00rootroot00000000000000xgrammar-0.1.19/3rdparty/cpptrace/000077500000000000000000000000001500705317600167755ustar00rootroot00000000000000xgrammar-0.1.19/3rdparty/dlpack/000077500000000000000000000000001500705317600164325ustar00rootroot00000000000000xgrammar-0.1.19/3rdparty/googletest/000077500000000000000000000000001500705317600173505ustar00rootroot00000000000000xgrammar-0.1.19/3rdparty/picojson/000077500000000000000000000000001500705317600170205ustar00rootroot00000000000000xgrammar-0.1.19/3rdparty/picojson/picojson.h000066400000000000000000001045721500705317600210260ustar00rootroot00000000000000/* * Copyright 2009-2010 Cybozu Labs, Inc. * Copyright 2011-2014 Kazuho Oku * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #pragma once #ifndef PICOJSON_USE_INT64 #define PICOJSON_USE_INT64 #ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS 1 #endif #endif // If PICOJSON_USE_ORDERED_OBJECT is set, picojson uses object_with_ordered_keys, which maintains // the insertion order of keys, i.e. the order of keys in the json string. // This macro is set by default. #ifndef PICOJSON_USE_ORDERED_OBJECT #define PICOJSON_USE_ORDERED_OBJECT 1 #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // for isnan/isinf #if __cplusplus >= 201103L #include #else extern "C" { #ifdef _MSC_VER #include #elif defined(__INTEL_COMPILER) #include #else #include #endif } #endif #ifndef PICOJSON_USE_RVALUE_REFERENCE #if (defined(__cpp_rvalue_references) && __cpp_rvalue_references >= 200610) || \ (defined(_MSC_VER) && _MSC_VER >= 1600) #define PICOJSON_USE_RVALUE_REFERENCE 1 #else #define PICOJSON_USE_RVALUE_REFERENCE 0 #endif #endif // PICOJSON_USE_RVALUE_REFERENCE #ifndef PICOJSON_NOEXCEPT #if PICOJSON_USE_RVALUE_REFERENCE #define PICOJSON_NOEXCEPT noexcept #else #define PICOJSON_NOEXCEPT throw() #endif #endif // experimental support for int64_t (see README.mkdn for detail) #ifdef PICOJSON_USE_INT64 #include #include #endif // to disable the use of localeconv(3), set PICOJSON_USE_LOCALE to 0 #ifndef PICOJSON_USE_LOCALE #define PICOJSON_USE_LOCALE 1 #endif #if PICOJSON_USE_LOCALE extern "C" { #include } #endif #ifndef PICOJSON_ASSERT #ifndef PICOJSON_DISABLE_EXCEPTION #define PICOJSON_ASSERT(e) \ do { \ if (!(e)) throw std::runtime_error(#e); \ } while (0) #else #define PICOJSON_ASSERT(e) \ do { \ if (!(e)) std::abort(); \ } while (0) #endif // PICOJSON_DISABLE_EXCEPTION #endif #ifdef _MSC_VER #define SNPRINTF _snprintf_s #pragma warning(push) #pragma warning(disable : 4244) // conversion from int to char #pragma warning(disable : 4127) // conditional expression is constant #pragma warning(disable : 4702) // unreachable code #else #define SNPRINTF snprintf #endif namespace picojson { enum { null_type, boolean_type, number_type, string_type, array_type, object_type #ifdef PICOJSON_USE_INT64 , int64_type #endif }; enum { INDENT_WIDTH = 2 }; struct null {}; class object_with_ordered_keys; class value { public: typedef std::vector array; #ifdef PICOJSON_USE_ORDERED_OBJECT typedef object_with_ordered_keys object; #else typedef std::unordered_map object; #endif union _storage { bool boolean_; double number_; #ifdef PICOJSON_USE_INT64 int64_t int64_; #endif std::string* string_; array* array_; object* object_; }; protected: int type_; _storage u_; public: value(); value(int type, bool); explicit value(bool b); #ifdef PICOJSON_USE_INT64 explicit value(int64_t i); #endif explicit value(double n); explicit value(const std::string& s); explicit value(const array& a); explicit value(const object& o); #if PICOJSON_USE_RVALUE_REFERENCE explicit value(std::string&& s); explicit value(array&& a); explicit value(object&& o); #endif explicit value(const char* s); value(const char* s, size_t len); ~value(); value(const value& x); value& operator=(const value& x); #if PICOJSON_USE_RVALUE_REFERENCE value(value&& x) PICOJSON_NOEXCEPT; value& operator=(value&& x) PICOJSON_NOEXCEPT; #endif void swap(value& x) PICOJSON_NOEXCEPT; template bool is() const; template const T& get() const; template T& get(); template void set(const T&); #if PICOJSON_USE_RVALUE_REFERENCE template void set(T&&); #endif bool evaluate_as_boolean() const; const value& get(const size_t idx) const; const value& get(const std::string& key) const; value& get(const size_t idx); value& get(const std::string& key); bool contains(const size_t idx) const; bool contains(const std::string& key) const; std::string to_str() const; template void serialize(Iter os, bool prettify = false) const; std::string serialize(bool prettify = false) const; private: template // NOLINTNEXTLINE(runtime/explicit) value(const T*); // intentionally defined to block implicit conversion of // pointer to bool template static void _indent(Iter os, int indent); template void _serialize(Iter os, int indent) const; std::string _serialize(int indent) const; void clear(); }; // The ordered version of hashmap. It has the same interface as std::unordered_map, but provides // ordered_keys() to return the keys in the order they were inserted. class object_with_ordered_keys : private std::unordered_map { public: using typename std::unordered_map::value_type; using typename std::unordered_map::iterator; using typename std::unordered_map::const_iterator; object_with_ordered_keys() = default; object_with_ordered_keys(const object_with_ordered_keys&) = default; object_with_ordered_keys(object_with_ordered_keys&&) = default; object_with_ordered_keys(std::initializer_list init) : std::unordered_map(init) { for (const auto& pair : init) { ordered_keys_.push_back(pair.first); } } object_with_ordered_keys& operator=(const object_with_ordered_keys&) = default; object_with_ordered_keys& operator=(object_with_ordered_keys&&) = default; using std::unordered_map::begin; using std::unordered_map::end; using std::unordered_map::cbegin; using std::unordered_map::cend; using std::unordered_map::empty; using std::unordered_map::size; using std::unordered_map::at; using std::unordered_map::count; using std::unordered_map::find; value& operator[](const std::string& key) { if (count(key) == 0) { ordered_keys_.push_back(key); } return std::unordered_map::operator[](key); } const value& operator[](const std::string& key) const { return std::unordered_map::at(key); } void clear() { std::unordered_map::clear(); ordered_keys_.clear(); } std::pair insert(const value_type& kv) { if (!count(kv.first)) { ordered_keys_.push_back(kv.first); } return std::unordered_map::insert(kv); } template std::pair emplace(Args&&... args) { return insert(value_type(std::forward(args)...)); } iterator erase(const_iterator it) { ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); return std::unordered_map::erase(it); } iterator erase(iterator it) { ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), it->first)); return std::unordered_map::erase(it); } size_t erase(const std::string& key) { if (std::unordered_map::erase(key)) { ordered_keys_.erase(std::find(ordered_keys_.begin(), ordered_keys_.end(), key)); return 1; } else { return 0; } } const std::vector& ordered_keys() const { return ordered_keys_; } friend bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs); private: std::vector ordered_keys_; }; inline bool operator==(const object_with_ordered_keys& lhs, const object_with_ordered_keys& rhs) { return static_cast&>(lhs) == static_cast&>(rhs); } typedef value::array array; typedef value::object object; inline value::value() : type_(null_type), u_() {} inline value::value(int type, bool) : type_(type), u_() { switch (type) { #define INIT(p, v) \ case p##type: \ u_.p = v; \ break INIT(boolean_, false); INIT(number_, 0.0); #ifdef PICOJSON_USE_INT64 INIT(int64_, 0); #endif INIT(string_, new std::string()); INIT(array_, new array()); INIT(object_, new object()); #undef INIT default: break; } } inline value::value(bool b) : type_(boolean_type), u_() { u_.boolean_ = b; } #ifdef PICOJSON_USE_INT64 inline value::value(int64_t i) : type_(int64_type), u_() { u_.int64_ = i; } #endif inline value::value(double n) : type_(number_type), u_() { if ( #ifdef _MSC_VER !_finite(n) #elif __cplusplus >= 201103L std::isnan(n) || std::isinf(n) #else isnan(n) || isinf(n) #endif ) { #ifndef PICOJSON_DISABLE_EXCEPTION throw std::overflow_error(""); #else std::abort(); #endif } u_.number_ = n; } inline value::value(const std::string& s) : type_(string_type), u_() { u_.string_ = new std::string(s); } inline value::value(const array& a) : type_(array_type), u_() { u_.array_ = new array(a); } inline value::value(const object& o) : type_(object_type), u_() { u_.object_ = new object(o); } #if PICOJSON_USE_RVALUE_REFERENCE inline value::value(std::string&& s) : type_(string_type), u_() { u_.string_ = new std::string(std::move(s)); } inline value::value(array&& a) : type_(array_type), u_() { u_.array_ = new array(std::move(a)); } inline value::value(object&& o) : type_(object_type), u_() { u_.object_ = new object(std::move(o)); } #endif inline value::value(const char* s) : type_(string_type), u_() { u_.string_ = new std::string(s); } inline value::value(const char* s, size_t len) : type_(string_type), u_() { u_.string_ = new std::string(s, len); } inline void value::clear() { switch (type_) { #define DEINIT(p) \ case p##type: \ delete u_.p; \ break DEINIT(string_); DEINIT(array_); DEINIT(object_); #undef DEINIT default: break; } } inline value::~value() { clear(); } inline value::value(const value& x) : type_(x.type_), u_() { switch (type_) { #define INIT(p, v) \ case p##type: \ u_.p = v; \ break INIT(string_, new std::string(*x.u_.string_)); INIT(array_, new array(*x.u_.array_)); INIT(object_, new object(*x.u_.object_)); #undef INIT default: u_ = x.u_; break; } } inline value& value::operator=(const value& x) { if (this != &x) { value t(x); swap(t); } return *this; } #if PICOJSON_USE_RVALUE_REFERENCE inline value::value(value&& x) PICOJSON_NOEXCEPT : type_(null_type), u_() { swap(x); } inline value& value::operator=(value&& x) PICOJSON_NOEXCEPT { swap(x); return *this; } #endif inline void value::swap(value& x) PICOJSON_NOEXCEPT { std::swap(type_, x.type_); std::swap(u_, x.u_); } #define IS(ctype, jtype) \ template <> \ inline bool value::is() const { \ return type_ == jtype##_type; \ } IS(null, null) IS(bool, boolean) #ifdef PICOJSON_USE_INT64 IS(int64_t, int64) #endif IS(std::string, string) IS(array, array) IS(object, object) #undef IS template <> inline bool value::is() const { return type_ == number_type #ifdef PICOJSON_USE_INT64 || type_ == int64_type #endif // NOLINTNEXTLINE(whitespace/semicolon) ; } #define GET(ctype, var) \ template <> \ inline const ctype& value::get() const { \ PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ return var; \ } \ template <> \ inline ctype& value::get() { \ PICOJSON_ASSERT("type mismatch! call is() before get()" && is()); \ return var; \ } GET(bool, u_.boolean_) GET(std::string, *u_.string_) GET(array, *u_.array_) GET(object, *u_.object_) #ifdef PICOJSON_USE_INT64 GET(double, (type_ == int64_type && (const_cast(this)->type_ = number_type, (const_cast(this)->u_.number_ = u_.int64_)), u_.number_)) GET(int64_t, u_.int64_) #else GET(double, u_.number_) #endif #undef GET #define SET(ctype, jtype, setter) \ template <> \ inline void value::set(const ctype& _val) { \ clear(); \ type_ = jtype##_type; \ setter \ } SET(bool, boolean, u_.boolean_ = _val;) SET(std::string, string, u_.string_ = new std::string(_val);) SET(array, array, u_.array_ = new array(_val);) SET(object, object, u_.object_ = new object(_val);) SET(double, number, u_.number_ = _val;) #ifdef PICOJSON_USE_INT64 SET(int64_t, int64, u_.int64_ = _val;) #endif #undef SET #if PICOJSON_USE_RVALUE_REFERENCE #define MOVESET(ctype, jtype, setter) \ template <> \ inline void value::set(ctype && _val) { \ clear(); \ type_ = jtype##_type; \ setter \ } MOVESET(std::string, string, u_.string_ = new std::string(std::move(_val));) MOVESET(array, array, u_.array_ = new array(std::move(_val));) MOVESET(object, object, u_.object_ = new object(std::move(_val));) #undef MOVESET #endif inline bool value::evaluate_as_boolean() const { switch (type_) { case null_type: return false; case boolean_type: return u_.boolean_; case number_type: return u_.number_ != 0; #ifdef PICOJSON_USE_INT64 case int64_type: return u_.int64_ != 0; #endif case string_type: return !u_.string_->empty(); default: return true; } } inline const value& value::get(const size_t idx) const { static value s_null; PICOJSON_ASSERT(is()); return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; } inline value& value::get(const size_t idx) { static value s_null; PICOJSON_ASSERT(is()); return idx < u_.array_->size() ? (*u_.array_)[idx] : s_null; } inline const value& value::get(const std::string& key) const { static value s_null; PICOJSON_ASSERT(is()); object::const_iterator i = u_.object_->find(key); return i != u_.object_->end() ? i->second : s_null; } inline value& value::get(const std::string& key) { static value s_null; PICOJSON_ASSERT(is()); object::iterator i = u_.object_->find(key); return i != u_.object_->end() ? i->second : s_null; } inline bool value::contains(const size_t idx) const { PICOJSON_ASSERT(is()); return idx < u_.array_->size(); } inline bool value::contains(const std::string& key) const { PICOJSON_ASSERT(is()); object::const_iterator i = u_.object_->find(key); return i != u_.object_->end(); } inline std::string value::to_str() const { switch (type_) { case null_type: return "null"; case boolean_type: return u_.boolean_ ? "true" : "false"; #ifdef PICOJSON_USE_INT64 case int64_type: { char buf[sizeof("-9223372036854775808")]; SNPRINTF(buf, sizeof(buf), "%" PRId64, u_.int64_); return buf; } #endif case number_type: { char buf[256]; double tmp; SNPRINTF(buf, sizeof(buf), fabs(u_.number_) < (1ULL << 53) && modf(u_.number_, &tmp) == 0 ? "%.f" : "%.17g", u_.number_); #if PICOJSON_USE_LOCALE char* decimal_point = localeconv()->decimal_point; if (strcmp(decimal_point, ".") != 0) { size_t decimal_point_len = strlen(decimal_point); for (char* p = buf; *p != '\0'; ++p) { if (strncmp(p, decimal_point, decimal_point_len) == 0) { return std::string(buf, p) + "." + (p + decimal_point_len); } } } #endif return buf; } case string_type: return *u_.string_; case array_type: return "array"; case object_type: return "object"; default: PICOJSON_ASSERT(0); #ifdef _MSC_VER __assume(0); #endif } return std::string(); } template void copy(const std::string& s, Iter oi) { std::copy(s.begin(), s.end(), oi); } template struct serialize_str_char { Iter oi; void operator()(char c) { switch (c) { #define MAP(val, sym) \ case val: \ copy(sym, oi); \ break MAP('"', "\\\""); MAP('\\', "\\\\"); MAP('/', "\\/"); MAP('\b', "\\b"); MAP('\f', "\\f"); MAP('\n', "\\n"); MAP('\r', "\\r"); MAP('\t', "\\t"); #undef MAP default: if (static_cast(c) < 0x20 || c == 0x7f) { char buf[7]; SNPRINTF(buf, sizeof(buf), "\\u%04x", c & 0xff); copy(buf, buf + 6, oi); } else { *oi++ = c; } break; } } }; template void serialize_str(const std::string& s, Iter oi) { *oi++ = '"'; serialize_str_char process_char = {oi}; std::for_each(s.begin(), s.end(), process_char); *oi++ = '"'; } template void value::serialize(Iter oi, bool prettify) const { return _serialize(oi, prettify ? 0 : -1); } inline std::string value::serialize(bool prettify) const { return _serialize(prettify ? 0 : -1); } template void value::_indent(Iter oi, int indent) { *oi++ = '\n'; for (int i = 0; i < indent * INDENT_WIDTH; ++i) { *oi++ = ' '; } } template void value::_serialize(Iter oi, int indent) const { switch (type_) { case string_type: serialize_str(*u_.string_, oi); break; case array_type: { *oi++ = '['; if (indent != -1) { ++indent; } for (array::const_iterator i = u_.array_->begin(); i != u_.array_->end(); ++i) { if (i != u_.array_->begin()) { *oi++ = ','; } if (indent != -1) { _indent(oi, indent); } i->_serialize(oi, indent); } if (indent != -1) { --indent; if (!u_.array_->empty()) { _indent(oi, indent); } } *oi++ = ']'; break; } case object_type: { *oi++ = '{'; if (indent != -1) { ++indent; } #if PICOJSON_USE_ORDERED_OBJECT for (auto i = u_.object_->ordered_keys().begin(); i != u_.object_->ordered_keys().end(); ++i) { if (i != u_.object_->ordered_keys().begin()) { *oi++ = ','; } if (indent != -1) { _indent(oi, indent); } serialize_str(*i, oi); *oi++ = ':'; if (indent != -1) { *oi++ = ' '; } u_.object_->at(*i)._serialize(oi, indent); } #else for (object::const_iterator i = u_.object_->begin(); i != u_.object_->end(); ++i) { if (i != u_.object_->begin()) { *oi++ = ','; } if (indent != -1) { _indent(oi, indent); } serialize_str(i->first, oi); *oi++ = ':'; if (indent != -1) { *oi++ = ' '; } i->second._serialize(oi, indent); } #endif if (indent != -1) { --indent; if (!u_.object_->empty()) { _indent(oi, indent); } } *oi++ = '}'; break; } default: copy(to_str(), oi); break; } if (indent == 0) { *oi++ = '\n'; } } inline std::string value::_serialize(int indent) const { std::string s; _serialize(std::back_inserter(s), indent); return s; } template class input { protected: Iter cur_, end_; bool consumed_; int line_; public: input(const Iter& first, const Iter& last) : cur_(first), end_(last), consumed_(false), line_(1) {} int getc() { if (consumed_) { if (*cur_ == '\n') { ++line_; } ++cur_; } if (cur_ == end_) { consumed_ = false; return -1; } consumed_ = true; return *cur_ & 0xff; } void ungetc() { consumed_ = false; } Iter cur() const { if (consumed_) { input* self = const_cast*>(this); self->consumed_ = false; ++self->cur_; } return cur_; } int line() const { return line_; } void skip_ws() { while (1) { int ch = getc(); if (!(ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r')) { ungetc(); break; } } } bool expect(const int expected) { skip_ws(); if (getc() != expected) { ungetc(); return false; } return true; } bool match(const std::string& pattern) { for (std::string::const_iterator pi(pattern.begin()); pi != pattern.end(); ++pi) { if (getc() != *pi) { ungetc(); return false; } } return true; } }; template // NOLINTNEXTLINE(runtime/references) inline int _parse_quadhex(input& in) { int uni_ch = 0, hex; for (int i = 0; i < 4; i++) { if ((hex = in.getc()) == -1) { return -1; } if ('0' <= hex && hex <= '9') { hex -= '0'; } else if ('A' <= hex && hex <= 'F') { hex -= 'A' - 0xa; } else if ('a' <= hex && hex <= 'f') { hex -= 'a' - 0xa; } else { in.ungetc(); return -1; } uni_ch = uni_ch * 16 + hex; } return uni_ch; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_codepoint(String& out, input& in) { int uni_ch; if ((uni_ch = _parse_quadhex(in)) == -1) { return false; } if (0xd800 <= uni_ch && uni_ch <= 0xdfff) { if (0xdc00 <= uni_ch) { // a second 16-bit of a surrogate pair appeared return false; } // first 16-bit of surrogate pair, get the next one if (in.getc() != '\\' || in.getc() != 'u') { in.ungetc(); return false; } int second = _parse_quadhex(in); if (!(0xdc00 <= second && second <= 0xdfff)) { return false; } uni_ch = ((uni_ch - 0xd800) << 10) | ((second - 0xdc00) & 0x3ff); uni_ch += 0x10000; } if (uni_ch < 0x80) { out.push_back(static_cast(uni_ch)); } else { if (uni_ch < 0x800) { out.push_back(static_cast(0xc0 | (uni_ch >> 6))); } else { if (uni_ch < 0x10000) { out.push_back(static_cast(0xe0 | (uni_ch >> 12))); } else { out.push_back(static_cast(0xf0 | (uni_ch >> 18))); out.push_back(static_cast(0x80 | ((uni_ch >> 12) & 0x3f))); } out.push_back(static_cast(0x80 | ((uni_ch >> 6) & 0x3f))); } out.push_back(static_cast(0x80 | (uni_ch & 0x3f))); } return true; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_string(String& out, input& in) { while (1) { int ch = in.getc(); if (ch < ' ') { in.ungetc(); return false; } else if (ch == '"') { return true; } else if (ch == '\\') { if ((ch = in.getc()) == -1) { return false; } switch (ch) { #define MAP(sym, val) \ case sym: \ out.push_back(val); \ break MAP('"', '\"'); MAP('\\', '\\'); MAP('/', '/'); MAP('b', '\b'); MAP('f', '\f'); MAP('n', '\n'); MAP('r', '\r'); MAP('t', '\t'); #undef MAP case 'u': if (!_parse_codepoint(out, in)) { return false; } break; default: return false; } } else { out.push_back(static_cast(ch)); } } return false; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_array(Context& ctx, input& in) { if (!ctx.parse_array_start()) { return false; } size_t idx = 0; if (in.expect(']')) { return ctx.parse_array_stop(idx); } do { if (!ctx.parse_array_item(in, idx)) { return false; } idx++; } while (in.expect(',')); return in.expect(']') && ctx.parse_array_stop(idx); } template // NOLINTNEXTLINE(runtime/references) inline bool _parse_object(Context& ctx, input& in) { if (!ctx.parse_object_start()) { return false; } if (in.expect('}')) { return true; } do { std::string key; if (!in.expect('"') || !_parse_string(key, in) || !in.expect(':')) { return false; } if (!ctx.parse_object_item(in, key)) { return false; } } while (in.expect(',')); return in.expect('}'); } template // NOLINTNEXTLINE(runtime/references) inline std::string _parse_number(input& in) { std::string num_str; while (1) { int ch = in.getc(); if (('0' <= ch && ch <= '9') || ch == '+' || ch == '-' || ch == 'e' || ch == 'E') { num_str.push_back(static_cast(ch)); } else if (ch == '.') { #if PICOJSON_USE_LOCALE num_str += localeconv()->decimal_point; #else num_str.push_back('.'); #endif } else { in.ungetc(); break; } } return num_str; } template // NOLINTNEXTLINE(runtime/references) inline bool _parse(Context& ctx, input& in) { in.skip_ws(); int ch = in.getc(); switch (ch) { #define IS(ch, text, op) \ case ch: \ if (in.match(text) && op) { \ return true; \ } else { \ return false; \ } IS('n', "ull", ctx.set_null()); IS('f', "alse", ctx.set_bool(false)); IS('t', "rue", ctx.set_bool(true)); #undef IS case '"': return ctx.parse_string(in); case '[': return _parse_array(ctx, in); case '{': return _parse_object(ctx, in); default: if (('0' <= ch && ch <= '9') || ch == '-') { double f; char* endp; in.ungetc(); std::string num_str(_parse_number(in)); if (num_str.empty()) { return false; } #ifdef PICOJSON_USE_INT64 { errno = 0; intmax_t ival = strtoimax(num_str.c_str(), &endp, 10); if (errno == 0 && std::numeric_limits::min() <= ival && ival <= std::numeric_limits::max() && endp == num_str.c_str() + num_str.size()) { ctx.set_int64(ival); return true; } } #endif f = strtod(num_str.c_str(), &endp); if (endp == num_str.c_str() + num_str.size()) { ctx.set_number(f); return true; } return false; } break; } in.ungetc(); return false; } class deny_parse_context { public: bool set_null() { return false; } bool set_bool(bool) { return false; } #ifdef PICOJSON_USE_INT64 bool set_int64(int64_t) { return false; } #endif bool set_number(double) { return false; } template bool parse_string(input&) { return false; } bool parse_array_start() { return false; } template bool parse_array_item(input&, size_t) { return false; } bool parse_array_stop(size_t) { return false; } bool parse_object_start() { return false; } template bool parse_object_item(input&, const std::string&) { return false; } }; class default_parse_context { protected: value* out_; public: // NOLINTNEXTLINE(runtime/explicit) default_parse_context(value* out) : out_(out) {} bool set_null() { *out_ = value(); return true; } bool set_bool(bool b) { *out_ = value(b); return true; } #ifdef PICOJSON_USE_INT64 bool set_int64(int64_t i) { *out_ = value(i); return true; } #endif bool set_number(double f) { *out_ = value(f); return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_string(input& in) { *out_ = value(string_type, false); return _parse_string(out_->get(), in); } bool parse_array_start() { *out_ = value(array_type, false); return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_array_item(input& in, size_t) { array& a = out_->get(); a.push_back(value()); default_parse_context ctx(&a.back()); return _parse(ctx, in); } bool parse_array_stop(size_t) { return true; } bool parse_object_start() { *out_ = value(object_type, false); return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_object_item(input& in, const std::string& key) { object& o = out_->get(); default_parse_context ctx(&o[key]); return _parse(ctx, in); } private: default_parse_context(const default_parse_context&); default_parse_context& operator=(const default_parse_context&); }; class null_parse_context { public: struct dummy_str { void push_back(int) {} }; public: null_parse_context() {} bool set_null() { return true; } bool set_bool(bool) { return true; } #ifdef PICOJSON_USE_INT64 bool set_int64(int64_t) { return true; } #endif bool set_number(double) { return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_string(input& in) { dummy_str s; return _parse_string(s, in); } bool parse_array_start() { return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_array_item(input& in, size_t) { return _parse(*this, in); } bool parse_array_stop(size_t) { return true; } bool parse_object_start() { return true; } template // NOLINTNEXTLINE(runtime/references) bool parse_object_item(input& in, const std::string&) { return _parse(*this, in); } private: null_parse_context(const null_parse_context&); null_parse_context& operator=(const null_parse_context&); }; // obsolete, use the version below template // NOLINTNEXTLINE(runtime/references) inline std::string parse(value& out, Iter& pos, const Iter& last) { std::string err; pos = parse(out, pos, last, &err); return err; } template // NOLINTNEXTLINE(runtime/references) inline Iter _parse(Context& ctx, const Iter& first, const Iter& last, std::string* err) { input in(first, last); if (!_parse(ctx, in) && err != NULL) { char buf[64]; SNPRINTF(buf, sizeof(buf), "syntax error at line %d near: ", in.line()); *err = buf; while (1) { int ch = in.getc(); if (ch == -1 || ch == '\n') { break; } else if (ch >= ' ') { err->push_back(static_cast(ch)); } } } return in.cur(); } template // NOLINTNEXTLINE(runtime/references) inline Iter parse(value& out, const Iter& first, const Iter& last, std::string* err) { default_parse_context ctx(&out); return _parse(ctx, first, last, err); } // NOLINTNEXTLINE(runtime/references) inline std::string parse(value& out, const std::string& s) { std::string err; parse(out, s.begin(), s.end(), &err); return err; } // NOLINTNEXTLINE(runtime/references) inline std::string parse(value& out, std::istream& is) { std::string err; parse(out, std::istreambuf_iterator(is.rdbuf()), std::istreambuf_iterator(), &err); return err; } template struct last_error_t { static std::string s; }; template // NOLINTNEXTLINE(runtime/string) std::string last_error_t::s; inline void set_last_error(const std::string& s) { last_error_t::s = s; } inline const std::string& get_last_error() { return last_error_t::s; } inline bool operator==(const value& x, const value& y) { if (x.is()) return y.is(); #define PICOJSON_CMP(type) \ if (x.is()) return y.is() && x.get() == y.get() PICOJSON_CMP(bool); PICOJSON_CMP(double); PICOJSON_CMP(std::string); PICOJSON_CMP(array); PICOJSON_CMP(object); #undef PICOJSON_CMP PICOJSON_ASSERT(0); #ifdef _MSC_VER __assume(0); #endif return false; } inline bool operator!=(const value& x, const value& y) { return !(x == y); } } // namespace picojson #if !PICOJSON_USE_RVALUE_REFERENCE namespace std { template <> inline void swap(picojson::value& x, picojson::value& y) { x.swap(y); } } // namespace std #endif inline std::istream& operator>>(std::istream& is, picojson::value& x) { picojson::set_last_error(std::string()); const std::string err(picojson::parse(x, is)); if (!err.empty()) { picojson::set_last_error(err); is.setstate(std::ios::failbit); } return is; } inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) { x.serialize(std::ostream_iterator(os)); return os; } #ifdef _MSC_VER #pragma warning(pop) #endif xgrammar-0.1.19/3rdparty/picojson/test_picojson.cpp000066400000000000000000000053401500705317600224110ustar00rootroot00000000000000/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include #include #include "picojson.h" using picojson::object_with_ordered_keys; void test_constructor() { object_with_ordered_keys obj; obj["foo"] = picojson::value(true); assert((obj.ordered_keys() == std::vector{"foo"})); object_with_ordered_keys obj1{{"foo", picojson::value(true)}, {"bar", picojson::value(false)}}; assert((obj1.ordered_keys() == std::vector{"foo", "bar"})); object_with_ordered_keys obj2(obj1); assert((obj2.ordered_keys() == std::vector{"foo", "bar"})); object_with_ordered_keys obj3(std::move(obj2)); assert((obj3.ordered_keys() == std::vector{"foo", "bar"})); obj = obj3; assert((obj.ordered_keys() == std::vector{"foo", "bar"})); } void test_modifier() { object_with_ordered_keys obj{{"foo", picojson::value(true)}, {"bar", picojson::value(false)}}; obj.insert({"abc", picojson::value(false)}); assert((obj.ordered_keys() == std::vector{"foo", "bar", "abc"})); obj.emplace("def", picojson::value(true)); assert((obj.ordered_keys() == std::vector{"foo", "bar", "abc", "def"})); obj.insert({"abc", picojson::value(true)}); assert((obj.ordered_keys() == std::vector{"foo", "bar", "abc", "def"})); auto it = obj.find("abc"); it = obj.erase(it); assert((obj.ordered_keys() == std::vector{"foo", "bar", "def"})); obj.erase("foo"); assert((obj.ordered_keys() == std::vector{"bar", "def"})); obj.clear(); assert((obj.ordered_keys() == std::vector{})); } void test_serializer() { picojson::object obj; obj["bar"] = picojson::value(static_cast(10)); obj["baz"] = picojson::value(10.5); obj["foo"] = picojson::value(true); picojson::value v(obj); assert((v.serialize(false) == "{\"bar\":10,\"baz\":10.5,\"foo\":true}")); } int main() { test_constructor(); test_modifier(); test_serializer(); return 0; } xgrammar-0.1.19/CMakeLists.txt000066400000000000000000000065061500705317600161730ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.18) project(xgrammar LANGUAGES CXX) if(EXISTS ${CMAKE_BINARY_DIR}/config.cmake) message(STATUS "Config file: ${CMAKE_BINARY_DIR}/config.cmake") include(${CMAKE_BINARY_DIR}/config.cmake) elseif(EXISTS ${PROJECT_SOURCE_DIR}/config.cmake) message(STATUS "Config file: ${PROJECT_SOURCE_DIR}/config.cmake") include(${PROJECT_SOURCE_DIR}/config.cmake) elseif(EXISTS ${PROJECT_SOURCE_DIR}/cmake/config.cmake) message(STATUS "Config file: ${PROJECT_SOURCE_DIR}/cmake/config.cmake") include(${PROJECT_SOURCE_DIR}/cmake/config.cmake) else() message(STATUS "No config.cmake found. Using the default config") endif() option(XGRAMMAR_BUILD_PYTHON_BINDINGS "Build Python bindings" ON) option(XGRAMMAR_BUILD_CXX_TESTS "Build C++ tests" OFF) option(XGRAMMAR_ENABLE_CPPTRACE "Enable C++ trace (Now only support Linux, and RelWithDebugInfo or Debug build)" OFF ) set(XGRAMMAR_CUDA_ARCHITECTURES native CACHE STRING "CUDA architectures" ) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) if(NOT CMAKE_BUILD_TYPE) message(STATUS "No build type specified; defaulting to CMAKE_BUILD_TYPE=RelWithDebugInfo.") set(CMAKE_BUILD_TYPE "RelWithDebugInfo" CACHE STRING "The build type" FORCE ) endif() message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") message(STATUS "Build Python bindings: ${XGRAMMAR_BUILD_PYTHON_BINDINGS}") message(STATUS "Build C++ tests: ${XGRAMMAR_BUILD_CXX_TESTS}") message(STATUS "CUDA architectures: ${XGRAMMAR_CUDA_ARCHITECTURES}") message(STATUS "Enable C++ trace: ${XGRAMMAR_ENABLE_CPPTRACE}") if(MSVC) set(CMAKE_CXX_FLAGS "/Wall ${CMAKE_CXX_FLAGS}") else() if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") set(CMAKE_CXX_FLAGS "-O3 ${CMAKE_CXX_FLAGS}") endif() set(CMAKE_CXX_FLAGS "-Wall -Wextra -Werror -Wno-pedantic -Wno-unused-parameter \ -Woverloaded-virtual -flto=auto ${CMAKE_CXX_FLAGS}" ) endif() set(XGRAMMAR_INCLUDE_PATH ${PROJECT_SOURCE_DIR}/3rdparty/picojson ${PROJECT_SOURCE_DIR}/3rdparty/dlpack/include ) file(GLOB_RECURSE XGRAMMAR_SOURCES_PATH "${PROJECT_SOURCE_DIR}/cpp/*.cc") list(FILTER XGRAMMAR_SOURCES_PATH EXCLUDE REGEX "${PROJECT_SOURCE_DIR}/cpp/nanobind/.*\\.cc") add_library(xgrammar STATIC ${XGRAMMAR_SOURCES_PATH}) target_include_directories(xgrammar PUBLIC include) target_include_directories(xgrammar SYSTEM PUBLIC ${XGRAMMAR_INCLUDE_PATH}) # link to cpptrace if(XGRAMMAR_ENABLE_CPPTRACE) add_subdirectory(${PROJECT_SOURCE_DIR}/3rdparty/cpptrace) target_link_libraries(xgrammar PUBLIC cpptrace::cpptrace) target_compile_definitions(xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=1) else() target_compile_definitions(xgrammar PUBLIC XGRAMMAR_ENABLE_CPPTRACE=0) endif() if(XGRAMMAR_BUILD_PYTHON_BINDINGS) add_subdirectory(${PROJECT_SOURCE_DIR}/cpp/nanobind) install(TARGETS xgrammar_bindings DESTINATION .) endif() if(XGRAMMAR_BUILD_CXX_TESTS) add_subdirectory(${PROJECT_SOURCE_DIR}/3rdparty/googletest) file(GLOB_RECURSE XGRAMMAR_TEST_SOURCES_PATH "${PROJECT_SOURCE_DIR}/tests/cpp/*.cc") enable_testing() add_executable(xgrammar_test ${XGRAMMAR_TEST_SOURCES_PATH}) target_include_directories(xgrammar_test PUBLIC ${PROJECT_SOURCE_DIR}/cpp) target_link_libraries(xgrammar_test xgrammar gtest gtest_main) include(GoogleTest) gtest_discover_tests(xgrammar_test) endif() xgrammar-0.1.19/LICENSE000066400000000000000000000261351500705317600144400ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. xgrammar-0.1.19/NOTICE000066400000000000000000000000661500705317600143320ustar00rootroot00000000000000XGrammar Copyright (c) 2024 by XGrammar Contributors xgrammar-0.1.19/README.md000066400000000000000000000045541500705317600147130ustar00rootroot00000000000000
# XGrammar [![Documentation](https://img.shields.io/badge/docs-latest-green)](https://xgrammar.mlc.ai/docs/) [![License](https://img.shields.io/badge/license-apache_2-blue)](https://github.com/mlc-ai/xgrammar/blob/main/LICENSE) [![PyPI](https://img.shields.io/pypi/v/xgrammar)](https://pypi.org/project/xgrammar) [![PyPI Downloads](https://static.pepy.tech/badge/xgrammar)](https://pepy.tech/projects/xgrammar) **Efficient, Flexible and Portable Structured Generation** [Get Started](#get-started) | [Documentation](https://xgrammar.mlc.ai/docs/) | [Blogpost](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar) | [Technical Report](https://arxiv.org/abs/2411.15100)
## News - [2025/02] XGrammar has been officially integrated into [Modular's MAX](https://docs.modular.com/max/serve/structured-output) - [2025/01] XGrammar has been officially integrated into [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). - [2024/12] XGrammar has been officially integrated into [vLLM](https://github.com/vllm-project/vllm). - [2024/12] We presented research talks on XGrammar at CMU Catalyst, Berkeley SkyLab, MIT HANLAB, THU IIIS, SJTU, Ant Group, SGLang Meetup, Qingke AI, Camel AI. The slides can be found [here](https://docs.google.com/presentation/d/1iS7tu2EV4IKRWDaR0F3YD7ubrNqtGYUStSskceneelc/edit?usp=sharing). - [2024/11] XGrammar has been officially integrated into [SGLang](https://github.com/sgl-project/sglang). - [2024/11] XGrammar has been officially integrated into [MLC-LLM](https://github.com/mlc-ai/mlc-llm). - [2024/11] We officially released XGrammar v0.1.0! ## Overview XGrammar is an open-source library for efficient, flexible, and portable structured generation. It supports general context-free grammar to enable a broad range of structures while bringing careful system optimizations to enable fast executions. XGrammar features a minimal and portable C++ backend that can be easily integrated into multiple environments and frameworks, and is co-designed with the LLM inference engine and enables zero-overhead structured generation in LLM inference. ## Get Started Please visit our [documentation](https://xgrammar.mlc.ai/docs/) to get started with XGrammar. - [Installation](https://xgrammar.mlc.ai/docs/start/install) - [Quick start](https://xgrammar.mlc.ai/docs/start/quick_start) xgrammar-0.1.19/cmake/000077500000000000000000000000001500705317600145045ustar00rootroot00000000000000xgrammar-0.1.19/cmake/config.cmake000066400000000000000000000002201500705317600167450ustar00rootroot00000000000000set(CMAKE_BUILD_TYPE RelWithDebInfo) set(XGRAMMAR_BUILD_PYTHON_BINDINGS ON) set(XGRAMMAR_BUILD_CXX_TESTS OFF) set(XGRAMMAR_ENABLE_CPPTRACE OFF) xgrammar-0.1.19/cpp/000077500000000000000000000000001500705317600142065ustar00rootroot00000000000000xgrammar-0.1.19/cpp/compiled_grammar_data_structure.h000066400000000000000000000104561500705317600230000ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/compiled_grammar_data_structure.h * \brief The header for the data structures of the compiled grammar. */ #ifndef XGRAMMAR_COMPILED_GRAMMAR_DATA_STRUCTURE_H_ #define XGRAMMAR_COMPILED_GRAMMAR_DATA_STRUCTURE_H_ #include #include #include #include #include #include #include // matcher_data_structure.h is included to use StackElement #include "persistent_stack.h" #include "support/dynamic_bitset.h" #include "support/utils.h" namespace xgrammar { /******************* CompiledGrammar Datastructures *******************/ /*! * \brief Preprocessed information, for a given specific StackElement, divides the token set * into three categories: accepted, rejected, and uncertain. * Accepted: tokens that can be determined by the current StackElement to be acceptable * Rejected: tokens that can be determined by the current StackElement to be unacceptable * Uncertain: tokens that need the state of the parent StackElements to determine if acceptable * * \note uncertain indices are stored directly. Accepted / rejected indices have three ways to * store to reduce memory and computation usage. See StoreType. * \note These indices are the indices of sorted_decoded_vocab in the CompiledGrammar * object, instead of the token ids. That helps the matching process. */ struct AdaptiveTokenMask { enum class StoreType { // Only store all accepted token indices. Then rejected indices = all_indices - accepted_indices // - uncertain_indices. This is useful when |accepted_indices| < |rejected_indices|. kAccepted = 0, // Only store all accepted token indices. Then accepted indices = all_indices - rejected_indices // - uncertain_indices. This is useful when |accepted_indices| > |rejected_indices|. kRejected = 1, // Store all accepted token indices in a bitset. This is useful when both |accepted_indices| and // |rejected_indices| are large. kAcceptedBitset = 2 }; StoreType store_type; static constexpr int USE_BITSET_THRESHOLD = 200; std::vector accepted_indices; std::vector rejected_indices; DynamicBitset accepted_bitset; std::vector uncertain_indices; AdaptiveTokenMask() = default; AdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, const std::vector& accepted_indices, const std::vector& rejected_indices, const std::vector& uncertain_indices ); std::string Print(const TokenizerInfo& tokenizer_info) const; std::size_t MemorySize() const; friend std::size_t MemorySize(const AdaptiveTokenMask& mask); }; /*! * \brief All information that we need to match tokens in the tokenizer to the specified grammar. * It is the result of preprocessing. * \sa xgrammar::GrammarMatcher */ class CompiledGrammar::Impl { public: /******************* The grammar and tokenizer info *******************/ /*! \brief The grammar for the GrammarMatcher. */ Grammar grammar; /*! \brief The tokenizer information. */ TokenizerInfo tokenizer_info; /******************* The adaptive token mask cache *******************/ struct StackElementEqual { std::size_t operator()(const StackElement& lhs, const StackElement& rhs) const noexcept { return lhs.sequence_id == rhs.sequence_id && lhs.element_id == rhs.element_id && lhs.left_utf8_bytes == rhs.left_utf8_bytes && lhs.element_in_string == rhs.element_in_string; } }; struct StackElementHash { std::size_t operator()(const StackElement& stack_element) const noexcept { return HashCombine( stack_element.sequence_id, stack_element.element_id, stack_element.left_utf8_bytes, stack_element.element_in_string ); } }; /*! \brief Mapping from the stack top element to the adaptive token mask. */ std::unordered_map adaptive_token_mask_cache; Grammar GetGrammar() const { return grammar; } TokenizerInfo GetTokenizerInfo() const { return tokenizer_info; } std::size_t MemorySize() const; }; } // namespace xgrammar #endif // XGRAMMAR_COMPILED_GRAMMAR_DATA_STRUCTURE_H_ xgrammar-0.1.19/cpp/ebnf_script_creator.h000066400000000000000000000124461500705317600204030ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/ebnf_script_creator.h * \brief The header for the creating EBNF script. */ #ifndef XGRAMMAR_EBNF_SCRIPT_CREATOR_H_ #define XGRAMMAR_EBNF_SCRIPT_CREATOR_H_ #include #include #include #include #include #include "support/encoding.h" #include "support/logging.h" namespace xgrammar { /*! * \brief A class for creating EBNF grammar scripts. * * This class helps build EBNF (Extended Backus-Naur Form) grammar scripts * by managing rules and their content. */ class EBNFScriptCreator { public: /*! \brief Constructor */ EBNFScriptCreator() = default; /*! * \brief Adds a new rule to the grammar with a suggested name * \param rule_name_hint Suggested name for the rule * \param rule_body The EBNF content/definition of the rule * \return The actual name assigned to the rule */ std::string AddRule(const std::string& rule_name_hint, const std::string& rule_body) { return AddRuleWithAllocatedName(AllocateRuleName(rule_name_hint), rule_body); } /*! * \brief Generates a new rule name based on a suggested name * \param rule_name_hint Suggested name for the rule * \return The actual name assigned to the rule */ std::string AllocateRuleName(const std::string& rule_name_hint) { if (rule_names_.find(rule_name_hint) == rule_names_.end()) { rule_names_.insert(rule_name_hint); return rule_name_hint; } for (int i = 0; i < NAME_SUFFIX_MAXIMUM; ++i) { std::string rule_name = rule_name_hint + "_" + std::to_string(i); if (rule_names_.find(rule_name) == rule_names_.end()) { rule_names_.insert(rule_name); return rule_name; } } XGRAMMAR_LOG(FATAL) << "Cannot find a unique rule name for " << rule_name_hint; } /*! * \brief Adds a new rule to the grammar with a allocated name. Used with AllocateRuleName() * \param rule_name The name of the rule to add * \param rule_body The EBNF content/definition of the rule * \return The actual name assigned to the rule */ std::string AddRuleWithAllocatedName(const std::string& rule_name, const std::string& rule_body) { XGRAMMAR_CHECK(rule_names_.find(rule_name) != rule_names_.end()) << "Rule name " << rule_name << " is not allocated"; rules_.emplace_back(rule_name, rule_body); return rule_name; } /*! * \brief Concatenates a list of strings with a space separator * \param items The list of strings to concatenate * \return The concatenated string */ static std::string Concat(const std::vector& items) { std::stringstream ss; ss << "("; for (int i = 0; i < static_cast(items.size()); ++i) { if (i > 0) { ss << " "; } ss << items[i]; } ss << ")"; return ss.str(); } /*! * \brief Joins a list of strings with an OR operator * \param items The list of strings to join * \return The joined string */ static std::string Or(const std::vector& items) { std::stringstream ss; ss << "("; for (int i = 0; i < static_cast(items.size()); ++i) { if (i > 0) { ss << " | "; } ss << items[i]; } ss << ")"; return ss.str(); } /*! * \brief Escape and quote a string * \param str The string to escape and quote * \return The escaped and quoted string */ static std::string Str(const std::string& str) { std::stringstream ss; ss << "\"" << PrintAsEscapedUTF8(str) << "\""; return ss.str(); } /*! * \brief Repeats an item a given number of times * \param item The item to repeat * \param min The minimum number of times to repeat the item * \param max The maximum number of times to repeat the item * \return The repeated string */ static std::string Repeat(const std::string& item, int min, int max) { std::stringstream ss; ss << item; if (min == 0 && max == 1) { ss << "?"; } else if (min == 0 && max == -1) { ss << "*"; } else if (min == 1 && max == -1) { ss << "+"; } else if (min == 0 && max == 0) { return ""; } else if (min == max) { ss << "{" << min << "}"; } else if (max == -1) { ss << "{" << min << ",}"; } else { ss << "{" << min << "," << max << "}"; } return ss.str(); } /*! * \brief Gets the complete EBNF grammar script * \return The full EBNF grammar script as a string */ std::string GetScript() { std::string script = ""; for (const auto& rule : rules_) { script += rule.first + " ::= " + rule.second + "\n"; } return script; } /*! * \brief Retrieves the content/definition of a specific rule * \param rule_name The name of the rule to look up * \return The EBNF content/definition of the specified rule */ std::string GetRuleContent(const std::string& rule_name) { auto it = std::find_if(rules_.begin(), rules_.end(), [rule_name](const auto& rule) { return rule.first == rule_name; }); if (it != rules_.end()) { return it->second; } return ""; } private: std::vector> rules_; std::unordered_set rule_names_; const int NAME_SUFFIX_MAXIMUM = 10000; }; } // namespace xgrammar #endif // XGRAMMAR_EBNF_SCRIPT_CREATOR_H_ xgrammar-0.1.19/cpp/fsm.cc000066400000000000000000001647061500705317600153200ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/fsm.cc */ #include "fsm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "support/logging.h" #include "support/union_find_set.h" namespace xgrammar { std::vector> HandleEscapes(const std::string& regex, int start); Result> CheckRepeat(const std::string& regex, int& start); void CompactFSM::GetEpsilonClosure( std::unordered_set* state_set, std::unordered_set* result ) const { if (result == nullptr) { result = state_set; } std::queue queue; for (const auto& state : *state_set) { queue.push(state); } while (!queue.empty()) { int current = queue.front(); queue.pop(); result->insert(current); for (const auto& edge : edges[current]) { if (edge.IsEpsilon()) { if (result->find(edge.target) != result->end()) { continue; } queue.push(edge.target); } } } return; } FSMEdge::FSMEdge(const short& _min, const short& _max, const int& target) : min(_min), max(_max), target(target) { if (IsCharRange() && min > max) { XGRAMMAR_DCHECK(false) << "Invalid FSMEdge: min > max. min=" << min << ", max=" << max; } } bool FSMEdge::IsEpsilon() const { return min == -1 && max == -1; } bool FSMEdge::IsRuleRef() const { return min == -1 && max != -1; } bool FSMEdge::IsCharRange() const { return min >= 0 && max >= 0; } short FSMEdge::GetRefRuleId() const { if (IsRuleRef()) { return max; } else { XGRAMMAR_DCHECK(false) << "Invalid FSMEdge: not a rule reference. min=" << min << ", max=" << max; return -1; } } void FSM::GetEpsilonClosure(std::unordered_set* state_set, std::unordered_set* result) const { if (result == nullptr) { result = state_set; } std::queue queue; for (const auto& state : *state_set) { queue.push(state); } while (!queue.empty()) { int current = queue.front(); queue.pop(); result->insert(current); for (const auto& edge : edges[current]) { if (edge.IsEpsilon()) { if (result->find(edge.target) != result->end()) { continue; } queue.push(edge.target); } } } return; } FSM FSM::Copy() const { FSM copy; copy.edges.resize(edges.size()); for (size_t i = 0; i < edges.size(); ++i) { copy.edges[i] = edges[i]; } return copy; } FSMWithStartEnd FSMWithStartEnd::Union(const std::vector& fsms) { FSMWithStartEnd result; int node_cnt = 1; result.start = 0; // In the new FSM, we define the start state is 0. result.fsm.edges.push_back(std::vector()); for (const auto& fsm_with_se : fsms) { result.fsm.edges[0].emplace_back(-1, -1, fsm_with_se.start + node_cnt); for (const auto& edges : fsm_with_se.fsm.edges) { result.fsm.edges.push_back(std::vector()); for (const auto& edge : edges) { result.fsm.edges.back().emplace_back(edge.min, edge.max, edge.target + node_cnt); } for (const auto& end : fsm_with_se.ends) { result.ends.insert(end + node_cnt); } } node_cnt += fsm_with_se.fsm.edges.size(); } return result; } FSMWithStartEnd FSMWithStartEnd::Not() const { FSMWithStartEnd result; // Build the DFA. if (!is_dfa) { result = ToDFA(); } else { result = Copy(); } int node_cnt = result.fsm.edges.size(); // Reverse all the final states. std::unordered_set final_states; for (int i = 0; i < node_cnt; ++i) { if (result.ends.find(i) == result.ends.end()) { final_states.insert(i); } } result.ends = final_states; // Add all the rules in the alphabet. std::unordered_set rules; for (const auto& edges : result.fsm.edges) { for (const auto& edge : edges) { if (edge.IsRuleRef()) { rules.insert(edge.GetRefRuleId()); } } } // Add a new state to avoid the blocking. result.fsm.edges.push_back(std::vector()); for (auto rule : rules) { result.fsm.edges.back().emplace_back(-1, rule, node_cnt); } result.fsm.edges.back().emplace_back(0, 0x00FF, node_cnt); result.ends.insert(node_cnt); for (size_t i = 0; i < fsm.edges.size(); i++) { const auto& node_edges = fsm.edges[i]; std::vector char_has_edges(0x100, false); std::unordered_set rule_has_edges; for (const auto& edge : node_edges) { if (edge.IsCharRange()) { for (int i = edge.min; i <= edge.max; ++i) { char_has_edges[i] = true; } } if (edge.IsRuleRef()) { rule_has_edges.insert(edge.GetRefRuleId()); } } // Add the left characters to the new state. int interval_start = -1; for (int j = 0; j < 0x100; ++j) { if (!char_has_edges[j]) { // The char doesn't have any edges. Thus, we can accept it in the // complement FSM. if (interval_start == -1) { interval_start = j; } } else { if (interval_start != -1) { // node_cnt is the node to accept all such characters. result.fsm.edges[i].emplace_back(interval_start, i - 1, node_cnt); interval_start = -1; } } } if (interval_start != -1) { result.fsm.edges[i].emplace_back(interval_start, 0xFF, node_cnt); } // Add the left rules to the new state. for (auto rule : rules) { if (rule_has_edges.find(rule) == rule_has_edges.end()) { result.fsm.edges.back().emplace_back(-1, rule, node_cnt); } } } return result; } void FSM::Advance( const std::vector& from, int value, std::vector* result, bool is_rule, bool is_closure ) const { result->clear(); std::unordered_set in_result; std::unordered_set result_closure; std::unordered_set start_set; for (const auto& state : from) { start_set.insert(state); } if (!is_closure) { GetEpsilonClosure(&start_set); } for (const auto& state : start_set) { const auto& edge_list = edges[state]; for (const auto& edge : edge_list) { if (edge.IsEpsilon()) { continue; } if (is_rule && edge.IsRuleRef()) { if (edge.GetRefRuleId() == value) { in_result.insert(edge.target); } continue; } if (!is_rule && edge.IsCharRange()) { if (value >= edge.min && value <= edge.max) { in_result.insert(edge.target); } continue; } } } for (const auto& state : in_result) { if (result_closure.find(state) != result_closure.end()) { continue; } std::unordered_set closure; closure.insert(state); GetEpsilonClosure(&closure); result_closure.insert(closure.begin(), closure.end()); } for (const auto& state : result_closure) { result->push_back(state); } return; } FSMWithStartEnd FSMWithStartEnd::Copy() const { FSMWithStartEnd copy; copy.is_dfa = is_dfa; copy.start = start; copy.ends = ends; copy.fsm = fsm.Copy(); return copy; } std::string FSMWithStartEnd::Print() const { std::string result; result += "FSM(num_nodes=" + std::to_string(fsm.edges.size()) + ", start=" + std::to_string(start) + ", end=["; for (const auto& end : ends) { result += std::to_string(end) + ", "; } result += "], edges=[\n"; for (int i = 0; i < int(fsm.edges.size()); ++i) { result += std::to_string(i) + ": ["; const auto& edges = fsm.edges[i]; for (int j = 0; j < static_cast(fsm.edges[i].size()); ++j) { const auto& edge = edges[j]; if (edge.min == edge.max) { result += "(" + std::to_string(edge.min) + ")->" + std::to_string(edge.target); } else { result += "(" + std::to_string(edge.min) + ", " + std::to_string(edge.max) + ")->" + std::to_string(edge.target); } if (j < static_cast(fsm.edges[i].size()) - 1) { result += ", "; } } result += "]\n"; } result += "])"; return result; } std::string CompactFSMWithStartEnd::Print() const { std::string result; result += "CompactFSM(num_nodes=" + std::to_string(fsm.edges.Size()) + ", start=" + std::to_string(start) + ", end=["; for (const auto& end : ends) { result += std::to_string(end) + ", "; } result += "], edges=[\n"; for (int i = 0; i < int(fsm.edges.Size()); ++i) { result += std::to_string(i) + ": ["; const auto& edges = fsm.edges[i]; for (int j = 0; j < static_cast(fsm.edges[i].size()); ++j) { const auto& edge = edges[j]; if (edge.min == edge.max) { result += "(" + std::to_string(edge.min) + ")->" + std::to_string(edge.target); } else { result += "(" + std::to_string(edge.min) + ", " + std::to_string(edge.max) + ")->" + std::to_string(edge.target); } if (j < static_cast(fsm.edges[i].size()) - 1) { result += ", "; } } result += "]\n"; } result += "])"; return result; } CompactFSM FSM::ToCompact() { CompactFSM result; for (int i = 0; i < static_cast(edges.size()); ++i) { std::sort(edges[i].begin(), edges[i].end(), [](const FSMEdge& a, const FSMEdge& b) { return a.min != b.min ? a.min < b.min : a.max < b.max; }); result.edges.Insert(edges[i]); } return result; } FSM CompactFSM::ToFSM() { FSM result; for (int i = 0; i < edges.Size(); i++) { const auto& row = edges[i]; result.edges.emplace_back(std::vector()); for (int j = 0; j < row.size(); j++) { result.edges.back().push_back(row[j]); } } return result; } void CompactFSM::Advance( const std::vector& from, int value, std::vector* result, bool is_rule, bool is_closure ) const { result->clear(); std::unordered_set in_result; std::unordered_set result_closure; std::unordered_set start_set; for (const auto& state : from) { start_set.insert(state); } if (!is_closure) { GetEpsilonClosure(&start_set); } for (const auto& state : start_set) { const auto& edge_list = edges[state]; for (const auto& edge : edge_list) { if (edge.IsEpsilon()) { continue; } if (is_rule && edge.IsRuleRef()) { if (edge.GetRefRuleId() == value) { in_result.insert(edge.target); } continue; } if (!is_rule && edge.IsCharRange()) { if (value >= edge.min && value <= edge.max) { in_result.insert(edge.target); } continue; } } } for (const auto& state : in_result) { if (result_closure.find(state) != result_closure.end()) { continue; } std::unordered_set closure; closure.insert(state); GetEpsilonClosure(&closure); result_closure.insert(closure.begin(), closure.end()); } for (const auto& state : result_closure) { result->push_back(state); } return; } FSMWithStartEnd FSMWithStartEnd::ToDFA() const { FSMWithStartEnd dfa; dfa.is_dfa = true; dfa.start = start; std::vector> closures; std::unordered_set rules; for (const auto& edges : fsm.edges) { for (const auto& edge : edges) { if (edge.IsRuleRef()) { rules.insert(edge.GetRefRuleId()); } } } int now_process = 0; std::unordered_set closure; closure.insert(start); fsm.GetEpsilonClosure(&closure); closures.push_back(closure); while (now_process < static_cast(closures.size())) { std::set interval_ends; dfa.fsm.edges.push_back(std::vector()); // Check if the closure is a final state. for (const auto& node : closures[now_process]) { if (ends.find(node) != ends.end()) { dfa.ends.insert(now_process); } const auto& edges = fsm.edges[node]; for (const auto& edge : edges) { if (edge.IsCharRange()) { interval_ends.insert(edge.min); interval_ends.insert(edge.max + 1); continue; } } } // This part is to get the all possible intervals. // Which can help reduce the transitions. using Interval = std::pair; std::vector intervals; intervals.reserve(interval_ends.size()); int last = -1; for (const auto& end : interval_ends) { if (last == -1) { last = end; continue; } intervals.emplace_back(last, end - 1); last = end; } for (const auto& interval : intervals) { std::unordered_set next_closure; for (const auto& node : closures[now_process]) { const auto& edges = fsm.edges[node]; for (const auto& edge : edges) { if (edge.IsCharRange()) { if (interval.first >= edge.min && interval.second <= edge.max) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.fsm.edges[now_process].emplace_back(interval.first, interval.second, j); flag = true; break; } } if (!flag) { dfa.fsm.edges[now_process].emplace_back(interval.first, interval.second, closures.size()); closures.push_back(next_closure); } } for (auto rule : rules) { std::unordered_set next_closure; for (const auto& node : closures[now_process]) { const auto& edges = fsm.edges[node]; for (const auto& edge : edges) { if (edge.IsRuleRef()) { if (rule == edge.GetRefRuleId()) { if (next_closure.find(edge.target) == next_closure.end()) { std::unordered_set epsilon_closure; epsilon_closure.insert(edge.target); fsm.GetEpsilonClosure(&epsilon_closure); next_closure.insert(epsilon_closure.begin(), epsilon_closure.end()); } } } } } bool flag = false; for (int j = 0; j < static_cast(closures.size()); j++) { if (closures[j] == next_closure) { dfa.fsm.edges[now_process].emplace_back(-1, rule, j); flag = true; break; } } if (!flag) { dfa.fsm.edges[now_process].emplace_back(-1, rule, closures.size()); closures.push_back(next_closure); } } now_process++; } return dfa; } FSMWithStartEnd FSMWithStartEnd::Concatenate(const std::vector& fsms) { FSMWithStartEnd result; result.is_dfa = false; int node_cnt = 0; result.start = fsms[0].start; for (size_t i = 0; i < fsms.size(); i++) { const auto& fsm_with_se = fsms[i]; for (const auto& edges : fsm_with_se.fsm.edges) { result.fsm.edges.push_back(std::vector()); for (const auto& edge : edges) { result.fsm.edges.back().emplace_back(edge.min, edge.max, edge.target + node_cnt); } } if (i == fsms.size() - 1) { for (const auto& end : fsm_with_se.ends) { result.ends.insert(end + node_cnt); } break; } for (const auto& end : fsm_with_se.ends) { result.fsm.edges[end + node_cnt].emplace_back( -1, -1, fsm_with_se.fsm.edges.size() + node_cnt + fsms[i + 1].start ); } node_cnt += fsm_with_se.fsm.edges.size(); } return result; } FSMWithStartEnd FSMWithStartEnd::MakeStar() const { FSMWithStartEnd result; result.is_dfa = false; result.fsm = fsm.Copy(); result.ends = ends; result.start = start; for (const auto& end : ends) { result.fsm.edges[end].emplace_back(-1, -1, start); } result.fsm.edges[start].emplace_back(-1, -1, *ends.begin()); return result; } FSMWithStartEnd FSMWithStartEnd::MakePlus() const { FSMWithStartEnd result; result.is_dfa = false; result.fsm = fsm.Copy(); result.ends = ends; result.start = start; for (const auto& end : ends) { result.fsm.edges[end].emplace_back(-1, -1, start); } return result; } FSMWithStartEnd FSMWithStartEnd::MakeOptional() const { FSMWithStartEnd result; result.is_dfa = false; result.fsm = fsm.Copy(); result.ends = ends; result.start = start; result.fsm.edges[start].emplace_back(-1, -1, *ends.begin()); return result; } FSMWithStartEnd::FSMWithStartEnd(const std::string& regex) { is_dfa = true; start = 0; auto& edges = fsm.edges; // Handle the regex string. if (!(regex[0] == '[' && regex[regex.size() - 1] == ']')) { edges.push_back(std::vector()); for (size_t i = 0; i < regex.size(); i++) { if (regex[i] != '\\') { if (regex[i] == '.') { edges.back().emplace_back(0, 0xFF, edges.size()); } else { edges.back().emplace_back( (unsigned char)(regex[i]), (unsigned char)(regex[i]), edges.size() ); } edges.push_back(std::vector()); continue; } std::vector> escape_vector = HandleEscapes(regex, i); for (const auto& escape : escape_vector) { edges.back().emplace_back( (unsigned char)(escape.first), (unsigned char)(escape.second), edges.size() ); } edges.push_back(std::vector()); i++; } ends.insert(edges.size() - 1); return; } // Handle the character class. if (regex[0] == '[' && regex[regex.size() - 1] == ']') { edges.push_back(std::vector()); edges.push_back(std::vector()); ends.insert(1); bool reverse = regex[1] == '^'; for (size_t i = reverse ? 2 : 1; i < regex.size() - 1; i++) { if (regex[i] != '\\') { if (!(((i + 2) < regex.size() - 1) && regex[i + 1] == '-')) { // A single char. edges[0].emplace_back(regex[i], regex[i], 1); continue; } // Handle the char range. if (regex[i + 2] != '\\') { edges[0].emplace_back(regex[i], regex[i + 2], 1); i = i + 2; continue; } auto escaped_edges = HandleEscapes(regex, i + 2); // Means it's not a range. if (escaped_edges.size() != 1 || escaped_edges[0].first != escaped_edges[0].second) { edges[0].emplace_back(regex[i], regex[i], 1); continue; } edges[0].emplace_back(regex[0], escaped_edges[0].first, 1); i = i + 3; continue; } auto escaped_edges = HandleEscapes(regex, i); i = i + 1; if (escaped_edges.size() != 1 || escaped_edges[0].first != escaped_edges[0].second) { // It's a multi-match escape char. for (const auto& edge : escaped_edges) { edges[0].emplace_back(edge.first, edge.second, 1); } continue; } if (!(((i + 2) < regex.size() - 1) && regex[i + 1] == '-')) { edges[0].emplace_back(escaped_edges[0].first, escaped_edges[0].second, 1); continue; } if (regex[i + 2] != '\\') { edges[0].emplace_back(escaped_edges[0].first, regex[i + 2], 1); i = i + 2; continue; } auto rhs_escaped_edges = HandleEscapes(regex, i + 2); if (rhs_escaped_edges.size() != 1 || rhs_escaped_edges[0].first != rhs_escaped_edges[0].second) { edges[0].emplace_back(escaped_edges[0].first, escaped_edges[0].second, 1); continue; } edges[0].emplace_back(escaped_edges[0].first, rhs_escaped_edges[0].first, 1); i = i + 3; continue; } bool has_edge[0x100]; memset(has_edge, 0, sizeof(has_edge)); for (const auto& edge : edges[0]) { for (int i = edge.min; i <= edge.max; i++) { has_edge[i] = true; } } edges[0].clear(); // Simplify the edges. e.g [abc] -> [a-c] int last = -1; if (reverse) { for (int i = 0; i < 0x100; i++) { if (!has_edge[i]) { if (last == -1) { last = i; } continue; } if (last != -1) { edges[0].emplace_back(last, i - 1, 1); last = -1; } } if (last != -1) { edges[0].emplace_back(last, 0xFF, 1); } } else { for (int i = 0; i < 0x100; i++) { if (has_edge[i]) { if (last == -1) { last = i; } continue; } if (last != -1) { edges[0].emplace_back(last, i - 1, 1); last = -1; } } if (last != -1) { edges[0].emplace_back(last, 0xFF, 1); } } return; } // TODO: The support for rules. XGRAMMAR_LOG(WARNING) << "rule is not supported yet."; } FSMWithStartEnd FSMWithStartEnd::MinimizeDFA() const { FSMWithStartEnd now_fsm; // To perform the algorithm, we must make sure the FSM is // a DFA. if (!is_dfa) { now_fsm = ToDFA(); } else { now_fsm = Copy(); } // Initialize the set. std::list> blocks; std::list> queue; std::unordered_set not_end; for (size_t i = 0; i < now_fsm.fsm.edges.size(); i++) { if (now_fsm.ends.find(i) == now_fsm.ends.end()) { not_end.insert(i); } } queue.push_back(not_end); queue.push_back(now_fsm.ends); blocks.push_back(now_fsm.ends); blocks.push_back(not_end); std::set interval_ends; std::unordered_set> intervals; std::unordered_set rules; std::unordered_map> previous_mapping; for (size_t i = 0; i < now_fsm.fsm.edges.size(); i++) { const auto& edges = now_fsm.fsm.edges[i]; for (const auto& edge : edges) { if (previous_mapping.find(edge.target) == previous_mapping.end()) { previous_mapping[edge.target] = std::unordered_set(); } previous_mapping[edge.target].insert(i); if (edge.IsCharRange()) { interval_ends.insert(edge.min); interval_ends.insert(edge.max + 1); continue; } if (edge.IsRuleRef()) { rules.insert(edge.GetRefRuleId()); } } } for (auto it = interval_ends.begin(); it != interval_ends.end(); ++it) { auto next_it = std::next(it); if (next_it != interval_ends.end()) { intervals.insert(std::make_pair(*it, *next_it - 1)); } } while (!queue.empty()) { // Initial the alphabet. auto block_x = *queue.begin(); queue.erase(queue.begin()); std::unordered_set prev_nodes; for (const auto& node : block_x) { if (previous_mapping.find(node) != previous_mapping.end()) { prev_nodes.insert(previous_mapping[node].begin(), previous_mapping[node].end()); } } // Check the intervals. std::list> blocks_copy = blocks; for (const auto& interval : intervals) { std::unordered_set from_block; for (const auto& node : prev_nodes) { const auto& edges = now_fsm.fsm.edges[node]; for (const auto& edge : edges) { if (block_x.find(edge.target) == block_x.end()) { continue; } if (edge.IsCharRange()) { if (interval.first >= edge.min && interval.second <= edge.max) { from_block.insert(node); } } } } for (const auto& block : blocks_copy) { std::unordered_set intersection; for (const auto& prev : from_block) { if (block.find(prev) != block.end()) { intersection.insert(prev); } } // The intersection is empty, or the intersection == block. if (intersection.empty() || intersection.size() == block.size()) { continue; } std::unordered_set difference; for (const auto& node : block) { if (intersection.find(node) == intersection.end()) { difference.insert(node); } } blocks.remove(block); blocks.remove(intersection); blocks.remove(difference); blocks.push_back(intersection); blocks.push_back(difference); bool found = false; for (auto iter = queue.begin(); iter != queue.end(); ++iter) { if (*iter == block) { found = true; break; } } if (found) { queue.remove(block); queue.push_back(intersection); queue.push_back(difference); } else { queue.push_back(intersection.size() < difference.size() ? intersection : difference); } } } // Do the same thing for the rules. blocks_copy = blocks; for (const auto& rule : rules) { std::unordered_set from_block; for (const auto& node : prev_nodes) { const auto& edges = now_fsm.fsm.edges[node]; for (const auto& edge : edges) { if (block_x.find(edge.target) == block_x.end()) { continue; } if (edge.IsRuleRef()) { if (rule == edge.GetRefRuleId()) { from_block.insert(node); } } } } for (const auto& block : blocks_copy) { std::unordered_set intersection; for (const auto& prev : from_block) { if (block.find(prev) != block.end()) { intersection.insert(prev); } } // The intersection is empty, or the intersection == block. if (intersection.empty() || intersection.size() == block.size()) { continue; } std::unordered_set difference; for (const auto& node : from_block) { if (intersection.find(node) == intersection.end()) { difference.insert(node); } } blocks.remove(block); blocks.remove(intersection); blocks.remove(difference); blocks.push_back(intersection); blocks.push_back(difference); bool found = false; for (auto iter = queue.begin(); iter != queue.end(); ++iter) { if (*iter == block) { found = true; break; } } if (found) { queue.remove(block); queue.push_back(intersection); queue.push_back(difference); } else { queue.push_back(intersection.size() < difference.size() ? intersection : difference); } } } } std::unordered_map old_to_new; int cnt = 0; for (const auto& block : blocks) { for (const auto& node : block) { old_to_new[node] = cnt; } cnt++; } FSMWithStartEnd new_fsm; new_fsm.is_dfa = true; new_fsm.start = old_to_new[now_fsm.start]; for (const auto& end : now_fsm.ends) { new_fsm.ends.insert(old_to_new[end]); } for (int i = 0; i < cnt; i++) { new_fsm.fsm.edges.push_back(std::vector()); } std::unordered_set been_built; for (size_t i = 0; i < now_fsm.fsm.edges.size(); i++) { if (been_built.find(old_to_new[i]) != been_built.end()) { continue; } been_built.insert(old_to_new[i]); for (const auto& edge : now_fsm.fsm.edges[i]) { new_fsm.fsm.edges[old_to_new[i]].emplace_back(edge.min, edge.max, old_to_new[edge.target]); } } return new_fsm; } std::vector> HandleEscapes(const std::string& regex, int start) { std::vector> result; switch (regex[start + 1]) { case 'n': { return std::vector>(1, std::make_pair('\n', '\n')); } case 't': { return std::vector>(1, std::make_pair('\t', '\t')); } case 'r': { return std::vector>(1, std::make_pair('\r', '\r')); } case '0': { return std::vector>(1, std::make_pair('\0', '\0')); } case 's': { return std::vector>(1, std::make_pair(0, ' ')); } case 'S': { return std::vector>(1, std::make_pair(' ' + 1, 0x00FF)); } case 'd': { return std::vector>(1, std::make_pair('0', '9')); } case 'D': { std::vector> result; result.emplace_back(0, '0' - 1); result.emplace_back('9' + 1, 0x00FF); return result; } case 'w': { std::vector> result; result.emplace_back('0', '9'); result.emplace_back('a', 'z'); result.emplace_back('A', 'Z'); result.emplace_back('_', '_'); return result; } case 'W': { std::vector> result; result.emplace_back(0, '0' - 1); result.emplace_back('9' + 1, 'A' - 1); result.emplace_back('Z' + 1, '_' - 1); result.emplace_back('_' + 1, 'a' - 1); result.emplace_back('z' + 1, 0x00FF); return result; } default: { return std::vector>( 1, std::make_pair(regex[start + 1], regex[start + 1]) ); } } } Result FSMWithStartEnd::Intersect( const FSMWithStartEnd& lhs, const FSMWithStartEnd& rhs, const int& num_of_nodes_limited ) { if (!lhs.IsLeaf() || !rhs.IsLeaf()) { return Result::Err(std::make_shared("Intersect only support leaf fsm!") ); } auto lhs_dfa = lhs.ToDFA(); auto rhs_dfa = rhs.ToDFA(); std::unordered_set rules_lhs; std::unordered_set rules; std::set interval_ends; std::vector> intervals; // This part is to build the equivalent alphabet. for (const auto& edges : lhs.fsm.edges) { for (const auto& edge : edges) { if (edge.IsRuleRef()) { rules_lhs.insert(edge.GetRefRuleId()); } } } for (const auto& edges : rhs.fsm.edges) { for (const auto& edge : edges) { if (edge.IsRuleRef()) { if (rules_lhs.find(edge.GetRefRuleId()) != rules_lhs.end()) { rules.insert(edge.GetRefRuleId()); } } } } for (const auto& edges : lhs_dfa.fsm.edges) { for (const auto& edge : edges) { if (edge.IsCharRange()) { interval_ends.insert(edge.min); interval_ends.insert(edge.max + 1); } } } for (const auto& edges : rhs_dfa.fsm.edges) { for (const auto& edge : edges) { if (edge.IsCharRange()) { interval_ends.insert(edge.min); interval_ends.insert(edge.max + 1); } } } for (auto it = interval_ends.begin(); it != interval_ends.end(); ++it) { auto next_it = std::next(it); if (next_it != interval_ends.end()) { intervals.emplace_back(*it, *next_it - 1); } } FSMWithStartEnd result; result.is_dfa = true; result.start = 0; std::unordered_map, int> state_map; std::unordered_set> visited; std::queue> queue; queue.push({lhs.start, rhs.start}); result.fsm.edges.push_back(std::vector()); state_map[{lhs.start, rhs.start}] = 0; while (!queue.empty()) { if (int(state_map.size()) > num_of_nodes_limited) { return Result::Err( std::make_shared("Intersection have too many nodes!") ); } auto state = queue.front(); queue.pop(); if (visited.find(state) != visited.end()) { continue; } visited.insert(state); int lhs_state = state.first; int rhs_state = state.second; for (const auto& interval : intervals) { for (const auto& lhs_edge : lhs_dfa.fsm.edges[lhs_state]) { if (!lhs_edge.IsCharRange()) { continue; } if (lhs_edge.min > interval.first || lhs_edge.max < interval.second) { continue; } for (const auto& rhs_edge : rhs_dfa.fsm.edges[rhs_state]) { if (!rhs_edge.IsCharRange()) { continue; } if (rhs_edge.min > interval.first || rhs_edge.max < interval.second) { continue; } auto next_state = std::make_pair(lhs_edge.target, rhs_edge.target); if (state_map.find(next_state) == state_map.end()) { state_map[next_state] = state_map.size(); queue.push(next_state); result.fsm.edges.push_back(std::vector()); } result.fsm.edges[state_map[{lhs_state, rhs_state}]].emplace_back( interval.first, interval.second, state_map[next_state] ); break; } } } for (const auto& rule : rules) { for (const auto& lhs_edge : lhs_dfa.fsm.edges[lhs_state]) { if (!lhs_edge.IsRuleRef()) { continue; } if (lhs_edge.GetRefRuleId() != rule) { continue; } for (const auto& rhs_edge : rhs_dfa.fsm.edges[rhs_state]) { if (!rhs_edge.IsRuleRef()) { continue; } if (rhs_edge.GetRefRuleId() != rule) { continue; } auto next_state = std::make_pair(lhs_edge.target, rhs_edge.target); if (state_map.find(next_state) == state_map.end()) { state_map[next_state] = state_map.size(); queue.push(next_state); result.fsm.edges.push_back(std::vector()); } result.fsm.edges[state_map[{lhs_state, rhs_state}]].emplace_back( -1, rule, state_map[next_state] ); break; } } } } for (const auto& state : visited) { if (lhs.ends.find(state.first) != lhs.ends.end() && rhs.ends.find(state.second) != rhs.ends.end()) { result.ends.insert(state_map[state]); } } return Result::Ok(result); } bool FSMWithStartEnd::Check(const std::string& str) const { std::unordered_set start_states_set; start_states_set.insert(start); fsm.GetEpsilonClosure(&start_states_set); std::vector from_states; std::vector result_states; for (const auto& start_state : start_states_set) { from_states.push_back(start_state); } for (const auto& character : str) { result_states.clear(); fsm.Advance(from_states, (unsigned char)(character), &result_states, false); from_states = result_states; } for (const auto& state : from_states) { if (ends.find(state) != ends.end()) { return true; } } return false; } bool CompactFSMWithStartEnd::Check(const std::string& str) const { std::unordered_set start_states_set; start_states_set.insert(start); fsm.GetEpsilonClosure(&start_states_set); std::vector from_states; std::vector result_states; for (const auto& start_state : start_states_set) { from_states.push_back(start_state); } for (const auto& character : str) { result_states.clear(); fsm.Advance(from_states, (unsigned char)(character), &result_states, false); from_states = result_states; } for (const auto& state : from_states) { if (ends.find(state) != ends.end()) { return true; } } return false; } bool FSMWithStartEnd::IsDFA() { if (is_dfa) { return true; } std::set interval_ends; std::unordered_set rules; for (const auto& edges : fsm.edges) { for (const auto& edge : edges) { if (edge.IsEpsilon()) { return false; } if (edge.IsCharRange()) { interval_ends.insert(edge.min); interval_ends.insert(edge.max + 1); continue; } if (edge.IsRuleRef()) { rules.insert(edge.GetRefRuleId()); continue; } } } using Interval = std::pair; std::unordered_set intervals; for (auto it = interval_ends.begin(); it != interval_ends.end(); ++it) { auto next_it = std::next(it); if (next_it != interval_ends.end()) { intervals.emplace(*it, *next_it - 1); } } for (const auto& edges : fsm.edges) { for (const auto& rule : rules) { bool find = false; for (const auto& edge : edges) { if (edge.IsRuleRef()) { if (edge.GetRefRuleId() == rule) { if (find) { return false; } find = true; } } } } for (const auto& interval : intervals) { bool find = false; for (const auto& edge : edges) { if (edge.IsCharRange()) { if (edge.min > interval.first || edge.max < interval.second) { continue; } if (find) { return false; } find = true; } } } } is_dfa = true; return true; } bool FSMWithStartEnd::IsLeaf() const { for (const auto& edges : fsm.edges) { for (const auto& edge : edges) { if (edge.IsRuleRef()) { return false; } } } return true; } void FSMWithStartEnd::SimplifyEpsilon() { if (IsDFA()) { return; } UnionFindSet union_find_set; std::unordered_map> previous_nodes; std::unordered_set has_epsilon; // Initialize the previous nodes, and find all the nodes that have // epsilon edges. for (size_t i = 0; i < fsm.edges.size(); i++) { const auto& edges = fsm.edges[i]; for (const auto& edge : edges) { if (previous_nodes.find(edge.target) == previous_nodes.end()) { previous_nodes[edge.target] = std::unordered_set(); } previous_nodes[edge.target].insert(i); if (edge.IsEpsilon()) { if (edges.size() != 1) { has_epsilon.insert(i); } else { // a -- epsilon --> b, and a doesn't have other outward edges. union_find_set.Make(i); union_find_set.Make(edge.target); union_find_set.Union(i, edge.target); } } } } // a --> epsilon --> b, and b doesn't have other inward edges. for (const auto& node : has_epsilon) { const auto& edges = fsm.edges[node]; for (const auto& edge : edges) { if (!edge.IsEpsilon()) { continue; } // Have other inward nodes. if (previous_nodes[edge.target].size() != 1) { continue; } bool has_other_edge = false; for (const auto& second_edge : edges) { if (second_edge.IsEpsilon()) { continue; } if (second_edge.target == edge.target) { has_other_edge = true; break; } } // The node can be merged. if (!has_other_edge) { union_find_set.Make(node); union_find_set.Make(edge.target); union_find_set.Union(node, edge.target); } } } // Merge the nodes. auto eq_classes = union_find_set.GetAllSets(); if (eq_classes.empty()) { return; } std::unordered_map new_to_old; for (size_t i = 0; i < eq_classes.size(); i++) { for (const auto& node : eq_classes[i]) { new_to_old[node] = i; } } int cnt = eq_classes.size(); for (size_t i = 0; i < fsm.edges.size(); i++) { if (new_to_old.find(i) == new_to_old.end()) { new_to_old[i] = cnt; cnt++; } } RebuildFSM(new_to_old, cnt); return; } void FSMWithStartEnd::SimplifyTransition() { bool changed = true; UnionFindSet union_find_set; while (changed) { union_find_set.Clear(); std::unordered_map> previous_nodes; // Initialize the previous nodes. for (size_t i = 0; i < fsm.edges.size(); i++) { const auto& edges = fsm.edges[i]; for (const auto& edge : edges) { if (previous_nodes.find(edge.target) == previous_nodes.end()) { previous_nodes[edge.target] = std::unordered_set(); } previous_nodes[edge.target].insert(i); } } // Case 1: Like ab | ac | ad, then they can be merged into a(b | c | d). changed = false; bool change_case1 = false; for (const auto& edges : fsm.edges) { for (size_t i = 0; i < edges.size(); i++) { for (size_t j = i + 1; j < edges.size(); j++) { if (IsEndNode(edges[i].target) != IsEndNode(edges[j].target)) { continue; } if (edges[i].target == edges[j].target) { continue; } if (edges[i].max != edges[j].max || edges[i].min != edges[j].min) { continue; } if (previous_nodes[edges[i].target].size() != 1 || previous_nodes[edges[j].target].size() != 1) { continue; } union_find_set.Make(edges[i].target); union_find_set.Make(edges[j].target); union_find_set.Union(edges[i].target, edges[j].target); change_case1 = true; } } } if (change_case1) { auto eq_classes = union_find_set.GetAllSets(); std::unordered_map old_to_new; for (size_t i = 0; i < eq_classes.size(); i++) { for (const auto& node : eq_classes[i]) { old_to_new[node] = i; } } int cnt = eq_classes.size(); for (size_t i = 0; i < fsm.edges.size(); i++) { if (old_to_new.find(i) == old_to_new.end()) { old_to_new[i] = cnt; cnt++; } } RebuildFSM(old_to_new, cnt); } union_find_set.Clear(); // Case 2: Like ba | ca | da, then they can be merged into (b | c | d)a. bool change_case2 = false; for (size_t i = 0; i < fsm.edges.size(); i++) { for (size_t j = i + 1; j < fsm.edges.size(); j++) { bool equivalent = true; for (const auto& edge_i : fsm.edges[i]) { bool same = false; for (const auto& edge_j : fsm.edges[j]) { if (edge_i.min == edge_j.min && edge_i.max == edge_j.max && edge_i.target == edge_j.target) { same = true; break; } } if (!same) { equivalent = false; break; } } if (!equivalent) { continue; } for (const auto& edge_j : fsm.edges[j]) { bool same = false; for (const auto& edge_i : fsm.edges[i]) { if (edge_i.min == edge_j.min && edge_i.max == edge_j.max && edge_i.target == edge_j.target) { same = true; break; } } if (!same) { equivalent = false; break; } } if (equivalent) { union_find_set.Make(i); union_find_set.Make(j); union_find_set.Union(i, j); change_case2 = true; } } } if (change_case2) { auto eq_classes = union_find_set.GetAllSets(); std::unordered_map old_to_new; for (size_t i = 0; i < eq_classes.size(); i++) { for (const auto& node : eq_classes[i]) { old_to_new[node] = i; } } int cnt = eq_classes.size(); for (size_t i = 0; i < fsm.edges.size(); i++) { if (old_to_new.find(i) == old_to_new.end()) { old_to_new[i] = cnt; cnt++; } } RebuildFSM(old_to_new, cnt); } changed = change_case1 || change_case2; } return; } void FSMWithStartEnd::RebuildFSM( std::unordered_map& old_to_new, const int& new_node_cnt ) { start = old_to_new[start]; decltype(ends) new_ends; for (const auto& end : ends) { new_ends.insert(old_to_new[end]); } ends = new_ends; decltype(fsm.edges) new_edges; struct Compare { bool operator()(const FSMEdge& lhs, const FSMEdge& rhs) const { if (lhs.min != rhs.min) { return lhs.min < rhs.min; } if (lhs.max != rhs.max) { return lhs.max < rhs.max; } return lhs.target < rhs.target; } }; std::vector> new_edges_set; new_edges_set.resize(new_node_cnt); new_edges.resize(new_node_cnt); for (size_t i = 0; i < fsm.edges.size(); i++) { const auto& edges = fsm.edges[i]; for (const auto& edge : edges) { if (edge.IsEpsilon() && old_to_new[i] == old_to_new[edge.target]) { continue; } new_edges_set[old_to_new[i]].insert({edge.min, edge.max, old_to_new[edge.target]}); } } for (size_t i = 0; i < new_edges_set.size(); i++) { for (const auto& edge : new_edges_set[i]) { new_edges[i].emplace_back(edge.min, edge.max, edge.target); } } fsm.edges = new_edges; return; } Result RegexIR::visit(const RegexIR::Leaf& node) const { FSMWithStartEnd result(node.regex); return Result::Ok(result); } Result RegexIR::visit(const RegexIR::Union& node) const { std::vector fsm_list; for (const auto& child : node.nodes) { auto visited = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, child); if (visited.IsErr()) { return visited; } fsm_list.push_back(visited.Unwrap()); } if (fsm_list.size() <= 1) { return Result::Err(std::make_shared("Invalid union")); } return Result::Ok(FSMWithStartEnd::Union(fsm_list)); } Result RegexIR::visit(const RegexIR::Symbol& node) const { if (node.node.size() != 1) { return Result::Err(std::make_shared("Invalid symbol")); } Result child = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, node.node[0]); if (child.IsErr()) { return child; } FSMWithStartEnd result; switch (node.symbol) { case RegexIR::RegexSymbol::plus: { result = child.Unwrap().MakePlus(); break; } case RegexIR::RegexSymbol::star: { result = child.Unwrap().MakeStar(); break; } case RegexIR::RegexSymbol::optional: { result = child.Unwrap().MakeOptional(); break; } } return Result::Ok(result); } Result RegexIR::visit(const RegexIR::Bracket& node) const { std::vector fsm_list; for (const auto& child : node.nodes) { auto visited = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, child); if (visited.IsErr()) { return visited; } fsm_list.push_back(visited.Unwrap()); } if (fsm_list.empty()) { return Result::Err(std::make_shared("Invalid bracket")); } return Result::Ok(FSMWithStartEnd::Concatenate(fsm_list)); } Result RegexIR::visit(const RegexIR::Repeat& node) const { if (node.nodes.size() != 1) { return Result::Err(std::make_shared("Invalid repeat")); } Result child = std::visit([&](auto&& arg) { return RegexIR::visit(arg); }, node.nodes[0]); if (child.IsErr()) { return child; } FSMWithStartEnd result; result = child.Unwrap(); std::unordered_set new_ends; if (node.lower_bound == 1) { for (const auto& end : result.ends) { new_ends.insert(end); } } // Handling {n,} if (node.upper_bound == RegexIR::REPEATNOUPPERBOUND) { for (int i = 2; i < node.lower_bound; i++) { result = FSMWithStartEnd::Concatenate(std::vector{result, child.Unwrap()}); } int one_of_end_node = *result.ends.begin(); result = FSMWithStartEnd::Concatenate(std::vector{result, child.Unwrap()}); for (const auto& end : result.ends) { result.fsm.edges[end].emplace_back(-1, -1, one_of_end_node); } return Result::Ok(result); } // Handling {n, m} or {n} for (int i = 2; i <= node.upper_bound; i++) { result = FSMWithStartEnd::Concatenate(std::vector{result, child.Unwrap()}); if (i >= node.lower_bound) { for (const auto& end : result.ends) { new_ends.insert(end); } } } result.ends = new_ends; return Result::Ok(result); } Result RegexToFSM(const std::string& regex) { RegexIR ir; using IRNode = std::variant; // We use a stack to store the nodes. std::stack stack; int left_middle_bracket = -1; for (size_t i = 0; i < regex.size(); i++) { if (i == 0 && regex[i] == '^') { continue; } if (i == regex.size() - 1 && regex[i] == '$') { continue; } // Handle The class. if (regex[i] == '[') { if (left_middle_bracket != -1) { return Result::Err(std::make_shared("Nested middle bracket!")); } left_middle_bracket = i; continue; } if (regex[i] == ']') { if (left_middle_bracket == -1) { return Result::Err(std::make_shared("Invalid middle bracket!")); } RegexIR::Leaf leaf; leaf.regex = regex.substr(left_middle_bracket, i - left_middle_bracket + 1); stack.push(leaf); left_middle_bracket = -1; continue; } if (left_middle_bracket != -1) { if (regex[i] == '\\') { i++; } continue; } if (regex[i] == '+' || regex[i] == '*' || regex[i] == '?') { if (stack.empty()) { return Result::Err( std::make_shared("Invalid regex: no node before operator!") ); } auto node = stack.top(); if (std::holds_alternative(node)) { return Result::Err( std::make_shared("Invalid regex: no node before operator!") ); } stack.pop(); auto child = std::get(node); RegexIR::Symbol symbol; symbol.node.push_back(child); switch (regex[i]) { case '+': { symbol.symbol = RegexIR::RegexSymbol::plus; break; } case '*': { symbol.symbol = RegexIR::RegexSymbol::star; break; } case '?': { symbol.symbol = RegexIR::RegexSymbol::optional; break; } } stack.push(symbol); continue; } if (regex[i] == '(' || regex[i] == '|') { stack.push(regex[i]); if (i < regex.size() - 2 && regex[i] == '(' && regex[i + 1] == '?' && regex[i + 2] == ':') { i += 2; continue; } if (i < regex.size() - 2 && regex[i] == '(' && regex[i + 1] == '?' && (regex[i + 2] == '!' || regex[i + 2] == '=')) { i += 2; // TODO(Linzhang Li): Handling the lookahead. continue; } continue; } if (regex[i] == ')') { std::stack nodes; bool paired = false; bool unioned = false; while ((!stack.empty()) && (!paired)) { auto node = stack.top(); stack.pop(); if (std::holds_alternative(node)) { char c = std::get(node); if (c == '(') { paired = true; break; } if (c == '|') { unioned = true; } nodes.push(node); } else { nodes.push(node); } } if (!paired) { return Result::Err( std::make_shared("Invalid regex: no paired bracket!" + std::to_string(__LINE__)) ); } if (nodes.empty()) { continue; } if (!unioned) { RegexIR::Bracket bracket; while (!nodes.empty()) { auto node = nodes.top(); nodes.pop(); auto child = std::get(node); bracket.nodes.push_back(child); } stack.push(bracket); } else { RegexIR::Union union_node; RegexIR::Bracket bracket; while (!nodes.empty()) { auto node = nodes.top(); nodes.pop(); if (std::holds_alternative(node)) { char c = std::get(node); if (c == '|') { union_node.nodes.push_back(bracket); bracket.nodes.clear(); continue; } return Result::Err(std::make_shared( "Invalid regex: no paired bracket!" + std::to_string(__LINE__) )); } if (std::holds_alternative(node)) { auto child = std::get(node); bracket.nodes.push_back(child); continue; } return Result::Err(std::make_shared( "Invalid regex: no paired bracket!" + std::to_string(__LINE__) )); } union_node.nodes.push_back(bracket); stack.push(union_node); } continue; } if (regex[i] == '{') { if (stack.empty()) { return Result::Err( std::make_shared("Invalid regex: no node before repeat!") ); } auto node = stack.top(); if (std::holds_alternative(node)) { return Result::Err( std::make_shared("Invalid regex: no node before repeat!") ); } stack.pop(); auto bounds_result = CheckRepeat(regex, i); if (bounds_result.IsErr()) { return Result::Err(bounds_result.UnwrapErr()); } auto child = std::get(node); RegexIR::Repeat repeat; repeat.lower_bound = bounds_result.Unwrap().first; repeat.upper_bound = bounds_result.Unwrap().second; repeat.nodes.push_back(child); stack.push(repeat); continue; } RegexIR::Leaf leaf; if (regex[i] != '\\') { leaf.regex = regex[i]; } else { leaf.regex = regex.substr(i, 2); i++; } stack.push(leaf); continue; } std::vector res_nodes; std::vector union_node_list; bool unioned = false; while (!stack.empty()) { if (std::holds_alternative(stack.top())) { char c = std::get(stack.top()); if (c == '|') { union_node_list.push_back(res_nodes); res_nodes.clear(); unioned = true; stack.pop(); continue; } return Result::Err(std::make_shared("Invalid regex: no paired!")); } auto node = stack.top(); stack.pop(); auto child = std::get(node); res_nodes.push_back(std::move(child)); } if (!unioned) { for (auto it = res_nodes.rbegin(); it != res_nodes.rend(); ++it) { ir.nodes.push_back(std::move(*it)); } } else { union_node_list.push_back(res_nodes); RegexIR::Union union_node; for (auto it = union_node_list.begin(); it != union_node_list.end(); ++it) { RegexIR::Bracket bracket; for (auto node = it->rbegin(); node != it->rend(); ++node) { bracket.nodes.push_back(std::move(*node)); } union_node.nodes.push_back(std::move(bracket)); } ir.nodes.push_back(std::move(union_node)); } return ir.Build(); } Result RegexIR::Build() const { if (nodes.empty()) { FSMWithStartEnd result; result.is_dfa = false; result.start = 0; result.fsm.edges.push_back(std::vector()); return Result::Ok(result); } std::vector fsm_list; for (const auto& node : nodes) { auto visited = std::visit([&](auto&& arg) { return visit(arg); }, node); if (visited.IsErr()) { return visited; } fsm_list.push_back(visited.Unwrap()); } return Result::Ok(FSMWithStartEnd::Concatenate(fsm_list)); } Result> CheckRepeat(const std::string& regex, size_t& start) { if (regex[start] != '{') { return Result>::Err(std::make_shared("Invalid regex: invalid repeat!" )); } start++; int lower_bound = 0; int upper_bound = 0; while (start < regex.size() && regex[start] == ' ') { start++; } while (start < regex.size() && regex[start] >= '0' && regex[start] <= '9') { lower_bound = lower_bound * 10 + (regex[start] - '0'); start++; } while (start < regex.size() && regex[start] == ' ') { start++; } if (start >= regex.size() || (regex[start] != ',' && regex[start] != '}')) { return Result>::Err(std::make_shared("Invalid regex: invalid repeat!" )); } if (regex[start] == '}') { upper_bound = lower_bound; } else { start++; while (start < regex.size() && regex[start] == ' ') { start++; } if (start < regex.size() && regex[start] == '}') { upper_bound = RegexIR::REPEATNOUPPERBOUND; return Result>::Ok(std::make_pair(lower_bound, upper_bound)); } while (start < regex.size() && regex[start] >= '0' && regex[start] <= '9') { upper_bound = upper_bound * 10 + (regex[start] - '0'); start++; } while (start < regex.size() && regex[start] == ' ') { start++; } if (start >= regex.size() || regex[start] != '}') { return Result>::Err( std::make_shared("Invalid regex: invalid repeat!") ); } } return Result>::Ok(std::make_pair(lower_bound, upper_bound)); } void FSMWithStartEnd::GetPossibleRules(const int& state, std::unordered_set* rules) const { rules->clear(); for (const auto& edge : fsm.edges[state]) { if (edge.IsRuleRef()) { rules->insert(edge.GetRefRuleId()); } } return; } void CompactFSMWithStartEnd::GetPossibleRules(const int& state, std::unordered_set* rules) const { rules->clear(); for (const auto& edge : fsm.edges[state]) { if (edge.IsRuleRef()) { rules->insert(edge.GetRefRuleId()); } } return; } std::ostream& operator<<(std::ostream& os, const FSMWithStartEnd& fsm) { os << "FSM(num_nodes=" << fsm.NumNodes() << ", start=" << fsm.StartNode() << ", end=["; for (auto end = fsm.ends.begin(); end != fsm.ends.end(); ++end) { os << *end; if (std::next(end) != fsm.ends.end()) { os << ", "; } } os << "], edges=[\n"; for (int i = 0; i < fsm.NumNodes(); ++i) { os << i << ": ["; const auto& edges = fsm.fsm.edges[i]; for (int j = 0; j < static_cast(edges.size()); ++j) { const auto& edge = edges[j]; if (edge.min == edge.max) { os << "(" << edge.min << ")->" << edge.target; } else { os << "(" << edge.min << ", " << edge.max << ")->" << edge.target; } if (j < static_cast(edges.size()) - 1) { os << ", "; } } os << "]\n"; } os << "])"; return os; } FSMWithStartEnd BuildTrie( const std::vector& patterns, std::vector* end_nodes ) { FSMWithStartEnd fsm(1); fsm.SetStartNode(0); if (end_nodes) { end_nodes->clear(); } for (const auto& pattern : patterns) { int current_node = 0; for (const auto& ch : pattern) { int16_t ch_int16 = static_cast(static_cast(ch)); int next_node = fsm.Transition(current_node, ch_int16); if (next_node == FSMWithStartEnd::NO_TRANSITION) { next_node = fsm.AddNode(); fsm.AddEdge(current_node, next_node, ch_int16, ch_int16); } current_node = next_node; } fsm.AddEndNode(current_node); if (end_nodes) { end_nodes->push_back(current_node); } } return fsm; } } // namespace xgrammar xgrammar-0.1.19/cpp/fsm.h000066400000000000000000000342251500705317600151520ustar00rootroot00000000000000/*! * Copyright (c) 2023 by Contributors * \file xgrammar/fsm.h */ #ifndef XGRAMMAR_FSM_H_ #define XGRAMMAR_FSM_H_ #include #include #include #include #include #include #include #include #include #include "../cpp/support/csr_array.h" namespace xgrammar { struct FSMEdge { /* The min and max are used to represent the range of characters. When min == -1 and max == -1, it means the edge is an epsilon transition. When min == -1 and max >= 0, then max represents the rule id. When min >= 0 and max >= 0, then it represents a range of characters. target is the target state id. */ short min, max; int target; FSMEdge(const short& _min, const short& _max, const int& target); /*! \brief Check if the edge is an epsilon transition. */ bool IsEpsilon() const; /*! \brief Check if the edge is a rule reference. */ bool IsRuleRef() const; /*! \brief Get the rule id of the edge. \return The rule id of the edge. \throw std::runtime_error if the edge is not a rule reference. */ short GetRefRuleId() const; /*! \brief Check if the edge is a character range. */ bool IsCharRange() const; }; class CompactFSM; class FSM { private: /*! \brief Get the epsilon closure of a state. \param state_set The current states. \param result The epsilon closure of the state. If nullptr, then the result will be stored in state_set. */ void GetEpsilonClosure( std::unordered_set* state_set, std::unordered_set* result = nullptr ) const; public: using Edge = FSMEdge; /*! \brief Transform a FSM to a compact FSM. \return The compact FSM. */ CompactFSM ToCompact(); /*! \brief Advance the FSM to the next state. \param from The current states. \param value The input value. \param result The next states, which can be seen as the result of the transition. \param is_closure Whether from is an epsilon closure. \param is_rule Whether the input value is a rule id. */ void Advance( const std::vector& from, int value, std::vector* result, bool is_closure = false, bool is_rule = false ) const; /*! \brief Return a copy of the FSM. */ FSM Copy() const; std::vector> edges; FSM() = default; friend class FSMWithStartEnd; }; class FSMWithStartEnd { public: bool is_dfa = false; FSM fsm; int start; std::unordered_set ends; /*! \brief Rebuild the FSM with the new state ids. \param old_to_new The mapping from old state ids to new state ids. */ void RebuildFSM(std::unordered_map& old_to_new, const int& new_node_cnt); /*! \brief Construct a FSM from a regex string. \details The regex string should only be the format like "abx" or [a-c0-9]. \details Any symbols like "a|b" or "a*b" are not supported. \param regex The regex string. */ FSMWithStartEnd(const std::string& regex); /*! \brief Assume the FSM accepts rule1, then the FSM will accept rule1*. \return The FSM that accepts rule1*. */ FSMWithStartEnd MakeStar() const; /*! \brief Assume the FSM accepts rule1, then the FSM will accept rule1+. \return The FSM that accepts rule1+. */ FSMWithStartEnd MakePlus() const; /*! \brief Assume the FSM accepts rule1, then the FSM will accept rule1?. \return The FSM that accepts rule1?. */ FSMWithStartEnd MakeOptional() const; /*! \brief Transform the FSM to a DFA. \return The DFA. */ FSMWithStartEnd ToDFA() const; /*! \brief Transform the FSM to accept the complement of the language. \return The complement FSM. */ FSMWithStartEnd Not() const; /*! \brief Minimize the DFA. \return The minimized DFA. */ FSMWithStartEnd MinimizeDFA() const; /*! \brief Return a copy of the FSM. \return The copy of the FSM. */ FSMWithStartEnd Copy() const; /*! \brief Print the FSM. \return The string representation of the FSM. */ std::string Print() const; /*! \brief Intersect the FSMs. \param lhs The left FSM. \param rhs The right FSM. \return The intersection of the FSMs. */ static Result Intersect( const FSMWithStartEnd& lhs, const FSMWithStartEnd& rhs, const int& num_of_nodes_limited = 1e6 ); /*! \brief Union the FSMs. \param fsms The FSMs to be unioned. \return The union of the FSMs. */ static FSMWithStartEnd Union(const std::vector& fsms); /*! \brief Concatenate the FSMs. \param fsms The FSMs to be concatenated, which should be in order. \return The concatenation of the FSMs. */ static FSMWithStartEnd Concatenate(const std::vector& fsms); /*! \brief Check if the FSM accepts the string. \param str The input string. \return True if the FSM accepts the string, false otherwise. */ bool Check(const std::string& str) const; /*! \brief Constructs an FSM with the specified number of nodes. */ FSMWithStartEnd(int num_nodes = 0, bool is_dfa = false) : is_dfa(is_dfa) { for (int i = 0; i < num_nodes; ++i) { fsm.edges.emplace_back(); } } inline static constexpr int NO_TRANSITION = -1; /*! * \brief Transitions from a given state based on an input character. * \param from The source state to transition from. * \param character The input character. * \return The target state if a valid transition exists, -1 otherwise. */ int Transition(int from, int16_t character) const { auto& edges = fsm.edges[from]; for (const auto& edge : edges) { if (edge.min <= character && edge.max >= character) { return edge.target; } } return NO_TRANSITION; } /*! \brief Returns the start node of the FSM. */ int StartNode() const { return start; } /*! * \brief Checks if a given node is an end/accepting state. * \param node The node to check. * \return True if the node is an end state, false otherwise. */ bool IsEndNode(int node) const { return std::any_of(ends.begin(), ends.end(), [node](int end_node) { return end_node == node; }); } /*! \brief Returns the total number of nodes in the FSM. */ int NumNodes() const { return fsm.edges.size(); } /*! * \brief Adds a transition edge between states with a character range. * \param from The source state. * \param to The target state. * \param min_ch The minimum character in the range (inclusive). * \param max_ch The maximum character in the range (inclusive). */ void AddEdge(int from, int to, int16_t min_ch, int16_t max_ch) { fsm.edges[from].push_back({min_ch, max_ch, to}); } /*! * \brief Adds a new node to the FSM. * \return The index of the newly added node. */ int AddNode() { fsm.edges.emplace_back(); return fsm.edges.size() - 1; } /*! * \brief Sets the start node of the FSM. * \param node The node to set as the start node. */ void SetStartNode(int node) { start = node; } /*! * \brief Adds an end/accepting node to the FSM. * \param node The node to add as an end node. */ void AddEndNode(int node) { ends.insert(node); } /*! \brief Check if the FSM is a DFA. \return True if the FSM is a DFA, false otherwise. */ bool IsDFA(); /*! \brief Check if the FSM is a leaf FSM. \return True if the FSM is a leaf FSM, false otherwise. */ bool IsLeaf() const; /*! \brief Merge some nodes by removing some epsilon transitions. \details For example, a -- \epsilon --> b, and b doesn't have \details any other inward edges, then we can merge the two nodes. */ void SimplifyEpsilon(); /*! \brief Merge some nodes which are approximately the same. \details Actually, if two nodes have the same outward edges, \details or the same inward edges, then we can merge them. */ void SimplifyTransition(); /*! \brief Get all the possible rule numbers for a given node. \param node_num The node number. \param rules The set of possible rule numbers. */ void GetPossibleRules(const int& node_num, std::unordered_set* rules) const; friend std::ostream& operator<<(std::ostream& os, const FSMWithStartEnd& fsm); }; class CompactFSM { public: /*! \brief Get the epsilon closure of a state. \param state_set The current states. \param result The epsilon closure of the state. If nullptr, then the result will be stored in state_set. */ void GetEpsilonClosure( std::unordered_set* state_set, std::unordered_set* result = nullptr ) const; /*! \brief Advance the FSM to the next state. \param from The current states. \param value The input value. \param result The next states, which can be seen as the result of the transition. \param is_closure Whether from is an epsilon closure. \param is_rule Whether the input value is a rule id. */ void Advance( const std::vector& from, int value, std::vector* result, bool is_closure = false, bool is_rule = false ) const; /*! \brief Transform the compact FSM to a FSM. \return The FSM. */ FSM ToFSM(); // The internal states are also public using Edge = FSMEdge; CSRArray edges; friend class CompactFSMWithStartEnd; }; class CompactFSMWithStartEnd { public: bool is_dfa = false; CompactFSM fsm; int start; std::unordered_set ends; using Edge = FSMEdge; /*! \brief Print the FSM. \return The string representation of the FSM. */ std::string Print() const; /*! \brief Check if the FSM accepts the string. \param str The input string. \return True if the FSM accepts the string, false otherwise. */ bool Check(const std::string& str) const; inline static constexpr int NO_TRANSITION = -1; int Transition(int from, int16_t character) const { auto edges = fsm.edges[from]; // TODO(yixin): test correctness for both cases if (edges.size() <= 16) { for (const auto& edge : edges) { if (edge.min > character) { return NO_TRANSITION; } else if (edge.max >= character) { return edge.target; } } return NO_TRANSITION; } else { auto it = std::lower_bound( edges.begin(), edges.end(), character, [](const Edge& edge, int16_t character) { return edge.min <= character; } ); if (it != edges.end() && it->min <= character) { return it->target; } return NO_TRANSITION; } } /*! \brief Returns the start node of the FSM. */ int StartNode() const { return start; } /*! * \brief Checks if a given node is an end/accepting state. * \param node The node to check. * \return True if the node is an end state, false otherwise. */ bool IsEndNode(int node) const { return std::any_of(ends.begin(), ends.end(), [node](int end_node) { return end_node == node; }); } /*! \brief Returns the total number of nodes in the FSM. */ int NumNodes() const { return fsm.edges.Size(); } friend std::ostream& operator<<(std::ostream& os, const CompactFSM& fsm); friend std::size_t MemorySize(const CompactFSMWithStartEnd& self) { return MemorySize(self.fsm.edges) + MemorySize(self.ends); } /*! \brief Get all the possible rule numbers for a given node. \param node_num The node number. \param rules The set of possible rule numbers.s */ void GetPossibleRules(const int& node_num, std::unordered_set* rules) const; }; /*! \brief Converts a regex string to a FSM. The parsing range is [start, end). \param regex The regex string. \return The FSM with start and end states. */ Result RegexToFSM(const std::string& regex); class RegexIR { public: struct Leaf; struct Symbol; struct Union; struct Bracket; struct Repeat; static constexpr int REPEATNOUPPERBOUND = -1; using Node = std::variant; // This struct is used to store the string in regex, or // the character class in regex. struct Leaf { std::string regex; }; // This struct is used to store the symbol in regex, i.e. // +, *, ? enum class RegexSymbol { star, plus, optional, }; struct Bracket { std::vector nodes; }; struct Symbol { RegexSymbol symbol; std::vector node; }; // This struct is used to represent a union symbol. struct Union { std::vector nodes; }; struct Repeat { std::vector nodes; int lower_bound = 0; int upper_bound = 0; }; struct LookAhead { bool is_positive; std::vector nodes; }; // This struct is used to represent a bracket in regex. std::vector nodes; /*! \brief Constructs a NFA from the regex IR. */ Result Build() const; /*! \brief the visit function for the variant. */ Result visit(const Leaf& node) const; Result visit(const Symbol& node) const; Result visit(const Union& node) const; Result visit(const Bracket& node) const; Result visit(const Repeat& node) const; Result visit(const LookAhead& node) const; }; /*! \brief Check repeat in regex. i.e {...} and {...,...} \param regex The regex string. \param start The start position of the repeat. i.e. regex[start] == '{'. After the function, start will be the position of '}'. \return The repeat range. */ Result> CheckRepeat(const std::string& regex, size_t& start); /*! \brief Handle escape characters. \param regex the corresponding string. \param start the pos escape characters start. */ std::vector> HandleEscapes(const std::string& regex, int start); /*! \brief Build a FSM from a list of patterns. \param patterns The patterns to be built. \param end_nodes The end nodes of the FSM. \return The FSM with start and end states. */ FSMWithStartEnd BuildTrie( const std::vector& patterns, std::vector* end_nodes = nullptr ); std::ostream& operator<<(std::ostream& os, const FSMWithStartEnd& fsm); } // namespace xgrammar #endif // XGRAMMAR_FSM_H_ xgrammar-0.1.19/cpp/grammar.cc000066400000000000000000000113571500705317600161520ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar.cc */ #include #include "grammar_functor.h" #include "grammar_parser.h" #include "grammar_serializer.h" #include "json_schema_converter.h" #include "regex_converter.h" #include "structural_tag.h" namespace xgrammar { std::string Grammar::ToString() const { return GrammarPrinter(*this).ToString(); } Grammar Grammar::FromEBNF(const std::string& ebnf_string, const std::string& root_rule_name) { auto grammar = ParseEBNF(ebnf_string, root_rule_name); grammar = GrammarNormalizer().Apply(grammar); return grammar; } Grammar Grammar::FromJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode, bool print_converted_ebnf ) { auto ebnf_string = JSONSchemaToEBNF(schema, any_whitespace, indent, separators, strict_mode); if (print_converted_ebnf) { XGRAMMAR_LOG(INFO) << "Converted EBNF: " << ebnf_string << std::endl; } return FromEBNF(ebnf_string); } Grammar Grammar::FromRegex(const std::string& regex, bool print_converted_ebnf) { auto ebnf_string = RegexToEBNF(regex); if (print_converted_ebnf) { XGRAMMAR_LOG(INFO) << "Converted EBNF: " << ebnf_string << std::endl; } return FromEBNF(ebnf_string); } Grammar Grammar::FromStructuralTag( const std::vector& tags, const std::vector& triggers ) { return StructuralTagToGrammar(tags, triggers); } // Optimized json grammar for the speed of the grammar matcher const std::string kJSONGrammarString = R"( root ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace ) value_non_str ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace | "0" fraction exponent | [1-9] [0-9]* fraction exponent | "-" [0-9] fraction exponent | "-" [1-9] [0-9]* fraction exponent | "true" | "false" | "null" ) (= [ \n\t,}\]]) members_and_embrace ::= ("\"" characters_and_colon [ \n\t]* members_suffix | "}") (= [ \n\t,}\]]) members_suffix ::= ( value_non_str [ \n\t]* member_suffix_suffix | "\"" characters_and_embrace | "\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) member_suffix_suffix ::= ( "}" | "," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) elements_or_embrace ::= ( "{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]" | "[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]" | "\"" characters_item elements_rest [ \n\t]* "]" | "0" fraction exponent elements_rest [ \n\t]* "]" | [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "-" "0" fraction exponent elements_rest [ \n\t]* "]" | "-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "true" elements_rest [ \n\t]* "]" | "false" elements_rest [ \n\t]* "]" | "null" elements_rest [ \n\t]* "]" | "]" ) elements ::= ( "{" [ \n\t]* members_and_embrace elements_rest | "[" [ \n\t]* elements_or_embrace elements_rest | "\"" characters_item elements_rest | "0" fraction exponent elements_rest | [1-9] [0-9]* fraction exponent elements_rest | "-" [0-9] fraction exponent elements_rest | "-" [1-9] [0-9]* fraction exponent elements_rest | "true" elements_rest | "false" elements_rest | "null" elements_rest ) elements_rest ::= ( "" | [ \n\t]* "," [ \n\t]* elements ) characters_and_colon ::= ( "\"" [ \n\t]* ":" | [^"\\\x00-\x1F] characters_and_colon | "\\" escape characters_and_colon ) (=[ \n\t]* [\"{[0-9tfn-]) characters_and_comma ::= ( "\"" [ \n\t]* "," | [^"\\\x00-\x1F] characters_and_comma | "\\" escape characters_and_comma ) (=[ \n\t]* "\"") characters_and_embrace ::= ( "\"" [ \n\t]* "}" | [^"\\\x00-\x1F] characters_and_embrace | "\\" escape characters_and_embrace ) (=[ \n\t]* [},]) characters_item ::= ( "\"" | [^"\\\x00-\x1F] characters_item | "\\" escape characters_item ) (= [ \n\t]* [,\]]) escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] fraction ::= "" | "." [0-9] [0-9]* exponent ::= "" | "e" sign [0-9] [0-9]* | "E" sign [0-9] [0-9]* sign ::= "" | "+" | "-" )"; Grammar Grammar::BuiltinJSONGrammar() { static const Grammar grammar = FromEBNF(kJSONGrammarString); return grammar; } Grammar Grammar::Union(const std::vector& grammars) { return GrammarUnionFunctor::Apply(grammars); } Grammar Grammar::Concat(const std::vector& grammars) { return GrammarConcatFunctor::Apply(grammars); } std::ostream& operator<<(std::ostream& os, const Grammar& grammar) { os << grammar.ToString(); return os; } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_builder.h000066400000000000000000000255001500705317600175150ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_builder.h * \brief The header for the building the BNF AST. */ #ifndef XGRAMMAR_GRAMMAR_BUILDER_H_ #define XGRAMMAR_GRAMMAR_BUILDER_H_ #include #include #include #include "grammar_data_structure.h" namespace xgrammar { /*! * \brief Helper class to build a BNF grammar. */ class GrammarBuilder { public: using Rule = Grammar::Impl::Rule; using RuleExprType = Grammar::Impl::RuleExprType; using RuleExpr = Grammar::Impl::RuleExpr; /*! \brief Default constructor. Creates a new grammar object. */ GrammarBuilder() : grammar_(std::make_shared()) {} /*! \brief Constructor. Creates a new grammar object from an existing grammar. */ GrammarBuilder(const Grammar& grammar) : grammar_(std::make_shared(*grammar.operator->())) { for (int i = 0; i < static_cast(grammar->NumRules()); ++i) { auto rule = grammar->GetRule(i); rule_name_to_id_[rule.name] = i; } } /*! * \brief Get the result grammar. This function will also set the root rule to the rule with the * specified name. The rule should be already added to the grammar. * \param root_rule_name The name of the root rule. Default is "root". */ Grammar Get(const std::string& root_rule_name = "root") { int32_t root_rule_id = GetRuleId(root_rule_name); XGRAMMAR_CHECK(root_rule_id != -1) << "The root rule with name \"" << root_rule_name << "\" is not found."; return Get(root_rule_id); } /*! * \brief Get the result grammar. This function will also set the root rule to the rule with * the specified id. The rule should be already added to the grammar. * \param root_rule_id The id of the root rule. */ Grammar Get(int32_t root_rule_id) { XGRAMMAR_CHECK( root_rule_id >= 0 && root_rule_id < static_cast(grammar_->rules_.size()) ) << "The root rule id " << root_rule_id << " is out of bound."; grammar_->root_rule_id_ = root_rule_id; return Grammar(grammar_); } /****************** RuleExpr handling ******************/ /*! \brief Add a rule_expr and return the rule_expr id. */ int32_t AddRuleExpr(const RuleExpr& rule_expr) { grammar_->rule_expr_indptr_.push_back(grammar_->rule_expr_data_.size()); grammar_->rule_expr_data_.push_back(static_cast(rule_expr.type)); grammar_->rule_expr_data_.push_back(rule_expr.data_len); grammar_->rule_expr_data_.insert( grammar_->rule_expr_data_.end(), rule_expr.data, rule_expr.data + rule_expr.data_len ); return static_cast(grammar_->rule_expr_indptr_.size()) - 1; } /*! * \brief Add a RuleExpr for string stored in bytes. * \param bytes A vector of int32_t, each representing a byte (0~255) in the string. * The string is stored in int32 vector to match the storage format of the grammar. */ int32_t AddByteString(const std::vector& bytes) { return AddRuleExpr({RuleExprType::kByteString, bytes.data(), static_cast(bytes.size())} ); } /*! * \brief Add a RuleExpr for string stored in bytes. * \param str The string to be added. */ int32_t AddByteString(const std::string& str) { std::vector bytes; bytes.reserve(str.size()); for (char c : str) { bytes.push_back(static_cast(c)); } return AddRuleExpr({RuleExprType::kByteString, bytes.data(), static_cast(bytes.size())} ); } /*! * \brief One element of a character class, containing a lower and a upper bound. Both bounds are * inclusive. */ struct CharacterClassElement { int32_t lower; int32_t upper; }; /*! * \brief Add a RuleExpr for a character class. * \param elements A vector of CharacterClassElement, each containing a lower and a upper bound. * \param is_negative Whether the character class is negated. */ int32_t AddCharacterClass( const std::vector& elements, bool is_negative = false ) { std::vector data; data.reserve(1 + elements.size() * 2); data.push_back(static_cast(is_negative)); for (const auto& range : elements) { data.push_back(range.lower); data.push_back(range.upper); } return AddRuleExpr( {RuleExprType::kCharacterClass, data.data(), static_cast(data.size())} ); } /*! * \brief Add a RuleExpr for a star quantifier of a character class. * \param elements A vector of CharacterClassElement, each containing a lower and a upper bound. * \param is_negative Whether the character class is negated. */ int32_t AddCharacterClassStar( const std::vector& elements, bool is_negative = false ) { std::vector data; data.reserve(1 + elements.size() * 2); data.push_back(static_cast(is_negative)); for (const auto& range : elements) { data.push_back(range.lower); data.push_back(range.upper); } return AddRuleExpr( {RuleExprType::kCharacterClassStar, data.data(), static_cast(data.size())} ); } /*! \brief Add a RuleExpr for empty string.*/ int32_t AddEmptyStr() { return AddRuleExpr({RuleExprType::kEmptyStr, nullptr, 0}); } /*! \brief Add a RuleExpr for rule reference.*/ int32_t AddRuleRef(int32_t rule_id) { std::vector data; data.push_back(rule_id); return AddRuleExpr({RuleExprType::kRuleRef, data.data(), static_cast(data.size())}); } /*! \brief Add a RuleExpr for RuleExpr sequence.*/ int32_t AddSequence(const std::vector& elements) { return AddRuleExpr( {RuleExprType::kSequence, elements.data(), static_cast(elements.size())} ); } /*! \brief Add a RuleExpr for RuleExpr choices.*/ int32_t AddChoices(const std::vector& choices) { return AddRuleExpr( {RuleExprType::kChoices, choices.data(), static_cast(choices.size())} ); } /*! * \brief Add a RuleExpr for tag dispatch. * \param tag_dispatch_list A list of pairs of tag_expr_id and rule_id. */ int32_t AddTagDispatch(const std::vector>& tag_dispatch_list) { std::vector data; data.reserve(tag_dispatch_list.size() * 2); for (const auto& [tag_expr_id, rule_id] : tag_dispatch_list) { data.push_back(tag_expr_id); data.push_back(rule_id); } return AddRuleExpr({RuleExprType::kTagDispatch, data.data(), static_cast(data.size())} ); } size_t NumRuleExprs() const { return grammar_->NumRuleExprs(); } /*! \brief Get the rule_expr with the given id. */ RuleExpr GetRuleExpr(int32_t rule_expr_id) { return grammar_->GetRuleExpr(rule_expr_id); } /****************** Rule handling ******************/ /*! \brief Add a rule and return the rule id. */ int32_t AddRule(const Rule& rule) { int32_t id = grammar_->rules_.size(); auto rules = grammar_->rules_; grammar_->rules_.push_back(rule); XGRAMMAR_CHECK(rule_name_to_id_.count(rule.name) == 0); rule_name_to_id_[rule.name] = id; return id; } int32_t AddRule(const std::string& name, int32_t body_expr_id) { return AddRule({name, body_expr_id}); } int32_t AddRuleWithHint(const std::string& name_hint, int32_t body_expr_id) { return AddRule({GetNewRuleName(name_hint), body_expr_id}); } size_t NumRules() const { return grammar_->NumRules(); } /*! \brief Get the rule with the given id. */ const Rule& GetRule(int32_t rule_id) const { return grammar_->rules_[rule_id]; } /*! * \brief Add an rule without body, and return the rule id. The rule body should be set later * with GrammarBuilder::UpdateRuleBody. This method is useful for cases where the rule id is * required to build the rule body. * \sa GrammarBuilder::UpdateRuleBody */ int32_t AddEmptyRule(const std::string& name) { return AddRule({name, -1}); } /*! * \brief Update the rule body of the given rule, specified by rule id. Can be used to set the * rule body of a rule inserted by GrammarBuilder::AddEmptyRule. */ void UpdateRuleBody(int32_t rule_id, int32_t body_expr_id) { XGRAMMAR_CHECK(rule_id >= 0 && rule_id < static_cast(grammar_->rules_.size())) << "Rule id " << rule_id << " is out of range."; grammar_->rules_[rule_id].body_expr_id = body_expr_id; } /*! * \brief Update the rule body of the given rule, specified by rule name. Can be used to set the * rule body of a rule inserted by GrammarBuilder::AddEmptyRule. */ void UpdateRuleBody(std::string rule_name, int32_t body_expr_id) { int32_t rule_id = GetRuleId(rule_name); XGRAMMAR_CHECK(rule_id != -1) << "Rule " << rule_name << " is not found."; UpdateRuleBody(rule_id, body_expr_id); } /*! * \brief Add a lookahead assertion to a rule referred by the given rule_id. The lookahead * assertion should be a sequence RuleExpr id. An id of -1 means no lookahead assertion. */ void AddLookaheadAssertion(int32_t rule_id, int32_t lookahead_assertion_id) { XGRAMMAR_CHECK(rule_id < static_cast(grammar_->rules_.size())) << "Rule id " << rule_id << " is out of range."; XGRAMMAR_CHECK(grammar_->rules_[rule_id].lookahead_assertion_id == -1) << "Rule " << rule_id << " already has a lookahead assertion."; grammar_->rules_[rule_id].lookahead_assertion_id = lookahead_assertion_id; } /*! * \brief Add a lookahead assertion to a rule referred by the given name. The lookahead * assertion should be a sequence RuleExpr id. An id of -1 means no lookahead assertion. */ void AddLookaheadAssertion(std::string rule_name, int32_t lookahead_assertion_id) { int32_t rule_id = GetRuleId(rule_name); XGRAMMAR_CHECK(rule_id != -1) << "Rule " << rule_name << " is not found."; AddLookaheadAssertion(rule_id, lookahead_assertion_id); } /*! * \brief Find a name for a new rule starting with the given name hint. Some integer suffix (_1, * _2, ...) may be added to avoid name conflict. */ std::string GetNewRuleName(const std::string& name_hint) { if (rule_name_to_id_.count(name_hint) == 0) { return name_hint; } else { int cnt = 1; while (rule_name_to_id_.count(name_hint + "_" + std::to_string(cnt)) != 0) { ++cnt; } return name_hint + "_" + std::to_string(cnt); } } /*! * \brief Get the rule id of the rule with the given name. Return -1 if not found. */ int32_t GetRuleId(const std::string& name) const { auto it = rule_name_to_id_.find(name); if (it == rule_name_to_id_.end()) { return -1; } else { return it->second; } } private: // Mutable pointer to the grammar object. std::shared_ptr grammar_; // Map from rule name to rule id. std::unordered_map rule_name_to_id_; }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_BUILDER_H_ xgrammar-0.1.19/cpp/grammar_compiler.cc000066400000000000000000000662171500705317600200510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/compiler.cc */ #include #include #include #include "compiled_grammar_data_structure.h" #include "fsm.h" #include "grammar_data_structure.h" #include "grammar_functor.h" #include "grammar_matcher_base.h" #include "support/logging.h" #include "support/thread_pool.h" #include "support/thread_safe_cache.h" #include "support/utils.h" #include "testing.h" #include "xgrammar/grammar.h" namespace std { /*! \brief Define the hash function for StructuralTagItem. */ template <> struct hash { size_t operator()(const xgrammar::StructuralTagItem& tag) const { return xgrammar::HashCombine( std::hash{}(tag.begin), std::hash{}(tag.schema), std::hash{}(tag.end) ); } }; } // namespace std namespace xgrammar { /******************* MemorySize *******************/ std::size_t MemorySize(const Grammar::Impl& impl) { // we assume strings are not long, so we don't iterate through all the rules return impl.rules_.size() * sizeof(impl.rules_[0]) + MemorySize(impl.rule_expr_data_) + MemorySize(impl.rule_expr_indptr_) + MemorySize(impl.root_tag_dispatch_fsm) + MemorySize(impl.tag_dispatch_end_node_to_rule_id) + MemorySize(impl.allow_empty_rule_ids); } std::size_t Grammar::Impl::MemorySize() const { return xgrammar::MemorySize(*this); } std::size_t MemorySize(const AdaptiveTokenMask& mask) { return MemorySize(mask.uncertain_indices) + MemorySize(mask.accepted_indices) + MemorySize(mask.rejected_indices) + MemorySize(mask.accepted_bitset); } std::size_t AdaptiveTokenMask::MemorySize() const { return xgrammar::MemorySize(*this); } std::size_t CompiledGrammar::Impl::MemorySize() const { std::size_t sum = 0; sum += grammar->MemorySize(); sum += adaptive_token_mask_cache.size() * sizeof(*adaptive_token_mask_cache.begin()); for (auto& [_, mask] : adaptive_token_mask_cache) sum += mask.MemorySize(); return sum; } std::size_t CompiledGrammar::MemorySizeBytes() const { return pimpl_->MemorySize(); } /******************* AdaptiveTokenMask and CompiledGrammar *******************/ AdaptiveTokenMask::AdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, const std::vector& accepted_indices, const std::vector& rejected_indices, const std::vector& uncertain_indices ) { auto size_acc = accepted_indices.size(); auto size_rej = rejected_indices.size(); store_type = size_acc >= USE_BITSET_THRESHOLD && size_rej >= USE_BITSET_THRESHOLD ? StoreType::kAcceptedBitset : size_acc < size_rej ? StoreType::kAccepted : StoreType::kRejected; if (store_type == StoreType::kAcceptedBitset) { accepted_bitset = DynamicBitset(vocab_size); for (auto idx : accepted_indices) { accepted_bitset.Set(sorted_decoded_vocab[idx].first, true); } } else if (store_type == StoreType::kAccepted) { this->accepted_indices = accepted_indices; } else { this->rejected_indices = rejected_indices; } this->uncertain_indices = uncertain_indices; } std::string AdaptiveTokenMask::Print(const TokenizerInfo& tokenizer_info) const { constexpr int kMaxPrintTokens = 100; std::stringstream ss; const auto& sorted_decoded_vocab = tokenizer_info.GetSortedDecodedVocab(); std::vector accepted_indices; std::vector rejected_indices; std::unordered_set uncertain_indices_set( uncertain_indices.begin(), uncertain_indices.end() ); accepted_indices.reserve(sorted_decoded_vocab.size()); rejected_indices.reserve(sorted_decoded_vocab.size()); if (store_type == StoreType::kAcceptedBitset) { for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { if (uncertain_indices_set.count(i)) { continue; } if (accepted_bitset[i]) { accepted_indices.push_back(i); } else { rejected_indices.push_back(i); } } } else if (store_type == StoreType::kAccepted) { accepted_indices = this->accepted_indices; // Reject indices = [0, sorted_decoded_vocab.size()) \ accepted_indices \ uncertain_indices int acc_ptr = 0; for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { while (acc_ptr < static_cast(accepted_indices.size()) && accepted_indices[acc_ptr] < i) { ++acc_ptr; } if (acc_ptr < static_cast(accepted_indices.size()) && accepted_indices[acc_ptr] == i) { continue; } if (uncertain_indices_set.count(i)) { continue; } rejected_indices.push_back(i); } } else { XGRAMMAR_DCHECK(store_type == StoreType::kRejected); rejected_indices = this->rejected_indices; // Accepted indices = [0, sorted_decoded_vocab.size()) \ rejected_indices \ uncertain_indices int rej_ptr = 0; for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { while (rej_ptr < static_cast(rejected_indices.size()) && rejected_indices[rej_ptr] < i) { ++rej_ptr; } if (rej_ptr < static_cast(rejected_indices.size()) && rejected_indices[rej_ptr] == i) { continue; } if (uncertain_indices_set.count(i)) { continue; } accepted_indices.push_back(i); } } std::string storage_type_str = store_type == StoreType::kAcceptedBitset ? "AcceptedBitset" : store_type == StoreType::kAccepted ? "Accepted" : "Rejected"; ss << "AdaptiveTokenMask(num_tokens=" << sorted_decoded_vocab.size() << ", accepted_num=" << accepted_indices.size() << ", rejected_num=" << rejected_indices.size() << ", uncertain_num=" << uncertain_indices.size() << ", storage_type=" << storage_type_str << ",\n"; // Convert indices to token ids for printing std::vector accepted_token_ids; std::vector rejected_token_ids; std::vector uncertain_token_ids; accepted_token_ids.reserve(accepted_indices.size()); rejected_token_ids.reserve(rejected_indices.size()); uncertain_token_ids.reserve(uncertain_indices.size()); for (auto idx : accepted_indices) { accepted_token_ids.push_back(sorted_decoded_vocab[idx].first); } std::sort(accepted_token_ids.begin(), accepted_token_ids.end()); for (auto idx : rejected_indices) { rejected_token_ids.push_back(sorted_decoded_vocab[idx].first); } std::sort(rejected_token_ids.begin(), rejected_token_ids.end()); for (auto idx : uncertain_indices) { uncertain_token_ids.push_back(sorted_decoded_vocab[idx].first); } std::sort(uncertain_token_ids.begin(), uncertain_token_ids.end()); ss << "accepted=" << PrintTokenByIds(accepted_token_ids, tokenizer_info, kMaxPrintTokens) << ",\nrejected=" << PrintTokenByIds(rejected_token_ids, tokenizer_info, kMaxPrintTokens) << ",\nuncertain=" << PrintTokenByIds(uncertain_token_ids, tokenizer_info, kMaxPrintTokens) << "\n)"; return ss.str(); } Grammar CompiledGrammar::GetGrammar() const { return pimpl_->GetGrammar(); } TokenizerInfo CompiledGrammar::GetTokenizerInfo() const { return pimpl_->GetTokenizerInfo(); } /************** Use GrammarMatcher to generate the AdaptiveTokenMaskCache **************/ /*! \brief The concrete implementation of GrammarMatcherNode. */ class GrammarMatcherForTokenMaskCache : public GrammarMatcherBase { public: // Do not expand the initial stack element: we want to find the accepted/rejected tokens // that exactly start from the initial stack element. GrammarMatcherForTokenMaskCache(const Grammar& grammar, StackElement init_stack_element) : GrammarMatcherBase(grammar, init_stack_element, false), init_rule_id(init_stack_element.rule_id) {} /*! * \brief Get the adaptive token mask for the given StackElement. * \param is_root_rule Whether to consider the parent rule. If false, there will be * no uncertain tokens. Useful for the root rule. */ AdaptiveTokenMask GetAdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, bool is_root_rule ); private: /*! \brief Check if a token can pass the lookahead assertion. */ bool IsTokenPassLookaheadAssertion( const std::string& token, const std::vector& can_reach_end_stack ); // The id of the initial rule. int32_t init_rule_id; // Temporary data for GetAdaptiveTokenMask. std::vector tmp_accepted_indices_; std::vector tmp_rejected_indices_; std::vector tmp_uncertain_indices_; std::vector tmp_can_reach_end_stack_; std::vector tmp_can_reach_end_prefix_or_stack_; }; bool GrammarMatcherForTokenMaskCache::IsTokenPassLookaheadAssertion( const std::string& token, const std::vector& can_reach_end_stack ) { auto lookahead_assertion_id = grammar_->GetRule(init_rule_id).lookahead_assertion_id; if (lookahead_assertion_id == -1) { return true; } auto lookahead_stack_element = StackElement(-1, lookahead_assertion_id, 0); PushInitialState(lookahead_stack_element, true); int token_len = token.size(); // Find all positions that can come to and end. Then check if the suffix from that position // can be accepted by the lookahead assertion. for (int i = static_cast(can_reach_end_stack.size()) - 1; i >= 0; --i) { if (!can_reach_end_stack[i]) { continue; } int last_accept_pos = i - 1; for (int pos = i; pos < token_len; ++pos) { if (!AcceptChar(token[pos])) { break; } last_accept_pos = pos; // Case 1. The whole rule is finished. if (CanReachEnd()) { // accepted chars: pos - i + 1 // we need to rollback the pushed initial state as well RollbackChars(pos - i + 2); return true; } } // Case 2. The whole token is accepted if (last_accept_pos == token_len - 1) { RollbackChars(last_accept_pos - i + 2); return true; } // Case 3. The token is not accepted. Check the next position. RollbackChars(last_accept_pos - i + 1); } RollbackChars(1); return false; } AdaptiveTokenMask GrammarMatcherForTokenMaskCache::GetAdaptiveTokenMask( size_t vocab_size, const std::vector>& sorted_decoded_vocab, bool is_root_rule ) { tmp_accepted_indices_.clear(); tmp_rejected_indices_.clear(); tmp_uncertain_indices_.clear(); // For every character in the current token, stores whether it is possible to reach the end of // the rule when matching until this character. Store it in a stack for later rollback. tmp_can_reach_end_stack_.assign({CanReachEnd()}); tmp_can_reach_end_prefix_or_stack_.assign({tmp_can_reach_end_stack_.back()}); int prev_matched_size = 0; for (int i = 0; i < static_cast(sorted_decoded_vocab.size()); ++i) { const auto& token = sorted_decoded_vocab[i].second; bool accepted = true; // Many tokens may contain the same prefix, so we will avoid unnecessary matching // by finding the longest common prefix with the previous token. if (i > 0) { const auto& prev_token = sorted_decoded_vocab[i - 1].second; int lcp_len = std::mismatch(token.begin(), token.end(), prev_token.begin(), prev_token.end()).first - token.begin(); if (lcp_len > prev_matched_size) { // Case 1. The common prefix is rejected by the matcher in the last token. Reject // directly. accepted = false; } else if (lcp_len < prev_matched_size) { // Case 2. The common prefix is shorter than the previous matched size. Rollback // the non-common part. RollbackChars(prev_matched_size - lcp_len); tmp_can_reach_end_stack_.erase( tmp_can_reach_end_stack_.end() - (prev_matched_size - lcp_len), tmp_can_reach_end_stack_.end() ); tmp_can_reach_end_prefix_or_stack_.erase( tmp_can_reach_end_prefix_or_stack_.end() - (prev_matched_size - lcp_len), tmp_can_reach_end_prefix_or_stack_.end() ); } prev_matched_size = std::min(prev_matched_size, lcp_len); } if (accepted) { // Accept the rest chars one by one for (int j = prev_matched_size; j < static_cast(token.size()); ++j) { if (!AcceptChar(token[j], false)) { accepted = false; break; } tmp_can_reach_end_stack_.push_back(CanReachEnd()); tmp_can_reach_end_prefix_or_stack_.push_back( tmp_can_reach_end_stack_.back() || tmp_can_reach_end_prefix_or_stack_.back() ); prev_matched_size = j + 1; } } bool can_reach_end = tmp_can_reach_end_prefix_or_stack_.back(); if (accepted) { tmp_accepted_indices_.push_back(i); } else if (can_reach_end && !is_root_rule && IsTokenPassLookaheadAssertion(token, tmp_can_reach_end_stack_)) { // 1. If the current rule is the root rule (is_root_rule=true), there are no // uncertain tokens. Not accepted tokens are just rejected. // 2. If a token cannot pass the lookahead assertion, it is rejected. tmp_uncertain_indices_.push_back(i); } else { tmp_rejected_indices_.push_back(i); } } // Rollback the last matched part RollbackChars(prev_matched_size); return AdaptiveTokenMask( vocab_size, sorted_decoded_vocab, tmp_accepted_indices_, tmp_rejected_indices_, tmp_uncertain_indices_ ); } /******************* GrammarCompiler::Impl *******************/ using SchemaKey = std::tuple, std::pair, bool>; using StructuralTagKey = std::tuple, std::vector>; using GrammarKey = std::pair; class GrammarCompiler::Impl { public: Impl( const TokenizerInfo& tokenizer_info, int max_threads, bool cache_enabled, long long max_memory_bytes ) : tokenizer_info_(tokenizer_info), max_threads_(max_threads), cache_enabled_(cache_enabled), compile_builtin_json_grammar_cache_([&] { return CompileJson(); }), compile_cache_(static_cast(max_memory_bytes), *this) {} /*! * \brief Build the tag dispatch fsm for the root rule and store in the compiled grammar. */ void BuildTagDispatchFSM(Grammar grammar, const Grammar::Impl::RuleExpr& root_rule_expr); /*! \brief Multi-thread compile the grammar. */ CompiledGrammar MultiThreadCompileGrammar(Grammar grammar); /*! \brief Compile the built-in JSON grammar. */ CompiledGrammar CompileJson(); /*! * \brief Compile different types of grammars. * \attention This template function is marked as deleted. * User must explicitly specialize the template to support new key types. */ template CompiledGrammar Compute(const KeyType& key) = delete; /*! \brief Forwards the key to the corresponding compile function. */ template CompiledGrammar operator()(const KeyType& key) { return Compute(key); } CompiledGrammar CompileBuiltinJSONGrammar(); CompiledGrammar CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode = true ); CompiledGrammar CompileStructuralTag( const std::vector& tags, const std::vector& triggers ); CompiledGrammar CompileRegex(const std::string& regex); CompiledGrammar CompileGrammar(const Grammar& grammar); void ClearCache(); long long GetCacheSizeBytes() const; long long CacheLimitBytes() const; private: using MultipleKey = std::variant; struct Computer { Computer(Impl& compiler) : compiler(compiler) {} // dispatch the key to the corresponding compile function CompiledGrammar operator()(const MultipleKey& key) const { return std::visit(compiler, key); } GrammarCompiler::Impl& compiler; }; struct SizeEstimator { std::size_t operator()(const CompiledGrammar& value) const { return value.MemorySizeBytes(); } }; /*! \brief The vocabulary associated with this storage class. */ const TokenizerInfo tokenizer_info_; /*! \brief The maximum number of threads to use. */ const int max_threads_; /*! \brief Whether the cache is enabled. */ const bool cache_enabled_; ThreadSafeCache compile_builtin_json_grammar_cache_; ThreadSafeLRUCache compile_cache_; }; void GrammarCompiler::Impl::BuildTagDispatchFSM( Grammar grammar, const Grammar::Impl::RuleExpr& root_rule_expr ) { std::vector tags; std::vector rule_ids; for (int i = 0; i < root_rule_expr.size(); i += 2) { auto byte_string_expr = grammar->GetRuleExpr(root_rule_expr[i]); std::string tag; for (int j = 0; j < byte_string_expr.size(); ++j) { tag += static_cast(byte_string_expr[j]); } tags.push_back(tag); rule_ids.push_back(root_rule_expr[i + 1]); } std::vector end_nodes; FSMWithStartEnd trie = BuildTrie(tags, &end_nodes); CompactFSMWithStartEnd compacted_fsm; compacted_fsm.fsm = trie.fsm.ToCompact(); compacted_fsm.ends = trie.ends; compacted_fsm.start = trie.start; grammar->root_tag_dispatch_fsm = compacted_fsm; for (int i = 0; i < static_cast(end_nodes.size()); ++i) { grammar->tag_dispatch_end_node_to_rule_id[end_nodes[i]] = rule_ids[i]; } } CompiledGrammar GrammarCompiler::Impl::MultiThreadCompileGrammar(Grammar grammar) { using RuleExprType = Grammar::Impl::RuleExprType; auto compiled_grammar_impl = std::make_shared(); compiled_grammar_impl->grammar = grammar; compiled_grammar_impl->tokenizer_info = tokenizer_info_; // Step 1. Compute the ids of rules that can be empty compiled_grammar_impl->grammar->allow_empty_rule_ids = AllowEmptyRuleAnalyzer::Apply(grammar); // Step 2. Compute the root tag dispatch fsm auto root_rule_id = grammar->GetRootRuleId(); auto root_rule_expr = grammar->GetRuleExpr(grammar->GetRule(root_rule_id).body_expr_id); if (root_rule_expr.type == RuleExprType::kTagDispatch) { BuildTagDispatchFSM(compiled_grammar_impl->grammar, root_rule_expr); } if (tokenizer_info_.GetVocabSize() == 0) { return CompiledGrammar(compiled_grammar_impl); } // Step 3. Compute the adaptive token mask cache // The token mask cache is computed for these positions in the grammar: // 1. All character class or character class star (with last_utf8_bytes=0, 1, 2, 3) // 2. All byte strings (with element_in_string=0, 1, 2, ...) // since other positions will be expanded to the above positions // TODO(Charlie): Figure out how to support ThreadPool and std::mutex in WebAssembly. // Only declare ThreadPool and mutex if max_threads > 1, so when max_threads = 1, we do // not need ThreadPool or std::mutex, which throws error in runtime in WebAssembly. std::optional thread_pool; std::optional adaptive_token_mask_cache_mutex; if (max_threads_ > 1) { thread_pool.emplace(max_threads_); adaptive_token_mask_cache_mutex.emplace(); } auto add_adaptive_token_mask = [&](const StackElement& stack_element, bool is_root_rule) { auto grammar_matcher = GrammarMatcherForTokenMaskCache(grammar, stack_element); auto cur_adaptive_token_mask_cache = grammar_matcher.GetAdaptiveTokenMask( tokenizer_info_.GetVocabSize(), tokenizer_info_.GetSortedDecodedVocab(), is_root_rule ); if (max_threads_ > 1) { std::lock_guard lock(adaptive_token_mask_cache_mutex.value()); compiled_grammar_impl->adaptive_token_mask_cache[stack_element] = cur_adaptive_token_mask_cache; } else { compiled_grammar_impl->adaptive_token_mask_cache[stack_element] = cur_adaptive_token_mask_cache; } }; auto add_task_adaptive_token_mask = [&](const StackElement& stack_element, bool is_root_rule) { // Execute depending on whether we use thread_pool if (max_threads_ > 1) { thread_pool->Execute([add_adaptive_token_mask, stack_element, is_root_rule]() { add_adaptive_token_mask(stack_element, is_root_rule); }); } else { add_adaptive_token_mask(stack_element, is_root_rule); } }; for (int32_t rule_id = 0; rule_id < static_cast(grammar->NumRules()); ++rule_id) { auto rule = grammar->GetRule(rule_id); auto rule_body = grammar->GetRuleExpr(rule.body_expr_id); if (rule_body.type == RuleExprType::kTagDispatch) { auto cur_stack_element = StackElement(rule_id, rule.body_expr_id, 0); for (int i = 0; i < grammar->root_tag_dispatch_fsm->NumNodes(); ++i) { cur_stack_element.element_id = i; add_task_adaptive_token_mask(cur_stack_element, rule_id == root_rule_id); } continue; } XGRAMMAR_DCHECK(rule_body.type == RuleExprType::kChoices); for (auto sequence_id : rule_body) { auto sequence = grammar->GetRuleExpr(sequence_id); if (sequence.type == RuleExprType::kEmptyStr) { continue; } XGRAMMAR_DCHECK(sequence.type == RuleExprType::kSequence); for (int element_id = 0; element_id < sequence.size(); ++element_id) { auto element = grammar->GetRuleExpr(sequence[element_id]); if (element.type == RuleExprType::kRuleRef) { continue; } auto cur_stack_element = StackElement(rule_id, sequence_id, element_id); if (element.type == RuleExprType::kByteString) { for (int idx = 0; idx < element.size(); ++idx) { cur_stack_element.element_in_string = idx; add_task_adaptive_token_mask(cur_stack_element, rule_id == root_rule_id); } } else { XGRAMMAR_DCHECK( element.type == RuleExprType::kCharacterClassStar || element.type == RuleExprType::kCharacterClass ); for (int left_utf8_bytes = 0; left_utf8_bytes <= 3; ++left_utf8_bytes) { cur_stack_element.left_utf8_bytes = left_utf8_bytes; add_task_adaptive_token_mask(cur_stack_element, rule_id == root_rule_id); } } } } } if (max_threads_ > 1) { thread_pool->Join(); } return CompiledGrammar(compiled_grammar_impl); } CompiledGrammar GrammarCompiler::Impl::CompileJson() { return MultiThreadCompileGrammar(Grammar::BuiltinJSONGrammar()); } template <> CompiledGrammar GrammarCompiler::Impl::Compute(const SchemaKey& key) { const auto& [schema, any_whitespace, indent, separators, strict_mode] = key; auto grammar = Grammar::FromJSONSchema(schema, any_whitespace, indent, separators, strict_mode); return MultiThreadCompileGrammar(grammar); } template <> CompiledGrammar GrammarCompiler::Impl::Compute(const StructuralTagKey& key) { const auto& [tags, triggers] = key; return MultiThreadCompileGrammar(Grammar::FromStructuralTag(tags, triggers)); } template <> CompiledGrammar GrammarCompiler::Impl::Compute(const std::string& key) { return MultiThreadCompileGrammar(Grammar::FromRegex(key)); } template <> CompiledGrammar GrammarCompiler::Impl::Compute(const GrammarKey& key) { const auto& [grammar_str, root_rule_name] = key; return MultiThreadCompileGrammar(Grammar::FromEBNF(grammar_str, root_rule_name)); } CompiledGrammar GrammarCompiler::Impl::CompileBuiltinJSONGrammar() { if (!cache_enabled_) { return MultiThreadCompileGrammar(Grammar::BuiltinJSONGrammar()); } return compile_builtin_json_grammar_cache_.Get(); } CompiledGrammar GrammarCompiler::Impl::CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode ) { if (!cache_enabled_) { return MultiThreadCompileGrammar( Grammar::FromJSONSchema(schema, any_whitespace, indent, separators, strict_mode) ); } auto separators_value = separators.value_or( (indent == std::nullopt) ? std::make_pair(", ", ": ") : std::make_pair(",", ": ") ); auto key = std::make_tuple(schema, any_whitespace, indent, separators_value, strict_mode); return compile_cache_.Get(key); } CompiledGrammar GrammarCompiler::Impl::CompileStructuralTag( const std::vector& tags, const std::vector& triggers ) { if (!cache_enabled_) { return MultiThreadCompileGrammar(Grammar::FromStructuralTag(tags, triggers)); } auto key = std::make_tuple(tags, triggers); return compile_cache_.Get(key); } CompiledGrammar GrammarCompiler::Impl::CompileRegex(const std::string& regex) { if (!cache_enabled_) { return MultiThreadCompileGrammar(Grammar::FromRegex(regex)); } return compile_cache_.Get(regex); } CompiledGrammar GrammarCompiler::Impl::CompileGrammar(const Grammar& grammar) { if (!cache_enabled_) { return MultiThreadCompileGrammar(grammar); } auto key = std::make_pair(grammar.ToString(), grammar->GetRootRule().name); return compile_cache_.Get(key); } void GrammarCompiler::Impl::ClearCache() { compile_builtin_json_grammar_cache_.Clear(); compile_cache_.Clear(); } long long GrammarCompiler::Impl::GetCacheSizeBytes() const { return static_cast(compile_cache_.MemorySize()); } long long GrammarCompiler::Impl::CacheLimitBytes() const { const auto size = compile_cache_.MaxMemorySize(); if (size == compile_cache_.UNLIMITED_SIZE) return -1; return static_cast(size); } /******************* GrammarCompiler *******************/ GrammarCompiler::GrammarCompiler( const TokenizerInfo& tokenizer_info, int max_threads, bool cache_enabled, long long max_memory_bytes ) : pimpl_(std::make_shared(tokenizer_info, max_threads, cache_enabled, max_memory_bytes)) { if (max_memory_bytes < -1) { XGRAMMAR_LOG(FATAL) << "Invalid max_memory_bytes: " << max_memory_bytes << ". " << "It should be -1 (unlimited) or a non-negative integer."; } } CompiledGrammar GrammarCompiler::CompileJSONSchema( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode ) { return pimpl_->CompileJSONSchema(schema, any_whitespace, indent, separators, strict_mode); } CompiledGrammar GrammarCompiler::CompileBuiltinJSONGrammar() { return pimpl_->CompileBuiltinJSONGrammar(); } CompiledGrammar GrammarCompiler::CompileStructuralTag( const std::vector& tags, const std::vector& triggers ) { return pimpl_->CompileStructuralTag(tags, triggers); } CompiledGrammar GrammarCompiler::CompileRegex(const std::string& regex) { return pimpl_->CompileRegex(regex); } CompiledGrammar GrammarCompiler::CompileGrammar(const Grammar& grammar) { return pimpl_->CompileGrammar(grammar); } void GrammarCompiler::ClearCache() { pimpl_->ClearCache(); } long long GrammarCompiler::GetCacheSizeBytes() const { return pimpl_->GetCacheSizeBytes(); } long long GrammarCompiler::CacheLimitBytes() const { return pimpl_->CacheLimitBytes(); } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_data_structure.h000066400000000000000000000164321500705317600211240ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar.h * \brief The header for the support of grammar-guided generation. */ #ifndef XGRAMMAR_GRAMMAR_DATA_STRUCTURE_H_ #define XGRAMMAR_GRAMMAR_DATA_STRUCTURE_H_ #include #include #include #include #include "fsm.h" #include "support/logging.h" #include "xgrammar/grammar.h" namespace xgrammar { /*! * \brief This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar. * The BNF definition here is standard BNF, and the characters are represented using regex-style * character classes (e.g. [a-z], [^a-z]). * * \details * ### Rules * The BNF grammar AST consists of a set of rules. Each rule contains a name and a definition, and * corresponds to a production in the grammar. The definition of a rule is a RuleExpr. Each rule * has a rule_id for reference. * * ### RuleExprs * RuleExpr is the definition of a rule or part of the definition of a rule. It can contain * elements, empty string, reference to other RuleExprs, or reference to other rules. Each RuleExpr * corresponds to an rule_expr_id for reference. * * For example, in the following rule: rule ::= ("a" "b") | "c" * ("a" "b"), "c", ("a" "b") | "c" are all RuleExprs. * * #### Types of RuleExprs * Every RuleExpr is represented by a type as well as a variable-length array containing its data. * RuleExpr has several types: * - Byte string: a string of bytes (0~255). Supports UTF-8 strings. * - Character class: a range of characters (each character is a unicode codepoint), e.g. [a-z], * [ac-z]. Can be negated: [^a-z], [^ac-z]. Now only ascii chars is allowed in [], but this * expression can accept/reject unicode chars. * - Character class star: a star quantifier of a character class. e.g. [a-z]*, [^a-z]*. * - EmptyStr: an empty string, i.e. "" * - Rule reference: a reference to another rule * - Sequence: a sequence of rule_exprs, e.g. ("a" "b"). These rule_exprs are concatenated together. * - Choices: a choice of rule_exprs, e.g. ("a" "b") | "c". Each rule_expr can be matched. * * #### Storage of RuleExprs * Each type of RuleExpr has a different data format. For the format of each type of RuleExpr, see * docs in Grammar::Impl::RuleExprType. * * We store all RuleExprs in csr_matrix style. That is, they are stored consecutively in one vector * (data vector) and the starting position of each RuleExpr is recorded in the indptr vector. * * \remark The character class star RuleExpr is for the special support for elements like [a-z]* * in the grammar. We add it to make the matching more efficient, as we can avoid recursion into * rules when matching a sequence of characters. It should be used like: * rule1 ::= ((element1 element2 rule2 ...) | ...) * rule2 ::= character_class_star_rule_expr(id_of_a_character_class_rule_expr) */ class Grammar::Impl { public: /*! \brief A rule with name. */ struct Rule { /*! \brief The name of the rule. */ std::string name; /*! \brief The RuleExpr id of the body of the rule. */ int32_t body_expr_id; /*! \brief The id of the associated lookahead assertion expr. For now it must be a id of a * sequence RuleExpr. -1 if not exists. */ int32_t lookahead_assertion_id = -1; }; /*! \brief Get the number of rules. */ size_t NumRules() const { return rules_.size(); } /*! \brief Get the rule with the given id. */ const Rule& GetRule(int32_t rule_id) const { XGRAMMAR_DCHECK(rule_id >= 0 && rule_id < static_cast(rules_.size())) << "rule_id " << rule_id << " is out of bound"; return rules_[rule_id]; } /*! \brief Get the root rule id of the grammar. */ int32_t GetRootRuleId() const { return root_rule_id_; } /*! \brief Get the root rule of the grammar. */ const Rule& GetRootRule() const { XGRAMMAR_DCHECK(root_rule_id_ >= 0 && root_rule_id_ < static_cast(rules_.size())) << "root_rule_id " << root_rule_id_ << " is out of bound"; return rules_[root_rule_id_]; } /*! \brief The type of the rule expr. */ enum class RuleExprType : int32_t { // data format: [byte0, byte1, ...] kByteString, // data format: [is_negative, lower0, upper0, lower1, upper1, ...] kCharacterClass, kCharacterClassStar, // data format: [] kEmptyStr, // data format: [rule_id] kRuleRef, // data format: [rule_expr_id0, rule_expr_id1, ...] kSequence, // data format: [rule_expr_id0, rule_expr_id1, ...] kChoices, // data format: [tag_expr0, rule_id0, tag_expr1, rule_id1, ...] // tag_expr should be a byte string, and rule_id should be a rule id kTagDispatch, }; /*! \brief The object representing a rule expr. */ struct RuleExpr { /*! \brief The type of the rule expr. */ RuleExprType type; /*! \brief The data of the RuleExpr. A variable-length array. */ const int32_t* data; /*! \brief The length of the data array. */ int32_t data_len; int32_t size() const { return data_len; } /*! \brief Get the i-th element of the data array. */ const int32_t& operator[](int i) const { XGRAMMAR_DCHECK(i >= 0 && i < static_cast(data_len)) << "Index " << i << " is out of bound"; return data[i]; } const int32_t* begin() const { return data; } const int32_t* end() const { return data + data_len; } }; /*! \brief Get the number of rule_exprs. */ size_t NumRuleExprs() const { return rule_expr_indptr_.size(); } /*! \brief Get the rule_expr with the given id. */ RuleExpr GetRuleExpr(int32_t rule_expr_id) const { XGRAMMAR_DCHECK( rule_expr_id >= 0 && rule_expr_id < static_cast(rule_expr_indptr_.size()) ) << "rule_expr_id " << rule_expr_id << " is out of bound"; int start_index = rule_expr_indptr_[rule_expr_id]; auto start_ptr = rule_expr_data_.data() + start_index; auto type = static_cast(start_ptr[0]); auto data_ptr = start_ptr + 2; auto data_len = start_ptr[1]; return {type, data_ptr, data_len}; } private: /*! \brief The rules of the grammar. rule_id corresponds the index of this vector. */ std::vector rules_; /*! \brief The data of all rule_exprs. */ std::vector rule_expr_data_; /*! \brief The start index of every rule_expr in rule_expr_data_. rule_expr_id is the index * to the elements in this vector. */ std::vector rule_expr_indptr_; /*! \brief The id of the root rule. */ int32_t root_rule_id_ = -1; public: /******************* Aux information for matching *******************/ /*! \brief The fsm for the root tag dispatch rule. If the grammar does not have a root tag * dispatch rule, it is not built. */ std::optional root_tag_dispatch_fsm = std::nullopt; /*! \brief The map from the end nodes of the root tag dispatch fsm to the rule ids. */ std::unordered_map tag_dispatch_end_node_to_rule_id; /*! \brief The ids of the rules that are allowed to be empty. */ std::vector allow_empty_rule_ids; friend class GrammarBuilder; friend class GrammarSerializer; friend class GrammarDeserializer; friend class GrammarCompiler; std::size_t MemorySize() const; friend std::size_t MemorySize(const Impl& impl); }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_DATA_STRUCTURE_H_ xgrammar-0.1.19/cpp/grammar_functor.cc000066400000000000000000001052731500705317600177130ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_functor.cc */ #include "grammar_functor.h" #include #include #include #include #include #include #include "grammar_data_structure.h" #include "support/encoding.h" namespace xgrammar { /*************************** Impl of grammar functors ***************************/ /*! * \brief Eliminates single-element sequence or choice or character class in the grammar. * \example `A ::= choices("a")` --> `A ::= "a"` (the body is a string) * \example `A ::= sequence("a")` --> `A ::= "a"` (the body is a string) * \example `A ::= [a-a]` --> `A ::= "a"` (the body is a string) */ class SingleElementExprEliminator : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: // Keep the sequence expr in lookahead assertion int32_t VisitLookaheadAssertion(int32_t lookahead_assertion_id) final { if (lookahead_assertion_id == -1) { return -1; } auto rule_expr = base_grammar_->GetRuleExpr(lookahead_assertion_id); XGRAMMAR_CHECK(rule_expr.type == RuleExprType::kSequence); std::vector sequence_ids; for (int32_t i : rule_expr) { sequence_ids.push_back(VisitExpr(i)); } return builder_.AddSequence(sequence_ids); } int32_t VisitSequence(const RuleExpr& rule_expr) final { std::vector sequence_ids; for (int32_t i : rule_expr) { sequence_ids.push_back(VisitExpr(i)); } if (sequence_ids.size() == 1) { return sequence_ids[0]; } return builder_.AddSequence(sequence_ids); } int32_t VisitChoices(const RuleExpr& rule_expr) final { std::vector choice_ids; for (int32_t i : rule_expr) { choice_ids.push_back(VisitExpr(i)); } if (choice_ids.size() == 1) { return choice_ids[0]; } return builder_.AddChoices(choice_ids); } int32_t VisitCharacterClass(const RuleExpr& rule_expr) final { if (rule_expr.data_len == 3 && rule_expr[0] == 0 && rule_expr[1] == rule_expr[2]) { std::string str = PrintAsUTF8(rule_expr[1]); std::vector bytes; bytes.reserve(str.size()); for (char c : str) { bytes.push_back(static_cast(c)); } return builder_.AddByteString(bytes); } return builder_.AddRuleExpr(rule_expr); } }; /*! * \brief Unwrap the rules containing nested expressions. After unwrapping, each rule will be in * the form: `rule_name ::= ("" | (element1_1 element1_2 ...) | (element2_1 element2_2 ...) | ...)`. * * I.e. a list of choices, each choice is a sequence of elements. Elements can be a character class * or a rule reference. And if the rule can be empty, the first choice will be an empty string. * * \example The rule `A ::= ((a) (((b)) (c)) "")` will be replaced by `A ::= ((a b c))`. One choice * containing a sequence of three elements. The empty string is removed. * \example The rule `A ::= (a | (b | (c | "")))` will be replaced by * `A ::= ("" | (a) | (b) | (c))`. The first choice is an empty string, and each of the other three * choices is a sequence containing a single element. * \example The rule `A ::= (a | (b (c | d)))` will be replaced by * `A ::= ((a) | (b B)), B ::= ((c) | (d))`. A new rule B is created to represent the nested * choices. */ class NestedRuleUnwrapper : public GrammarMutator { public: using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { Init(grammar); for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { builder_.AddEmptyRule(base_grammar_->GetRule(i).name); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto rule_expr = base_grammar_->GetRuleExpr(rule.body_expr_id); cur_rule_name_ = rule.name; auto new_body_expr_id = VisitRuleBody(rule_expr); builder_.UpdateRuleBody(i, new_body_expr_id); builder_.AddLookaheadAssertion(i, VisitLookaheadAssertion(rule.lookahead_assertion_id)); } return builder_.Get(base_grammar_->GetRootRule().name); } private: int32_t VisitLookaheadAssertion(int32_t lookahead_assertion_id) final { if (lookahead_assertion_id == -1) { return -1; } auto assertion_expr = base_grammar_->GetRuleExpr(lookahead_assertion_id); return builder_.AddSequence(VisitSequence_(assertion_expr)); } /*! \brief Visit a RuleExpr as a rule body. */ int32_t VisitRuleBody(const RuleExpr& rule_expr) { switch (rule_expr.type) { case RuleExprType::kSequence: return builder_.AddChoices({builder_.AddSequence(VisitSequence_(rule_expr))}); case RuleExprType::kChoices: return builder_.AddChoices(VisitChoices_(rule_expr)); case RuleExprType::kEmptyStr: return builder_.AddChoices({builder_.AddEmptyStr()}); case RuleExprType::kByteString: case RuleExprType::kCharacterClass: case RuleExprType::kCharacterClassStar: case RuleExprType::kRuleRef: return builder_.AddChoices({builder_.AddSequence({builder_.AddRuleExpr(rule_expr)})}); case RuleExprType::kTagDispatch: return VisitTagDispatch(rule_expr); default: XGRAMMAR_LOG(FATAL) << "Unexpected sequence type: " << static_cast(rule_expr.type); } } /*! * \brief Visit a RuleExpr containing choices. * \returns A list of new choice RuleExpr ids. */ std::vector VisitChoices_(const RuleExpr& rule_expr) { std::vector new_choice_ids; bool found_empty = false; for (auto i : rule_expr) { auto choice_expr = base_grammar_->GetRuleExpr(i); switch (choice_expr.type) { case RuleExprType::kSequence: VisitSequenceInChoices(choice_expr, &new_choice_ids, &found_empty); break; case RuleExprType::kChoices: VisitChoicesInChoices(choice_expr, &new_choice_ids, &found_empty); break; case RuleExprType::kEmptyStr: found_empty = true; break; case RuleExprType::kByteString: case RuleExprType::kCharacterClass: case RuleExprType::kCharacterClassStar: case RuleExprType::kRuleRef: VisitElementInChoices(choice_expr, &new_choice_ids); break; case RuleExprType::kTagDispatch: XGRAMMAR_LOG(FATAL) << "TagDispatch should not be in choices"; default: XGRAMMAR_LOG(FATAL) << "Unexpected choice type: " << static_cast(choice_expr.type); } } if (found_empty) { new_choice_ids.insert(new_choice_ids.begin(), builder_.AddEmptyStr()); } XGRAMMAR_ICHECK(new_choice_ids.size() >= 1); return new_choice_ids; } /*! \brief Visit a sequence RuleExpr that is one of a list of choices. */ void VisitSequenceInChoices( const RuleExpr& rule_expr, std::vector* new_choice_ids, bool* found_empty ) { auto sub_sequence_ids = VisitSequence_(rule_expr); if (sub_sequence_ids.size() == 0) { *found_empty = true; } else { new_choice_ids->push_back(builder_.AddSequence(sub_sequence_ids)); } } /*! \brief Visit a choice RuleExpr that is one of a list of choices. */ void VisitChoicesInChoices( const RuleExpr& rule_expr, std::vector* new_choice_ids, bool* found_empty ) { auto sub_choice_ids = VisitChoices_(rule_expr); bool contains_empty = builder_.GetRuleExpr(sub_choice_ids[0]).type == RuleExprType::kEmptyStr; if (contains_empty) { *found_empty = true; new_choice_ids->insert( new_choice_ids->end(), sub_choice_ids.begin() + 1, sub_choice_ids.end() ); } else { new_choice_ids->insert(new_choice_ids->end(), sub_choice_ids.begin(), sub_choice_ids.end()); } } /*! \brief Visit an atom element RuleExpr that is one of a list of choices. */ void VisitElementInChoices(const RuleExpr& rule_expr, std::vector* new_choice_ids) { auto sub_expr_id = builder_.AddRuleExpr(rule_expr); new_choice_ids->push_back(builder_.AddSequence({sub_expr_id})); } /*! * \brief Visit a RuleExpr containing a sequence. * \returns A list of new sequence RuleExpr ids. */ std::vector VisitSequence_(const RuleExpr& rule_expr) { std::vector new_sequence_ids; for (auto i : rule_expr) { auto element_expr = base_grammar_->GetRuleExpr(i); switch (element_expr.type) { case RuleExprType::kSequence: VisitSequenceInSequence(element_expr, &new_sequence_ids); break; case RuleExprType::kChoices: VisitChoiceInSequence(element_expr, &new_sequence_ids); break; case RuleExprType::kEmptyStr: break; case RuleExprType::kByteString: case RuleExprType::kCharacterClass: case RuleExprType::kCharacterClassStar: case RuleExprType::kRuleRef: VisitElementInSequence(element_expr, &new_sequence_ids); break; case RuleExprType::kTagDispatch: XGRAMMAR_LOG(FATAL) << "TagDispatch should not be in sequence"; default: XGRAMMAR_LOG(FATAL) << "Unexpected sequence type: " << static_cast(element_expr.type); } } return new_sequence_ids; } /*! \brief Visit a sequence RuleExpr that is one element in another sequence. */ void VisitSequenceInSequence(const RuleExpr& rule_expr, std::vector* new_sequence_ids) { auto sub_sequence_ids = VisitSequence_(rule_expr); new_sequence_ids->insert( new_sequence_ids->end(), sub_sequence_ids.begin(), sub_sequence_ids.end() ); } /*! \brief Visit a choice RuleExpr that is one element in a sequence. */ void VisitChoiceInSequence(const RuleExpr& rule_expr, std::vector* new_sequence_ids) { auto sub_choice_ids = VisitChoices_(rule_expr); if (sub_choice_ids.size() == 1) { auto choice_element_expr = builder_.GetRuleExpr(sub_choice_ids[0]); if (choice_element_expr.type != RuleExprType::kEmptyStr) { new_sequence_ids->insert( new_sequence_ids->end(), choice_element_expr.begin(), choice_element_expr.end() ); } } else { auto new_choice_id = builder_.AddChoices(sub_choice_ids); auto new_choice_rule_id = builder_.AddRuleWithHint(cur_rule_name_ + "_choice", new_choice_id); new_sequence_ids->push_back(builder_.AddRuleRef(new_choice_rule_id)); } } /*! \brief Visit an atom element RuleExpr that is in a sequence. */ void VisitElementInSequence(const RuleExpr& rule_expr, std::vector* new_sequence_ids) { new_sequence_ids->push_back(builder_.AddRuleExpr(rule_expr)); } }; class StructureNormalizerImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { return NestedRuleUnwrapper().Apply(SingleElementExprEliminator().Apply(grammar)); } }; class ByteStringFuserImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: /*! * \brief Visit a RuleExpr containing a sequence. * \returns A list of new sequence RuleExpr ids. */ int32_t VisitSequence(const RuleExpr& rule_expr) final { std::vector new_sequence_ids; std::vector cur_byte_string; for (auto i : rule_expr) { auto element_expr = base_grammar_->GetRuleExpr(i); if (element_expr.type == RuleExprType::kByteString) { cur_byte_string.insert(cur_byte_string.end(), element_expr.begin(), element_expr.end()); continue; } else { if (!cur_byte_string.empty()) { new_sequence_ids.push_back(builder_.AddByteString(cur_byte_string)); cur_byte_string.clear(); } new_sequence_ids.push_back(builder_.AddRuleExpr(element_expr)); } } if (!cur_byte_string.empty()) { new_sequence_ids.push_back(builder_.AddByteString(cur_byte_string)); } return builder_.AddSequence(new_sequence_ids); } }; class RuleInlinerImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; private: int32_t VisitChoices(const RuleExpr& rule_expr) final { std::vector new_choice_ids; for (int i : rule_expr) { auto choice_expr = base_grammar_->GetRuleExpr(i); if (choice_expr.type == RuleExprType::kEmptyStr) { new_choice_ids.push_back(VisitExpr(i)); continue; } XGRAMMAR_ICHECK(choice_expr.type == RuleExprType::kSequence); auto first_element = base_grammar_->GetRuleExpr(choice_expr[0]); if (first_element.type != RuleExprType::kRuleRef) { new_choice_ids.push_back(VisitExpr(choice_expr)); continue; } auto rule_ref_id = first_element[0]; if (can_rule_be_inlined_.count(rule_ref_id) == 0) { can_rule_be_inlined_[rule_ref_id] = CheckIfRuleCanBeInlined(rule_ref_id); } if (!can_rule_be_inlined_[rule_ref_id]) { new_choice_ids.push_back(VisitExpr(choice_expr)); continue; } // Do inlining std::vector other_elements; for (int i = 1; i < choice_expr.size(); ++i) { other_elements.push_back(VisitExpr(choice_expr[i])); } auto ref_rule = base_grammar_->GetRule(rule_ref_id); auto ref_rule_expr = base_grammar_->GetRuleExpr(ref_rule.body_expr_id); for (auto ref_choice_id : ref_rule_expr) { auto ref_choice_expr = base_grammar_->GetRuleExpr(ref_choice_id); XGRAMMAR_ICHECK(ref_choice_expr.type == RuleExprType::kSequence); std::vector choice_to_add; for (auto ref_element_id : ref_choice_expr) { choice_to_add.push_back(VisitExpr(ref_element_id)); } choice_to_add.insert(choice_to_add.end(), other_elements.begin(), other_elements.end()); new_choice_ids.push_back(builder_.AddSequence(choice_to_add)); } } return builder_.AddChoices(new_choice_ids); } /** * The rule should be: a sequence of choices, cannot be empty, cannot refer to other rules */ bool CheckIfRuleCanBeInlined(int32_t rule_id) { auto rule = base_grammar_->GetRule(rule_id); auto rule_expr = base_grammar_->GetRuleExpr(rule.body_expr_id); if (rule_expr.type != RuleExprType::kChoices) { return false; } if (rule_expr.size() == 0) { return false; } for (auto choice_id : rule_expr) { auto choice_expr = base_grammar_->GetRuleExpr(choice_id); if (choice_expr.type == RuleExprType::kEmptyStr) { return false; } XGRAMMAR_ICHECK(choice_expr.type == RuleExprType::kSequence); for (auto element_id : choice_expr) { auto element_expr = base_grammar_->GetRuleExpr(element_id); if (element_expr.type == RuleExprType::kRuleRef) { return false; } } } return true; } std::unordered_map can_rule_be_inlined_; }; /*! * \brief Analyze all referenced rules or the main rule. Return a list of all referenced rule ids. * This is useful for dead code elimination. */ class UsedRulesAnalyzer : public GrammarVisitor> { public: UsedRulesAnalyzer() = default; std::vector Apply(const Grammar& grammar) final { base_grammar_ = grammar; std::set visited; std::queue().swap(visit_queue_); visit_queue_.push(base_grammar_->GetRootRuleId()); while (!visit_queue_.empty()) { auto rule_id = visit_queue_.front(); visit_queue_.pop(); if (visited.count(rule_id)) { continue; } visited.insert(rule_id); auto rule = base_grammar_->GetRule(rule_id); VisitExpr(rule.body_expr_id); } return std::vector(visited.begin(), visited.end()); } void VisitTagDispatch(const RuleExpr& rule_expr) { for (int i = 0; i < rule_expr.size(); i += 2) { visit_queue_.push(rule_expr[i + 1]); } } void VisitRuleRef(const RuleExpr& rule_expr) { visit_queue_.push(rule_expr[0]); } private: std::queue visit_queue_; }; class DeadCodeEliminatorImpl : public GrammarMutator { public: using GrammarMutator::Apply; using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { Init(grammar); auto used_rules = UsedRulesAnalyzer().Apply(grammar); rule_id_map_.clear(); for (auto rule_id : used_rules) { rule_id_map_[rule_id] = builder_.AddEmptyRule(grammar->GetRule(rule_id).name); } for (auto rule_id : used_rules) { auto rule = grammar->GetRule(rule_id); auto new_body_expr_id = VisitExpr(rule.body_expr_id); builder_.UpdateRuleBody(rule_id_map_[rule_id], new_body_expr_id); builder_.AddLookaheadAssertion( rule_id_map_[rule_id], VisitLookaheadAssertion(rule.lookahead_assertion_id) ); } XGRAMMAR_CHECK(rule_id_map_.count(grammar->GetRootRuleId()) > 0); return builder_.Get(rule_id_map_[grammar->GetRootRuleId()]); } int32_t VisitTagDispatch(const RuleExpr& rule_expr) final { std::vector> tag_dispatch_list; for (int i = 0; i < rule_expr.size(); i += 2) { XGRAMMAR_DCHECK(rule_id_map_.count(rule_expr[i + 1]) > 0); auto new_rule_id = rule_id_map_[rule_expr[i + 1]]; tag_dispatch_list.push_back({VisitExpr(rule_expr[i]), new_rule_id}); } return builder_.AddTagDispatch(tag_dispatch_list); } int32_t VisitRuleRef(const RuleExpr& rule_expr) final { XGRAMMAR_DCHECK(rule_id_map_.count(rule_expr[0]) > 0); auto new_rule_id = rule_id_map_[rule_expr[0]]; return builder_.AddRuleRef(new_rule_id); } private: std::unordered_map rule_id_map_; }; class LookaheadAssertionAnalyzerImpl : public GrammarMutator { public: using GrammarMutator::GrammarMutator; Grammar Apply(const Grammar& grammar) final { InitWithCopy(grammar); auto root_rule = grammar->GetRootRule(); auto root_rule_expr = base_grammar_->GetRuleExpr(root_rule.body_expr_id); if (root_rule_expr.type == RuleExprType::kTagDispatch) { return grammar; } for (int i = 0; i < static_cast(grammar->NumRules()); ++i) { auto rule = grammar->GetRule(i); if (i == grammar->GetRootRuleId() || rule.lookahead_assertion_id != -1) { continue; } auto look_head_assertion_id = DetectLookaheadAssertion(i); if (look_head_assertion_id != -1) { builder_.AddLookaheadAssertion(i, look_head_assertion_id); } } return builder_.Get(grammar->GetRootRuleId()); } int32_t DetectLookaheadAssertion(int32_t rule_id) { std::vector found_sequence; // Element ids bool found = false; for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto rule_expr = base_grammar_->GetRuleExpr(rule.body_expr_id); if (rule_expr.type == RuleExprType::kTagDispatch) { for (int j = 1; j < rule_expr.size(); j += 2) { if (rule_expr[j] == rule_id) { return -1; } } continue; } XGRAMMAR_DCHECK(rule_expr.type == RuleExprType::kChoices); for (auto sequence_id : rule_expr) { auto sequence_expr = base_grammar_->GetRuleExpr(sequence_id); if (sequence_expr.type != RuleExprType::kSequence) { continue; } auto last_element = base_grammar_->GetRuleExpr(sequence_expr.end()[-1]); if (last_element.type == RuleExprType::kRuleRef && last_element[0] == rule_id && i != rule_id) { return -1; } for (int j = 0; j < sequence_expr.size() - 1; ++j) { auto element_expr = base_grammar_->GetRuleExpr(sequence_expr[j]); if (element_expr.type != RuleExprType::kRuleRef || element_expr[0] != rule_id) { continue; } if (found) { return -1; } found = true; for (int k = j + 1; k < sequence_expr.size(); ++k) { found_sequence.push_back(sequence_expr[k]); } } } } if (!found) { return -1; } return builder_.AddSequence(found_sequence); } }; /*! * \brief A class that normalizes a grammar by applying a series of transformations. * * The normalizer applies the following transformations in order: * 1. SingleElementExprEliminator - Eliminates single element expressions * 2. NestedRuleUnwrapper - Unwraps nested rules * 3. ByteStringFuser - Fuses consecutive byte strings */ class GrammarNormalizerImpl : public GrammarMutator { public: GrammarNormalizerImpl() = default; Grammar Apply(const Grammar& grammar) final { std::vector> normalizer_mutators = GetNormalizerList(); base_grammar_ = grammar; for (auto& mutator : normalizer_mutators) { base_grammar_ = mutator->Apply(base_grammar_); } return base_grammar_; } private: // Return the list of all normalizers in the class. The normalizers are applied one by one. std::vector> GetNormalizerList() { std::vector> normalizer_mutators; normalizer_mutators.emplace_back(std::make_unique()); normalizer_mutators.emplace_back(std::make_unique()); normalizer_mutators.emplace_back(std::make_unique()); normalizer_mutators.emplace_back(std::make_unique()); normalizer_mutators.emplace_back(std::make_unique()); return normalizer_mutators; } }; /*! * \brief Base class for grammar mutators that add subgrammars. * * Provides functionality to visit a subgrammar and add its rules to the builder * while maintaining proper rule references and names. */ class SubGrammarAdder : public GrammarMutator { public: SubGrammarAdder() = default; protected: /*! * \brief Visit a subgrammar and add the rules to the builder. * \param grammar The subgrammar to visit. * \return The new id of the root rule of this subgrammar. */ int32_t VisitSubGrammar(const Grammar& grammar) { base_grammar_ = grammar; new_rule_ids_names.reserve(grammar->NumRules()); new_rule_ids_names.clear(); for (int i = 0; i < static_cast(grammar->NumRules()); ++i) { auto new_name = builder_.GetNewRuleName(grammar->GetRule(i).name); auto new_id = builder_.AddEmptyRule(new_name); new_rule_ids_names.emplace_back(new_id, new_name); } for (int i = 0; i < static_cast(grammar->NumRules()); ++i) { auto rule = grammar->GetRule(i); cur_rule_name_ = new_rule_ids_names[i].second; auto new_body_expr_id = VisitExpr(rule.body_expr_id); builder_.UpdateRuleBody(new_rule_ids_names[i].first, new_body_expr_id); auto new_lookahead_assertion_id = VisitLookaheadAssertion(rule.lookahead_assertion_id); builder_.AddLookaheadAssertion(new_rule_ids_names[i].first, new_lookahead_assertion_id); } return new_rule_ids_names[grammar->GetRootRuleId()].first; } int32_t VisitRuleRef(const RuleExpr& rule_expr) final { return builder_.AddRuleRef(new_rule_ids_names[rule_expr[0]].first); } std::vector> new_rule_ids_names; }; /*! * \brief Implementation of grammar union operation. * * Creates a new grammar that accepts strings from any of the input grammars. * The resulting grammar has a new root rule that chooses between the root rules * of all input grammars. */ class GrammarUnionFunctorImpl : public SubGrammarAdder { public: GrammarUnionFunctorImpl() = default; Grammar Apply(const std::vector& grammars) { builder_ = GrammarBuilder(); auto root_rule_id = builder_.AddEmptyRule("root"); std::vector new_root_choices; new_root_choices.reserve(grammars.size()); for (const auto& grammar : grammars) { auto new_root_id_for_grammar = VisitSubGrammar(grammar); auto new_rule_ref = builder_.AddRuleRef(new_root_id_for_grammar); auto new_rule_ref_seq = builder_.AddSequence({new_rule_ref}); new_root_choices.push_back(new_rule_ref_seq); } builder_.UpdateRuleBody(root_rule_id, builder_.AddChoices(new_root_choices)); return builder_.Get(root_rule_id); } // Avoid hiding the original Apply(const Grammar&) Grammar Apply(const Grammar& grammar) final { XGRAMMAR_LOG(FATAL) << "Should not be called"; } }; /*! * \brief Implementation of grammar concatenation operation. * * Creates a new grammar that accepts strings that are concatenations of strings * from the input grammars in order. The resulting grammar has a new root rule * that concatenates the root rules of all input grammars. */ class GrammarConcatFunctorImpl : public SubGrammarAdder { public: GrammarConcatFunctorImpl() = default; Grammar Apply(const std::vector& grammars) { builder_ = GrammarBuilder(); auto root_rule_id = builder_.AddEmptyRule("root"); std::vector new_root_sequence; new_root_sequence.reserve(grammars.size()); for (const auto& grammar : grammars) { auto new_root_id_for_grammar = VisitSubGrammar(grammar); auto new_rule_ref = builder_.AddRuleRef(new_root_id_for_grammar); new_root_sequence.push_back(new_rule_ref); } auto new_root_seq = builder_.AddSequence(new_root_sequence); builder_.UpdateRuleBody(root_rule_id, builder_.AddChoices({new_root_seq})); return builder_.Get(root_rule_id); } // Avoid hiding the original Apply(const Grammar&) Grammar Apply(const Grammar& grammar) final { XGRAMMAR_LOG(FATAL) << "Should not be called"; } }; /*! * \brief Finds the rule reference graph of a grammar. * * The rule reference graph shows which rules reference which other rules. * The returned graph is inverted: it points from referee to referer. */ class RuleRefGraphFinder : public GrammarVisitor>> { public: RuleRefGraphFinder() = default; std::vector> Apply(const Grammar& grammar) { base_grammar_ = grammar; rule_visit_graph_ = std::vector>(base_grammar_->NumRules()); for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto rule_expr = base_grammar_->GetRuleExpr(rule.body_expr_id); cur_rule_id_ = i; VisitExpr(rule_expr); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { std::sort(rule_visit_graph_[i].begin(), rule_visit_graph_[i].end()); auto end_it = std::unique(rule_visit_graph_[i].begin(), rule_visit_graph_[i].end()); rule_visit_graph_[i].erase(end_it, rule_visit_graph_[i].end()); } return std::move(rule_visit_graph_); } private: void VisitRuleRef(const RuleExpr& rule_expr) { rule_visit_graph_[rule_expr[0]].push_back(cur_rule_id_); } void VisitTagDispatch(const RuleExpr& rule_expr) { for (int i = 1; i < rule_expr.size(); i += 2) { rule_visit_graph_[rule_expr[i]].push_back(cur_rule_id_); } } // Inversed reference graph: pointing from referee to referer std::vector> rule_visit_graph_; int32_t cur_rule_id_; }; /*! * \brief Analyzes which rules in a grammar can match the empty string. */ class AllowEmptyRuleAnalyzerImpl : public GrammarVisitor> { public: AllowEmptyRuleAnalyzerImpl() = default; std::vector Apply(const Grammar& grammar) final { base_grammar_ = grammar; // Step 1: Find rules that explicitly allow empty string std::unordered_set empty_rule_id_set; FindExplicitEmptyRules(&empty_rule_id_set); // Step 2: Find rules that indirectly allow empty string. Using the Bellman-Ford algorithm // on the rule reference graph. std::vector> rule_ref_graph = RuleRefGraphFinder().Apply(grammar); FindIndirectEmptyRules(&empty_rule_id_set, rule_ref_graph); auto result = std::vector(empty_rule_id_set.begin(), empty_rule_id_set.end()); std::sort(result.begin(), result.end()); return result; } void FindExplicitEmptyRules(std::unordered_set* empty_rule_id_set) { for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); auto rule_expr = base_grammar_->GetRuleExpr(rule.body_expr_id); if (rule_expr.type == RuleExprType::kTagDispatch) { empty_rule_id_set->insert(i); continue; } XGRAMMAR_DCHECK(rule_expr.type == RuleExprType::kChoices); if (base_grammar_->GetRuleExpr(rule_expr[0]).type == RuleExprType::kEmptyStr) { empty_rule_id_set->insert(i); continue; } for (auto seq_id : rule_expr) { auto seq_expr = base_grammar_->GetRuleExpr(seq_id); if (std::all_of(seq_expr.begin(), seq_expr.end(), [&](int32_t i) { return base_grammar_->GetRuleExpr(i).type == RuleExprType::kCharacterClassStar; })) { empty_rule_id_set->insert(i); break; } } } } bool SeqExprIsEpsilon( const RuleExpr& seq_expr, const std::unordered_set& empty_rule_id_set ) { if (seq_expr.type == RuleExprType::kEmptyStr) { return true; } XGRAMMAR_DCHECK(seq_expr.type == RuleExprType::kSequence); return std::all_of(seq_expr.begin(), seq_expr.end(), [&](int32_t i) { auto element_expr = base_grammar_->GetRuleExpr(i); return (element_expr.type == RuleExprType::kRuleRef && empty_rule_id_set.count(element_expr[0])) || element_expr.type == RuleExprType::kCharacterClassStar; }); } void FindIndirectEmptyRules( std::unordered_set* empty_rule_id_set, const std::vector>& rule_ref_graph ) { std::queue queue; for (auto i : *empty_rule_id_set) { queue.push(i); } while (!queue.empty()) { auto rule_id = queue.front(); queue.pop(); XGRAMMAR_DCHECK(rule_id >= 0 && rule_id < static_cast(rule_ref_graph.size())); for (auto referer_rule_id : rule_ref_graph[rule_id]) { if (empty_rule_id_set->count(referer_rule_id)) { continue; } auto rule = base_grammar_->GetRule(referer_rule_id); auto rule_expr = base_grammar_->GetRuleExpr(rule.body_expr_id); XGRAMMAR_DCHECK(rule_expr.type != RuleExprType::kTagDispatch) << "TagDispatch rules should already exist in empty_rule_id_set"; bool is_epsilon = std::any_of(rule_expr.begin(), rule_expr.end(), [&](int32_t i) { auto seq_expr = base_grammar_->GetRuleExpr(i); return SeqExprIsEpsilon(seq_expr, *empty_rule_id_set); }); if (is_epsilon) { empty_rule_id_set->insert(referer_rule_id); queue.push(referer_rule_id); } } } } }; class StructuralTagGrammarCreatorImpl : public SubGrammarAdder { public: Grammar Apply( const std::vector& triggers, const std::vector>>& tag_groups ) { XGRAMMAR_CHECK(triggers.size() == tag_groups.size()) << "Number of triggers must match number of tag groups"; builder_ = GrammarBuilder(); auto root_rule_id = builder_.AddEmptyRule("root"); // Create rules for each trigger group std::vector> trigger_rule_pairs; trigger_rule_pairs.reserve(triggers.size()); for (size_t i = 0; i < triggers.size(); i++) { // Skip empty trigger groups if (tag_groups[i].empty()) { continue; } auto rule_name = "trigger_rule_" + std::to_string(i); auto rule_id = builder_.AddEmptyRule(rule_name); // Convert trigger string to byte string expr auto trigger_expr_id = builder_.AddByteString(triggers[i]); // Create choices for each tag in this trigger group std::vector choices; choices.reserve(tag_groups[i].size()); for (const auto& [tag, schema_grammar] : tag_groups[i]) { // Create sequence: start_suffix + schema + end std::vector seq_elements; seq_elements.reserve(3); // Add begin suffix (everything after trigger) XGRAMMAR_DCHECK(tag.begin.size() >= triggers[i].size()) << "Tag begin must be at least as long as trigger"; if (tag.begin.size() > triggers[i].size()) { seq_elements.push_back(builder_.AddByteString(tag.begin.substr(triggers[i].size()))); } // Create and visit schema grammar for this tag auto schema_rule_id = VisitSubGrammar(schema_grammar); seq_elements.push_back(builder_.AddRuleRef(schema_rule_id)); // Add end string if (!tag.end.empty()) { seq_elements.push_back(builder_.AddByteString(tag.end)); } choices.push_back(builder_.AddSequence(seq_elements)); } builder_.UpdateRuleBody(rule_id, builder_.AddChoices(choices)); trigger_rule_pairs.emplace_back(trigger_expr_id, rule_id); } // Create root TagDispatch rule std::vector> tag_dispatch_data; tag_dispatch_data.reserve(trigger_rule_pairs.size()); for (const auto& [trigger_id, rule_id] : trigger_rule_pairs) { tag_dispatch_data.emplace_back(trigger_id, rule_id); } builder_.UpdateRuleBody(root_rule_id, builder_.AddTagDispatch(tag_dispatch_data)); return builder_.Get(root_rule_id); } // Avoid hiding the original Apply(const Grammar&) Grammar Apply(const Grammar& grammar) final { XGRAMMAR_LOG(FATAL) << "Should not be called"; } }; /*************************** Forward grammar functors to their impl ***************************/ Grammar GrammarNormalizer::Apply(const Grammar& grammar) { return GrammarNormalizerImpl().Apply(grammar); } Grammar GrammarUnionFunctor::Apply(const std::vector& grammars) { return GrammarUnionFunctorImpl().Apply(grammars); } Grammar GrammarConcatFunctor::Apply(const std::vector& grammars) { return GrammarConcatFunctorImpl().Apply(grammars); } std::vector AllowEmptyRuleAnalyzer::Apply(const Grammar& grammar) { return AllowEmptyRuleAnalyzerImpl().Apply(grammar); } Grammar StructuralTagGrammarCreator::Apply( const std::vector& triggers, const std::vector>>& tag_groups ) { return StructuralTagGrammarCreatorImpl().Apply(triggers, tag_groups); } Grammar RuleInliner::Apply(const Grammar& grammar) { return RuleInlinerImpl().Apply(grammar); } Grammar ByteStringFuser::Apply(const Grammar& grammar) { return ByteStringFuserImpl().Apply(grammar); } Grammar DeadCodeEliminator::Apply(const Grammar& grammar) { return DeadCodeEliminatorImpl().Apply(grammar); } Grammar StructureNormalizer::Apply(const Grammar& grammar) { return StructureNormalizerImpl().Apply(grammar); } Grammar LookaheadAssertionAnalyzer::Apply(const Grammar& grammar) { return LookaheadAssertionAnalyzerImpl().Apply(grammar); } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_functor.h000066400000000000000000000243231500705317600175510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_functor.h * \brief The header for the simplification of the BNF AST. */ #ifndef XGRAMMAR_GRAMMAR_FUNCTOR_H_ #define XGRAMMAR_GRAMMAR_FUNCTOR_H_ #include #include #include "grammar_builder.h" #include "grammar_data_structure.h" namespace xgrammar { /*! * \brief Base class for visitors and mutators of the BNF grammar. * \tparam T The type of the return value of visitor functions. Typical values: * - int32_t: the id of the new rule_expr * - void: no return value * \tparam ReturnType The type of the return value of the transform function Apply(). Typical values * are void (for visitor) and Grammar (for mutator). */ template class GrammarFunctor { public: /*! * \brief Constructor. * \param grammar The grammar to visit or mutate. */ explicit GrammarFunctor() {} /*! * \brief Apply the transformation to the grammar, or visit the grammar. * \return The transformed grammar, or the visiting result, or void. */ virtual ReturnType Apply(const Grammar& grammar) { // The initializer MUST be called at first when overriding the Apply() function. Init(grammar); if constexpr (std::is_same::value) { for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); cur_rule_name_ = rule.name; VisitExpr(rule.body_expr_id); VisitLookaheadAssertion(rule.lookahead_assertion_id); } return ReturnType(); } else if constexpr (std::is_same::value && std::is_same::value) { // First add empty rules to ensure the new rule ids the same as the old ones, then update // the rule bodies for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { builder_.AddEmptyRule(base_grammar_->GetRule(i).name); } for (int i = 0; i < static_cast(base_grammar_->NumRules()); ++i) { auto rule = base_grammar_->GetRule(i); cur_rule_name_ = rule.name; auto new_body_expr_id = VisitExpr(rule.body_expr_id); builder_.UpdateRuleBody(i, new_body_expr_id); // Handle lookahead assertion builder_.AddLookaheadAssertion(i, VisitLookaheadAssertion(rule.lookahead_assertion_id)); } return builder_.Get(base_grammar_->GetRootRule().name); } else { return ReturnType(); } } /*! \brief Virtual destructor. */ virtual ~GrammarFunctor() = default; protected: using Rule = Grammar::Impl::Rule; using RuleExpr = Grammar::Impl::RuleExpr; using RuleExprType = Grammar::Impl::RuleExprType; /*! \brief Initialize the functor. Should be called at the beginning of Apply(). */ virtual void Init(const Grammar& grammar) { base_grammar_ = grammar; builder_ = GrammarBuilder(); } virtual void InitWithCopy(const Grammar& grammar) { base_grammar_ = grammar; builder_ = GrammarBuilder(grammar); } /*! \brief Visit a lookahead assertion expr referred by id. */ virtual T VisitLookaheadAssertion(int32_t lookahead_assertion_id) { if (lookahead_assertion_id == -1) { if constexpr (std::is_same::value) { return -1; } else { return T(); } } return VisitExpr(lookahead_assertion_id); } /*! \brief Visit a RuleExpr by id. */ virtual T VisitExpr(int32_t old_rule_expr_id) { return VisitExpr(base_grammar_->GetRuleExpr(old_rule_expr_id)); } /*! \brief Visit a RuleExpr. Dispatch to the corresponding Visit function. */ virtual T VisitExpr(const RuleExpr& rule_expr) { switch (rule_expr.type) { case RuleExprType::kSequence: return VisitSequence(rule_expr); case RuleExprType::kChoices: return VisitChoices(rule_expr); case RuleExprType::kEmptyStr: return VisitEmptyStr(rule_expr); case RuleExprType::kByteString: return VisitByteString(rule_expr); case RuleExprType::kCharacterClass: return VisitCharacterClass(rule_expr); case RuleExprType::kCharacterClassStar: return VisitCharacterClassStar(rule_expr); case RuleExprType::kRuleRef: return VisitRuleRef(rule_expr); case RuleExprType::kTagDispatch: return VisitTagDispatch(rule_expr); default: XGRAMMAR_LOG(FATAL) << "Unexpected sequence type: " << static_cast(rule_expr.type); } } /*! \brief Visit a choices RuleExpr. */ virtual T VisitChoices(const RuleExpr& rule_expr) { if constexpr (std::is_same::value) { for (auto i : rule_expr) { VisitExpr(i); } } else if constexpr (std::is_same::value) { std::vector choice_ids; for (int32_t i : rule_expr) { choice_ids.push_back(VisitExpr(i)); } return builder_.AddChoices(choice_ids); } else { return T(); } } /*! \brief Visit a sequence RuleExpr. */ virtual T VisitSequence(const RuleExpr& rule_expr) { if constexpr (std::is_same::value) { for (auto i : rule_expr) { VisitExpr(i); } } else if constexpr (std::is_same::value) { std::vector sequence_ids; for (int32_t i : rule_expr) { sequence_ids.push_back(VisitExpr(i)); } return builder_.AddSequence(sequence_ids); } else { return T(); } } virtual T VisitTagDispatch(const RuleExpr& rule_expr) { if constexpr (std::is_same::value) { for (int i = 0; i < rule_expr.size(); i += 2) { VisitExpr(rule_expr[i]); } } else if constexpr (std::is_same::value) { std::vector> tag_dispatch_list; for (int i = 0; i < rule_expr.size(); i += 2) { tag_dispatch_list.push_back({VisitExpr(rule_expr[i]), rule_expr[i + 1]}); } return builder_.AddTagDispatch(tag_dispatch_list); } else { return T(); } } /*! \brief Visit an element RuleExpr, including empty string, character class, and rule ref. */ virtual T VisitElement(const RuleExpr& rule_expr) { if constexpr (std::is_same::value) { return; } else if constexpr (std::is_same::value) { return builder_.AddRuleExpr(rule_expr); } else { return T(); } } /*! \brief Visit an empty string RuleExpr. */ virtual T VisitEmptyStr(const RuleExpr& rule_expr) { return VisitElement(rule_expr); } /*! \brief Visit a character class RuleExpr. */ virtual T VisitByteString(const RuleExpr& rule_expr) { return VisitElement(rule_expr); } /*! \brief Visit a character class RuleExpr. */ virtual T VisitCharacterClass(const RuleExpr& rule_expr) { return VisitElement(rule_expr); } /*! \brief Visit a star quantifier RuleExpr. */ virtual T VisitCharacterClassStar(const RuleExpr& rule_expr) { return VisitElement(rule_expr); } /*! \brief Visit a rule reference RuleExpr. */ virtual T VisitRuleRef(const RuleExpr& rule_expr) { return VisitElement(rule_expr); } /*! \brief The grammar to visit or mutate. */ Grammar base_grammar_; /*! * \brief The builder to build the new grammar. It is empty when the mutator is constructed, and * can be used to build a new grammar in subclasses. */ GrammarBuilder builder_; /*! \brief The name of the current rule being visited. */ std::string cur_rule_name_; }; /*! * \brief Visitor of Grammar. * \tparam ReturnType The return type of the Apply() function. Denotes the collected information. */ template using GrammarVisitor = GrammarFunctor; /*! * \brief Mutator of Grammar. The Apply() function returns the updated grammar. */ using GrammarMutator = GrammarFunctor; /*************************** Grammar manipulation methods ***************************/ /****** All below methods are implemented as functor to hide the implementation ******/ /*! * \brief Normalize a Grammar: expand the nested rules, combine consequent sequences and strings, * etc. */ class GrammarNormalizer { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Find the union of multiple grammars as a new grammar. */ class GrammarUnionFunctor { public: static Grammar Apply(const std::vector& grammars); }; /*! * \brief Find the concatenation of multiple grammars as a new grammar. */ class GrammarConcatFunctor { public: static Grammar Apply(const std::vector& grammars); }; /*! * \brief Analyze the grammar to find the rules that are allowed to be empty. */ class AllowEmptyRuleAnalyzer { public: static std::vector Apply(const Grammar& grammar); }; /*! * \brief Create a grammar that recognizes structural tags based on their triggers. See * StructuralTagToGrammar() for more details. * * \param triggers The trigger strings that identify each tag group * \param tag_groups The tags and their schema grammars, grouped by trigger. tag_groups[i][j] is the * j-th tag that matches triggers[i], and its corresponding schema grammar. * \return A grammar that matches all the tagged patterns. */ class StructuralTagGrammarCreator { public: static Grammar Apply( const std::vector& triggers, const std::vector>>& tag_groups ); }; /*! * \brief Normalize the structure of the grammar. It will ensure each rule is a choices of * sequences of elements, or a tag dispatch. The expanded context will be a sequence of elements. */ class StructureNormalizer { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Fuse the byte string elements in the grammar. */ class ByteStringFuser { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Inline the rule references in the grammar. */ class RuleInliner { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Eliminate the not referenced rules in the grammar. */ class DeadCodeEliminator { public: static Grammar Apply(const Grammar& grammar); }; /*! * \brief Analyze and add lookahead assertions in the grammar. */ class LookaheadAssertionAnalyzer { public: static Grammar Apply(const Grammar& grammar); }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_FUNCTOR_H_ xgrammar-0.1.19/cpp/grammar_matcher.cc000066400000000000000000001012631500705317600176510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_matcher.cc * \brief This source file implement the matcher class, especially the logic related to LLM tokens, * like accepting tokens, leveraging the token mask cache to generate the mask, etc. matcher_base.cc * implements the basic matching algorithm from strings to grammar. */ #include #include "compiled_grammar_data_structure.h" #include "grammar_data_structure.h" #include "grammar_matcher_base.h" #include "persistent_stack.h" #include "support/dynamic_bitset.h" #include "support/encoding.h" #include "support/int_set.h" #include "support/logging.h" #include "testing.h" namespace xgrammar { /******************* Tool functions for token mask *******************/ int32_t GetBitmaskSize(int vocab_size) { return DynamicBitset::GetBufferSize(vocab_size); } DLDataType GetBitmaskDLType() { return DLDataType{kDLInt, 32, 1}; } int32_t* CheckAndGetBitmaskPtr(const DLTensor& token_bitmask, int vocab_size, int index) { XGRAMMAR_CHECK(token_bitmask.dtype.code == kDLInt && token_bitmask.dtype.bits == 32) << "The provied bitmask's dtype is not valid: should be int32"; int32_t buffer_size = GetBitmaskSize(vocab_size); if (token_bitmask.ndim == 1) { XGRAMMAR_CHECK(token_bitmask.shape[0] == buffer_size) << "The provided bitmask's shape is not valid: should be (" << buffer_size << ", )"; XGRAMMAR_CHECK(index == 0) << "The index should be 0 when the bitmask is 1D"; } else { XGRAMMAR_CHECK(token_bitmask.ndim == 2) << "The provided bitmask's shape is not valid: should be (batch_size, " << buffer_size << ")"; XGRAMMAR_CHECK(token_bitmask.shape[1] == buffer_size) << "The provided bitmask's shape is not valid: should be (batch_size, " << buffer_size << ")"; XGRAMMAR_CHECK(index >= 0 && index < token_bitmask.shape[0]) << "The provided index is out of bounds"; } XGRAMMAR_CHECK( token_bitmask.device.device_type == kDLCPU || token_bitmask.device.device_type == kDLCUDAHost || token_bitmask.device.device_type == kDLROCMHost ) << "The provided bitmask's device is not valid: should be CPU"; return reinterpret_cast(token_bitmask.data) + index * buffer_size; } void _DebugGetMaskedTokensFromBitmask( std::vector* rejected_tokens, const DLTensor& token_bitmask, int vocab_size, int index ) { int32_t* data_ptr = CheckAndGetBitmaskPtr(token_bitmask, vocab_size, index); DynamicBitset bitset(vocab_size, reinterpret_cast(data_ptr)); rejected_tokens->clear(); for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { rejected_tokens->push_back(i); } } std::pair _IsSingleTokenBitmask(const DLTensor& bitmask, int vocab_size, int index) { int32_t* data_ptr = CheckAndGetBitmaskPtr(bitmask, vocab_size, index); DynamicBitset bitset(vocab_size, reinterpret_cast(data_ptr)); if (bitset.Count() == 1) { return std::make_pair(true, bitset.FindFirstOne()); } else { return std::make_pair(false, -1); } } void ApplyTokenBitmaskInplaceCPU( DLTensor* logits, const DLTensor& bitmask, int vocab_size, std::optional> indices ) { // Check device and dim XGRAMMAR_CHECK( logits->device.device_type == kDLCPU || logits->device.device_type == kDLCUDAHost || logits->device.device_type == kDLROCMHost ) << "The provided logits's device is not valid: should be CPU"; XGRAMMAR_CHECK( bitmask.device.device_type == kDLCPU || bitmask.device.device_type == kDLCUDAHost || bitmask.device.device_type == kDLROCMHost ) << "The provided bitmask's device is not valid: should be CPU"; XGRAMMAR_CHECK(logits->ndim == 2 || logits->ndim == 1) << "The provided logits's shape is not valid: should be 2D or 1D"; XGRAMMAR_CHECK(bitmask.ndim == 2 || bitmask.ndim == 1) << "The provided bitmask's shape is not valid: should be 2D or 1D"; // Check type XGRAMMAR_CHECK( logits->dtype.code == kDLFloat && logits->dtype.bits == 32 && logits->dtype.lanes == 1 ) << "The provided logits's dtype is not valid: should be float32"; XGRAMMAR_CHECK( bitmask.dtype.code == kDLInt && bitmask.dtype.bits == 32 && bitmask.dtype.lanes == 1 ) << "The provided bitmask's dtype is not valid: should be int32"; // Check shape std::pair logits_shape = logits->ndim == 2 ? std::make_pair(static_cast(logits->shape[0]), static_cast(logits->shape[1])) : std::make_pair(1, static_cast(logits->shape[0])); std::pair bitmask_shape = bitmask.ndim == 2 ? std::make_pair(static_cast(bitmask.shape[0]), static_cast(bitmask.shape[1])) : std::make_pair(1, static_cast(bitmask.shape[0])); XGRAMMAR_CHECK( vocab_size <= bitmask_shape.second * DynamicBitset::BITS_PER_BLOCK && vocab_size <= logits_shape.second ); if (!indices.has_value()) { XGRAMMAR_CHECK(logits_shape.first == bitmask_shape.first) << "When indices is not provided, the logits's batch size should be equal to the " "bitmask's batch size, but got " << logits_shape.first << " vs " << bitmask_shape.first; } // Apply mask if (indices.has_value()) { for (auto idx : indices.value()) { uint32_t* data_ptr = reinterpret_cast(bitmask.data) + idx * bitmask_shape.second; DynamicBitset bitset(vocab_size, data_ptr); auto logits_ptr = reinterpret_cast(logits->data) + idx * logits_shape.second; for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { logits_ptr[i] = -std::numeric_limits::infinity(); } } } else { for (int idx = 0; idx < logits_shape.first; ++idx) { uint32_t* data_ptr = reinterpret_cast(bitmask.data) + idx * bitmask_shape.second; DynamicBitset bitset(vocab_size, data_ptr); auto logits_ptr = reinterpret_cast(logits->data) + idx * logits_shape.second; for (int i = bitset.FindFirstZero(); i != -1; i = bitset.FindNextZero(i)) { logits_ptr[i] = -std::numeric_limits::infinity(); } } } } /******************* Grammar Matcher with Adaptive Token Mask *******************/ /* * Note on the matching algorithm (this is the old description for the matching algorithm, please * refer to https://arxiv.org/pdf/2411.15100 for the latest description) * * Given a context-free grammar, we match the characters in a string one by one. * * We adopt a non-deterministic pushdown automata (NPDA) in matching. To be specific, we maintain * several stacks, each of which represents a possible path in the NPDA, and update the stacks * during matching. * * ## Stack Structure (see grammar_matcher_state.h) * The element of every stack is a StackElement object, referring a position in the grammar. If a * StackElement points to a RuleRef element (referring to another rule), the next element of the * stack will be a position in this rule. If a StackElement is a CharacterClass element, it will be * the last in the stack, meaning *the next* character to match. * * ## Matching Process (see grammar_matcher_base.h) * When accepting a new character and it is accepted by a stack, the last element of the stack will * be advanced to the next position in the grammar. If it gets to the end of the rule, several * elements at the end may be popped out, and the last element of the stack will be advanced. * * One stack may split since there may be multiple possible next positions. In this case, similar * stacks with different top elements will be added. When one stack cannot accept the new character, * it will be removed from the stacks. * * ## Storage of Stacks (see grammar_matcher_state.h) * Note these stacks form a tree structure as when splitting, the new stacks share the same prefix. * We store all StackElements as a tree, where every path from tree root to a node represents a * stack. To represent stack tops, we attach additional pointers pointing the stack top nodes. * Also, We maintain a history of the stack top pointers, so we can rollback to the previous state. * * All tree nodes are maintained by a buffer, and utilize reference counting to recycle. If a node * is neither pointed by a stack top pointer, not pointed by some child nodes, it will be freed. * * ## Example * ### Grammar * root ::= [a] R * R ::= [b] S [c] | [b] [c] T * S ::= "" | [c] [d] * T ::= [e] * * ### The previous step * Previous accepted string: ab * Previous stack tree: * A------ * | \ \ * B D< E< * | * C< * * A: (rule root, choice 0, element 1) * B: (rule R, choice 0, element 1) * C: (rule S, choice 1, element 0) * D: (rule R, choice 0, element 2) * E: (rule R, choice 1, element 1) * < means the stack top pointers in the previous step. * The stacks in the previous step is: (A, B, C), (A, D), (A, E) * * ### The current step * Current accepted string: abc * Current stack tree: * A----------------- G<< * | \ \ \ * B--- D< E< H * | \ | * C< F<< I<< * * F: (rule S, choice 1, element 1) * G: (rule root, choice 0, element 2) (means the matching process has finished, and will be deleted * when the next char comes) * H: (rule R, choice 1, element 2) * I: (rule T, choice 0, element 0) * << means the stack top pointers in the current step. * The stacks in the current step is: (A, B, F), (A, H, I), (G,) * * ## Preprocess (see grammar_matcher_preproc.h) * We will store all information about tokens that needed in matching in a CompiledGrammar * object. Tokens are sorted by codepoint, allowing us to reuse the repeated prefixes between * different tokens. * * For a given position in a rule, if we only consider this rule and its sub-rules during matching, * without considering its parent rules (in actual matching, we also need to consider its parent * rules), we can already determine that some tokens are acceptable while others are definitely * rejected. Therefore, for a position in a rule, we can divide the token set into three categories: * - accepted_indices: If a token is accepted by this rule * - rejected_indices: If a token is rejected by this rule * - uncertain_indices: Whether it can be accepted depends on the information from the parent * level during actual matching. To be specific, If this token has a prefix that has not been * rejected and has reached the end of this rule, then it is possible for it to be further accepted * by the parent rule. * * During actual matching, we will directly accept or reject the tokens in accepted_indices and * rejected_indices, and only consider the tokens in uncertain_indices. That speeds up the matching * process. */ /* \brief The concrete implementation of GrammarMatcherNode. */ class GrammarMatcher::Impl : public GrammarMatcherBase { public: Impl( const CompiledGrammar& compiled_grammar, std::optional> override_stop_tokens = std::nullopt, bool terminate_without_stop_token = false, int max_rollback_tokens = 0 ) : GrammarMatcherBase(compiled_grammar->grammar), compiled_grammar_(compiled_grammar), tokenizer_info_(compiled_grammar->tokenizer_info), stop_token_ids_(override_stop_tokens.value_or(tokenizer_info_.GetStopTokenIds())), terminate_without_stop_token_(terminate_without_stop_token), max_rollback_tokens_(max_rollback_tokens), tmp_accepted_bitset_(tokenizer_info_.GetVocabSize()) { XGRAMMAR_CHECK(!override_stop_tokens.has_value() || !override_stop_tokens->empty()) << "The override_stop_tokens should not be empty"; } bool AcceptToken(int32_t token_id, bool debug_print = false); bool FillNextTokenBitmask(DLTensor* next_token_bitmask, int index, bool debug_print = false); std::string FindJumpForwardString(); void Rollback(int num_tokens); bool IsTerminated() const; void Reset() { stack_tops_history_.Reset(); token_length_history.clear(); PushInitialState(kInvalidStackElement, true); } int GetMaxRollbackTokens() const { return max_rollback_tokens_; } const std::vector& GetStopTokenIds() const { return stop_token_ids_; } bool _DebugAcceptString(const std::string& input_str, bool debug_print = false); private: using StoreType = AdaptiveTokenMask::StoreType; /*! * \brief If is_uncertain_saved is true, find the next token in uncertain_indices. Otherwise, * find the next token that is set to true in uncertain_tokens_bitset. * \param iterator_uncertain The helper iterator to iterate over uncertain_indices or * uncertain_tokens_bitset. * \returns The index of the next token, or -1 if no more token. */ int GetNextUncertainToken( bool is_uncertain_saved, int* iterator_uncertain, const std::vector& uncertain_indices, const std::vector& uncertain_tokens_bitset ); /*! \brief Set the acceptable next token in next_token_bitmask. */ void SetTokenBitmask( int32_t* bitmask_data_ptr, const DynamicBitset& accepted_bitset, const std::vector& rejected_indices, bool can_reach_end, bool allow_special_token = false ); /*! * \brief Accept the stop token and terminates the matcher. * \returns Whether the stop token can be accepted. */ bool AcceptStopToken(); bool IsStopTokenAccepted() const; /*! \brief Check if the token bitmask is all-true. */ bool IsTokenBitmaskAllTrue(int32_t* bitmask_data_ptr); std::string PrintBitmask(int32_t* bitmask_data_ptr, const TokenizerInfo& tokenizer_info); CompiledGrammar compiled_grammar_; TokenizerInfo tokenizer_info_; std::vector stop_token_ids_; bool terminate_without_stop_token_; int max_rollback_tokens_; std::deque token_length_history; // Temporary data for FillNextTokenBitmask. They are stored here to avoid repeated allocation. DynamicBitset tmp_accepted_bitset_; std::vector tmp_rejected_indices_; std::vector tmp_rejected_indices_delta_; }; bool GrammarMatcher::Impl::AcceptStopToken() { if (terminate_without_stop_token_) { return false; } if (!CanReachEnd()) { return false; } stack_tops_history_.PushHistory({}); // Terminate the matcher by setting the stack to empty token_length_history.push_back(1); // When rolling back a stop token, we need to rollback 1 state return true; } bool GrammarMatcher::Impl::IsTerminated() const { if (terminate_without_stop_token_) { return CanReachEnd(); } return IsStopTokenAccepted(); } bool GrammarMatcher::Impl::IsStopTokenAccepted() const { return stack_tops_history_.GetLatest().empty(); } // TODO(yixin): Polish verbose logging bool GrammarMatcher::Impl::AcceptToken(int32_t token_id, bool debug_print) { if (IsStopTokenAccepted()) { if (debug_print) { XGRAMMAR_LOG(INFO) << "The matcher has terminated after accepting the stop token, but is " "trying to accept new token with id " << token_id; } return false; } XGRAMMAR_CHECK(token_id >= 0 && token_id < tokenizer_info_.GetVocabSize()) << "Invalid token id " << token_id << " for GrammarMatcher"; if (debug_print) { XGRAMMAR_LOG(INFO) << "Accepting token id " << token_id << ", string: \"" << PrintAsEscapedUTF8(tokenizer_info_.GetDecodedVocab()[token_id]) << "\", state state:\n" << PrintStackState(); } // Handle the stop token if (std::find(stop_token_ids_.begin(), stop_token_ids_.end(), token_id) != stop_token_ids_.end()) { bool accepted = AcceptStopToken(); if (debug_print) { XGRAMMAR_LOG(INFO) << "The token is an end token. Is accepted: " << accepted; } return accepted; } const auto& special_token_ids = tokenizer_info_.GetSpecialTokenIds(); if (std::find(special_token_ids.begin(), special_token_ids.end(), token_id) != special_token_ids.end()) { XGRAMMAR_LOG(FATAL) << "Token id " << token_id << ": " << tokenizer_info_.GetDecodedVocab()[token_id] << " is regarded as a special token, and cannot be accepted by the " "GrammarMatcher"; } const auto& token = tokenizer_info_.GetDecodedVocab()[token_id]; int pos = 0; for (auto char_value : token) { if (!AcceptChar(char_value, debug_print)) { if (debug_print) { XGRAMMAR_LOG(INFO) << "The token is rejected at position " << pos << ", character " << PrintAsEscapedUTF8(char_value); } RollbackChars(pos); return false; } ++pos; } token_length_history.push_back(token.size()); if (static_cast(token_length_history.size()) > max_rollback_tokens_) { DiscardEarliestChars(token_length_history.front()); token_length_history.pop_front(); } if (debug_print) { XGRAMMAR_LOG(INFO) << "The token is accepted. State after accepting:\n" << PrintStackState(); } return true; } bool GrammarMatcher::Impl::_DebugAcceptString(const std::string& input_str, bool debug_print) { if (IsStopTokenAccepted()) { if (debug_print) { XGRAMMAR_LOG(INFO) << "The matcher has terminated after accepting the stop token, but is " "trying to accept new string " << PrintAsEscapedUTF8(input_str); } return false; } int accepted_cnt = 0; for (auto char_value : input_str) { if (!AcceptChar(char_value, debug_print)) { if (debug_print) { XGRAMMAR_LOG(INFO) << "Matching failed after accepting " << accepted_cnt << " characters"; } RollbackChars(accepted_cnt); return false; } ++accepted_cnt; } token_length_history.push_back(input_str.size()); if (static_cast(token_length_history.size()) > max_rollback_tokens_) { DiscardEarliestChars(token_length_history.front()); token_length_history.pop_front(); } if (debug_print) { XGRAMMAR_LOG(INFO) << "String \"" << PrintAsEscapedUTF8(input_str) << "\" is accepted. State after accepting:\n" << PrintStackState(); } return true; } std::string GrammarMatcher::Impl::PrintBitmask( int32_t* bitmask_data_ptr, const TokenizerInfo& tokenizer_info ) { constexpr int kMaxPrintTokens = 100; std::vector accepted_ids; std::vector rejected_ids; auto bitset = DynamicBitset(tokenizer_info.GetVocabSize(), reinterpret_cast(bitmask_data_ptr)); for (int i = 0; i < tokenizer_info.GetVocabSize(); ++i) { if (bitset[i]) { accepted_ids.push_back(i); } else { rejected_ids.push_back(i); } } std::stringstream ss; ss << "TokenBitmask(num_tokens=" << tokenizer_info.GetVocabSize() << ", accepted_num=" << accepted_ids.size() << ", rejected_num=" << rejected_ids.size() << ",\naccepted_ids=" << PrintTokenByIds(accepted_ids, tokenizer_info, kMaxPrintTokens) << ",\nrejected_ids=" << PrintTokenByIds(rejected_ids, tokenizer_info, kMaxPrintTokens) << ")"; return ss.str(); } bool GrammarMatcher::Impl::IsTokenBitmaskAllTrue(int32_t* bitmask_data_ptr) { DynamicBitset next_token_bitset( tokenizer_info_.GetVocabSize(), reinterpret_cast(bitmask_data_ptr) ); return next_token_bitset.All(); } bool GrammarMatcher::Impl::FillNextTokenBitmask( DLTensor* next_token_bitmask, int index, bool debug_print ) { XGRAMMAR_CHECK(!IsStopTokenAccepted()) << "GrammarMatcher has terminated after accepting the stop token, but is trying to " "find the next token mask"; int32_t* bitmask_data_ptr = CheckAndGetBitmaskPtr(*next_token_bitmask, tokenizer_info_.GetVocabSize(), index); const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); const auto& adaptive_token_mask_cache = compiled_grammar_->adaptive_token_mask_cache; const auto& latest_stack_tops = stack_tops_history_.GetLatest(); // We check all the stacks one by one, and find the accepted token set or the rejected token set // for each stack. We will try to find the small one of the two sets. // The final accepted token set is the union of the accepted token sets of all stacks. // The final rejected token set is the intersection of the rejected token sets of all stacks. // Note these indices store the indices in sorted_decoded_vocab, instead of the token ids. tmp_accepted_bitset_.Reset(); // {-1} means the universal set, i.e. all tokens initially tmp_rejected_indices_.assign({-1}); // If there is a stack top that is a tag dispatch, we allow special tokens to be accepted // because in function calling cases, only the part within the tag is constrained bool have_tag_dispatch = false; if (debug_print) { XGRAMMAR_LOG(INFO) << "FillNextTokenBitmask: index=" << index << ", num of stacks=" << latest_stack_tops.size(); } int stack_top_cnt = -1; for (auto top : latest_stack_tops) { ++stack_top_cnt; auto cur_stack_element = persistent_stack_[top]; auto cur_sequence = grammar_->GetRuleExpr(cur_stack_element.sequence_id); if (cur_sequence.type != RuleExprType::kTagDispatch && cur_stack_element.parent_id == StackElement::kNoParent && cur_stack_element.element_id == cur_sequence.size()) { continue; } if (cur_sequence.type == RuleExprType::kTagDispatch) { have_tag_dispatch = true; } auto adaptive_token_mask_it = adaptive_token_mask_cache.find(cur_stack_element); XGRAMMAR_CHECK(adaptive_token_mask_it != adaptive_token_mask_cache.end()) << "The adaptive token mask is not found for stack element: " << persistent_stack_.PrintStackElement(cur_stack_element); const auto& adaptive_token_mask = adaptive_token_mask_it->second; if (debug_print) { XGRAMMAR_LOG(INFO) << "FillNextTokenBitmask: Stack #" << stack_top_cnt << ", num_uncertain_tokens=" << adaptive_token_mask.uncertain_indices.size() << ": " << persistent_stack_.PrintStackByTopId(top) << "\n"; } // For each stack, we will check every uncertain token and put them into the accepted or // rejected list. // Step 2. Update the accepted tokens in accepted_indices_delta, or the rejected tokens in // rejected_indices_delta. // If the accepted tokens are saved, it means it is likely to be smaller than the rejected // tokens, so we will just find the accepted tokens, and vice versa. tmp_rejected_indices_delta_.clear(); // Examine only the current one stack stack_tops_history_.PushHistory({persistent_stack_.NewNode(cur_stack_element)}); const std::string* prev_token = nullptr; int prev_matched_size = 0; for (auto cur_token_idx : adaptive_token_mask.uncertain_indices) { const auto& cur_token = sorted_decoded_vocab[cur_token_idx].second; bool accepted = true; // Step 2.1. Find the longest common prefix with the accepted part of the previous token. // We can reuse the previous matched size to avoid unnecessary matching. if (prev_token) { int lcp_len = std::mismatch( cur_token.begin(), cur_token.end(), prev_token->begin(), prev_token->end() ) .first - cur_token.begin(); if (lcp_len > prev_matched_size) { accepted = false; } else if (lcp_len < prev_matched_size) { RollbackChars(prev_matched_size - lcp_len); } prev_matched_size = std::min(prev_matched_size, lcp_len); } // Step 2.2. Find if the current token is accepted or rejected. if (accepted) { for (int j = prev_matched_size; j < static_cast(cur_token.size()); ++j) { if (!AcceptChar(cur_token[j], false)) { accepted = false; break; } prev_matched_size = j + 1; } } // Step 2.3. Push the result to the delta list. if (adaptive_token_mask.store_type == StoreType::kAcceptedBitset || adaptive_token_mask.store_type == StoreType::kAccepted) { if (accepted) { tmp_accepted_bitset_.Set(sorted_decoded_vocab[cur_token_idx].first, true); } } else { if (!accepted) { tmp_rejected_indices_delta_.push_back(cur_token_idx); } } prev_token = &cur_token; } RollbackChars(prev_matched_size + 1); // Step 3. Update the accepted_indices or rejected_indices if (adaptive_token_mask.store_type == StoreType::kAcceptedBitset) { tmp_accepted_bitset_ |= adaptive_token_mask.accepted_bitset; } else if (adaptive_token_mask.store_type == StoreType::kAccepted) { for (auto idx : adaptive_token_mask.accepted_indices) { tmp_accepted_bitset_.Set(sorted_decoded_vocab[idx].first, true); } } else { // rejected_indices = Intersect( // rejected_indices, // adaptive_token_mask.rejected_indices + rejected_indices_delta) IntsetUnion(&tmp_rejected_indices_delta_, adaptive_token_mask.rejected_indices); IntsetIntersection(&tmp_rejected_indices_, tmp_rejected_indices_delta_); } } // Finally update the rejected_ids bitset bool can_reach_end = CanReachEnd(); SetTokenBitmask( bitmask_data_ptr, tmp_accepted_bitset_, tmp_rejected_indices_, can_reach_end, have_tag_dispatch ); if (debug_print) { XGRAMMAR_LOG(INFO) << "Filled bitmask: " << PrintBitmask(bitmask_data_ptr, tokenizer_info_); } return !IsTokenBitmaskAllTrue(bitmask_data_ptr); } std::string GrammarMatcher::Impl::FindJumpForwardString() { XGRAMMAR_CHECK(!IsStopTokenAccepted()) << "GrammarMatcher has terminated after accepting the stop token, but is trying to " "get the jump forward string"; std::string result; int num_accepted_chars = 0; bool can_find_next_char = true; while (can_find_next_char) { const auto& stack_tops = stack_tops_history_.GetLatest(); // 1. Check that for every stack top, the next possible char is unique and the same // -1 means not found yet; 0~255 means the next char int next_char = -1; for (auto stack_top : stack_tops) { auto stack_element = persistent_stack_[stack_top]; auto cur_sequence = grammar_->GetRuleExpr(stack_element.sequence_id); // We cannot deduce the next char for tag dispatch if (cur_sequence.type == RuleExprType::kTagDispatch) { can_find_next_char = false; continue; } // The state comes to the end of the grammar if (stack_element.parent_id == StackElement::kNoParent && stack_element.element_id == cur_sequence.size()) { can_find_next_char = false; break; } auto cur_element = grammar_->GetRuleExpr(cur_sequence[stack_element.element_id]); if (cur_element.type == RuleExprType::kByteString) { XGRAMMAR_DCHECK(stack_element.element_in_string < cur_element.size()); if (next_char == -1) { next_char = cur_element[stack_element.element_in_string]; } else if (next_char != cur_element[stack_element.element_in_string]) { can_find_next_char = false; break; } } else { XGRAMMAR_DCHECK( cur_element.type == RuleExprType::kCharacterClass || cur_element.type == RuleExprType::kCharacterClassStar ); if (stack_element.left_utf8_bytes > 0 || cur_element.size() != 3 || cur_element[0] != 0 || cur_element[1] != cur_element[2]) { can_find_next_char = false; break; } else if (next_char == -1) { next_char = cur_element[1]; } else if (next_char != cur_element[1]) { can_find_next_char = false; break; } } } if (next_char == -1) { can_find_next_char = false; } // 2. If found, accept the char and iterate to the next position if (can_find_next_char) { result += static_cast(next_char); tmp_new_stack_tops_.clear(); for (auto stack_top : stack_tops) { auto cur_stack_element = persistent_stack_[stack_top]; auto new_stack_element = AdvanceStackElementWithChar(cur_stack_element, next_char); if (new_stack_element == cur_stack_element) { ExpandEquivalentStackElements(new_stack_element, &tmp_new_stack_tops_, stack_top); } else { ExpandEquivalentStackElements(new_stack_element, &tmp_new_stack_tops_); } } stack_tops_history_.PushHistory(tmp_new_stack_tops_); ++num_accepted_chars; } } // Rollback all chars accepted RollbackChars(num_accepted_chars); return result; } void GrammarMatcher::Impl::Rollback(int num_tokens) { XGRAMMAR_CHECK(num_tokens <= static_cast(token_length_history.size())) << "Intended to rollback " << num_tokens << " tokens, but only the last " << token_length_history.size() << " steps of history are saved"; while (num_tokens > 0) { int steps = token_length_history.back(); RollbackChars(steps); token_length_history.pop_back(); --num_tokens; } } void GrammarMatcher::Impl::SetTokenBitmask( int32_t* bitmask_data_ptr, const DynamicBitset& accepted_bitset, const std::vector& rejected_indices, bool can_reach_end, bool allow_special_token ) { // next_token_bitmask = set(all accepted tokens) = // 1. all_tokens - (rejected_ids / accepted_ids) // (when rejected_ids != {-1}, i.e. rejected_ids is not the universal set) // 2. accepted_ids // (otherwise, when rejected_ids is the universal set) DynamicBitset next_token_bitset( tokenizer_info_.GetVocabSize(), reinterpret_cast(bitmask_data_ptr) ); const auto& sorted_decoded_vocab = tokenizer_info_.GetSortedDecodedVocab(); if (rejected_indices.size() == 1 && rejected_indices[0] == -1) { // If rejected_indices is the universal set, the final accepted token set is just // accepted_indices next_token_bitset = accepted_bitset; if (allow_special_token) { for (int id : tokenizer_info_.GetSpecialTokenIds()) { next_token_bitset.Set(id, true); } } if (can_reach_end) { // add end tokens for (int id : stop_token_ids_) { next_token_bitset.Set(id, true); } } } else { // Otherwise, the final rejected token set is (rejected_indices \ accepted_indices) next_token_bitset.Set(); for (auto i : rejected_indices) { auto id = sorted_decoded_vocab[i].first; if (!accepted_bitset[id]) { next_token_bitset.Set(id, false); } } if (!allow_special_token) { for (int id : tokenizer_info_.GetSpecialTokenIds()) { next_token_bitset.Set(id, false); } } if (!can_reach_end) { for (int id : stop_token_ids_) { next_token_bitset.Set(id, false); } } } } int GrammarMatcher::Impl::GetNextUncertainToken( bool is_uncertain_saved, int* iterator_uncertain, const std::vector& uncertain_indices, const std::vector& uncertain_tokens_bitset ) { if (is_uncertain_saved) { ++*iterator_uncertain; if (*iterator_uncertain == static_cast(uncertain_indices.size())) { return -1; } return uncertain_indices[*iterator_uncertain]; } else { ++*iterator_uncertain; while (*iterator_uncertain < static_cast(uncertain_tokens_bitset.size()) && !uncertain_tokens_bitset[*iterator_uncertain]) { ++*iterator_uncertain; } if (*iterator_uncertain == static_cast(uncertain_tokens_bitset.size())) { return -1; } return *iterator_uncertain; } } GrammarMatcher::GrammarMatcher( const CompiledGrammar& compiled_grammar, std::optional> override_stop_tokens, bool terminate_without_stop_token, int max_rollback_tokens ) : pimpl_(std::make_shared( compiled_grammar, override_stop_tokens, terminate_without_stop_token, max_rollback_tokens )) {} bool GrammarMatcher::AcceptToken(int32_t token_id, bool debug_print) { return pimpl_->AcceptToken(token_id, debug_print); } bool GrammarMatcher::FillNextTokenBitmask( DLTensor* next_token_bitmask, int index, bool debug_print ) { return pimpl_->FillNextTokenBitmask(next_token_bitmask, index, debug_print); } std::string GrammarMatcher::FindJumpForwardString() { return pimpl_->FindJumpForwardString(); } void GrammarMatcher::Rollback(int num_tokens) { pimpl_->Rollback(num_tokens); } bool GrammarMatcher::IsTerminated() const { return pimpl_->IsTerminated(); } void GrammarMatcher::Reset() { pimpl_->Reset(); } int GrammarMatcher::GetMaxRollbackTokens() const { return pimpl_->GetMaxRollbackTokens(); } const std::vector& GrammarMatcher::GetStopTokenIds() const { return pimpl_->GetStopTokenIds(); } bool GrammarMatcher::_DebugAcceptString(const std::string& input_str, bool debug_print) { return pimpl_->_DebugAcceptString(input_str, debug_print); } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_matcher_base.cc000066400000000000000000000350371500705317600206500ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_matcher_base.cc * \brief This source file implements the basic matching algorithm from strings to grammar. * matcher.cc will handle the logic related to LLM tokens, like accepting tokens, leveraging the * token mask cache to generate the mask, etc. */ #include "grammar_matcher_base.h" #include #include #include "grammar_data_structure.h" #include "persistent_stack.h" #include "support/encoding.h" #include "support/utils.h" namespace xgrammar { constexpr int32_t kUnexpandedRuleStartSequenceId = 128000; constexpr int32_t kDispatchedTagDispatchElementId = -1; /*! \brief Check the codepoint is contained in the character class. */ bool GrammarMatcherBase::CheckIfAccepted(const StackElement& stack_element, uint8_t char_value) const { auto current_sequence = grammar_->GetRuleExpr(stack_element.sequence_id); if (current_sequence.type == Grammar::Impl::RuleExprType::kTagDispatch) { XGRAMMAR_DCHECK(stack_element.element_id != -1); return true; } if (stack_element.parent_id == StackElement::kNoParent && current_sequence.size() == stack_element.element_id) { // This StackElement means previous elements has matched the complete rule. // But we are still need to accept a new character, so this stack will become invalid. return false; } auto current_element = grammar_->GetRuleExpr(current_sequence[stack_element.element_id]); if (current_element.type == RuleExprType::kCharacterClass || current_element.type == RuleExprType::kCharacterClassStar) { if (stack_element.left_utf8_bytes > 0) { return (char_value & 0xC0) == 0x80; } auto [accepted, num_bytes, codepoint] = HandleUTF8FirstByte(char_value); if (!accepted) { return false; } bool is_negative = static_cast(current_element[0]); if (num_bytes > 1) { return is_negative; } for (int i = 1; i < current_element.size(); i += 2) { if (static_cast(current_element[i]) <= char_value && char_value <= static_cast(current_element[i + 1])) { return !is_negative; } } return is_negative; } else if (current_element.type == RuleExprType::kByteString) { return static_cast(current_element[stack_element.element_in_string]) == char_value; } else { XGRAMMAR_LOG(FATAL) << "Unexpected RuleExprType in CheckIfAccepted: " << static_cast(current_element.type); } } StackElement GrammarMatcherBase::MoveToNextPosition(const StackElement& stack_element) { StackElement new_stack_element = stack_element; new_stack_element.element_id += 1; new_stack_element.element_in_string = 0; new_stack_element.left_utf8_bytes = 0; XGRAMMAR_DCHECK( new_stack_element.element_id <= grammar_->GetRuleExpr(stack_element.sequence_id).size() ); return new_stack_element; } StackElement GrammarMatcherBase::AdvanceStackElementWithChar( const StackElement& stack_element, uint8_t char_value ) { auto current_sequence = grammar_->GetRuleExpr(stack_element.sequence_id); if (current_sequence.type == Grammar::Impl::RuleExprType::kTagDispatch) { auto root_tag_dispatch_fsm = grammar_->root_tag_dispatch_fsm; if (!root_tag_dispatch_fsm) { XGRAMMAR_LOG(FATAL) << "The grammar does not have a root tag dispatch rule; it is not built."; XGRAMMAR_UNREACHABLE(); } auto start_node = root_tag_dispatch_fsm->StartNode(); auto next_node = root_tag_dispatch_fsm->Transition(stack_element.element_id, char_value); auto new_stack_element = stack_element; if (next_node == CompactFSMWithStartEnd::NO_TRANSITION) { // Case 1. The new char cannot continue to be accepted by the tag dispatch fsm. // We try to accept the new char from the start node. If accepted, we go to the target node. // If it still cannot be accepted, we stay at the start node. auto new_next_node = root_tag_dispatch_fsm->Transition(start_node, char_value); new_stack_element.element_id = new_next_node == CompactFSMWithStartEnd::NO_TRANSITION ? start_node : new_next_node; } else if (!root_tag_dispatch_fsm->IsEndNode(next_node)) { // Case 2. The new char can continue to be accepted by the tag dispatch fsm. // We need to update the element id to the next node. new_stack_element.element_id = next_node; } else { // Case 3. The new char can continue to be accepted by the tag dispatch fsm. // We need to dispatch the tag dispatch fsm to the end node. // We need to create a new stack element to represent the dispatched tag dispatch. new_stack_element.element_id = kDispatchedTagDispatchElementId; auto new_stack_element_id = persistent_stack_.NewNode(new_stack_element); XGRAMMAR_DCHECK(grammar_->tag_dispatch_end_node_to_rule_id.count(next_node)) << "The end node of the tag dispatch fsm does not correspond to any rule id"; auto refered_rule_id = grammar_->tag_dispatch_end_node_to_rule_id.at(next_node); new_stack_element = StackElement(refered_rule_id, kUnexpandedRuleStartSequenceId, 0, new_stack_element_id); } return new_stack_element; } auto current_element = grammar_->GetRuleExpr(current_sequence[stack_element.element_id]); StackElement new_stack_element = stack_element; switch (current_element.type) { case RuleExprType::kCharacterClass: { if (stack_element.left_utf8_bytes > 1) { new_stack_element.left_utf8_bytes -= 1; return new_stack_element; } else if (stack_element.left_utf8_bytes == 1) { return MoveToNextPosition(stack_element); } // If no left utf8 bytes, check the first byte to find the left bytes needed. XGRAMMAR_DCHECK(stack_element.left_utf8_bytes == 0); auto [accepted, num_bytes, codepoint] = HandleUTF8FirstByte(char_value); XGRAMMAR_DCHECK(accepted); if (num_bytes > 1) { new_stack_element.left_utf8_bytes = num_bytes - 1; return new_stack_element; } return MoveToNextPosition(stack_element); } case RuleExprType::kCharacterClassStar: { if (stack_element.left_utf8_bytes >= 1) { new_stack_element.left_utf8_bytes -= 1; } else { XGRAMMAR_DCHECK(stack_element.left_utf8_bytes == 0); auto [accepted, num_bytes, codepoint] = HandleUTF8FirstByte(char_value); XGRAMMAR_DCHECK(accepted); new_stack_element.left_utf8_bytes = num_bytes - 1; } return new_stack_element; } case RuleExprType::kByteString: { if (stack_element.element_in_string + 1 < current_element.size()) { new_stack_element.element_in_string += 1; return new_stack_element; } return MoveToNextPosition(stack_element); } default: XGRAMMAR_LOG(FATAL) << "Unexpected RuleExprType in AdvanceStackElementWithChar: " << static_cast(current_element.type); } } void GrammarMatcherBase::ExpandEquivalentStackElements( const StackElement& cur_stack_element, std::vector* new_stack_tops, int32_t cur_stack_element_id, bool consider_parent ) { auto f_add_current_stack_element = [&]() { if (cur_stack_element_id != -1) { return cur_stack_element_id; } else { return persistent_stack_.NewNode(cur_stack_element); } }; // Step 1. Handle unexpanded rules. if (cur_stack_element.sequence_id == kUnexpandedRuleStartSequenceId) { auto cur_rule_id = cur_stack_element.rule_id; auto cur_rule_body_id = grammar_->GetRule(cur_rule_id).body_expr_id; auto cur_rule_body = grammar_->GetRuleExpr(cur_rule_body_id); if (cur_rule_body.type == RuleExprType::kTagDispatch) { auto new_stack_element = StackElement( cur_rule_id, cur_rule_body_id, grammar_->root_tag_dispatch_fsm->StartNode(), cur_stack_element.parent_id ); new_stack_tops->push_back(persistent_stack_.NewNode(new_stack_element)); return; } else { XGRAMMAR_DCHECK(cur_rule_body.type == RuleExprType::kChoices); for (auto sequence_id : cur_rule_body) { auto ref_rule_sequence = grammar_->GetRuleExpr(sequence_id); if (ref_rule_sequence.type == RuleExprType::kEmptyStr && cur_stack_element.parent_id != StackElement::kNoParent) { // If the empty string is in a root rule, it indicates the end of the grammar and we // just add it as a stack top to indicate the matching ends. continue; } auto new_stack_element = StackElement(cur_rule_id, sequence_id, 0, cur_stack_element.parent_id); ExpandEquivalentStackElements(new_stack_element, new_stack_tops, -1, false); } return; } } auto cur_sequence = grammar_->GetRuleExpr(cur_stack_element.sequence_id); // If the current sequence is a tag dispatch, it do not have any other equivalent stack elements. if (cur_sequence.type == RuleExprType::kTagDispatch) { new_stack_tops->push_back(f_add_current_stack_element()); return; } // Step 2. The stack element points to the end of a rule. if (cur_sequence.size() == cur_stack_element.element_id) { if (cur_stack_element.parent_id == StackElement::kNoParent) { // Case 2.1. The stack element points to the end of the grammar (meaning the matching // succeeded). Insert it and add as a stack top. new_stack_tops->push_back(f_add_current_stack_element()); } else if (consider_parent) { // Case 2.2. When consider_parent is true, we should recurse to the parent rule. auto new_stack_element = persistent_stack_[cur_stack_element.parent_id]; auto parent_sequence = grammar_->GetRuleExpr(new_stack_element.sequence_id); if (parent_sequence.type == RuleExprType::kTagDispatch) { new_stack_element.element_id = grammar_->root_tag_dispatch_fsm->StartNode(); } else { new_stack_element.element_id += 1; } XGRAMMAR_DCHECK(new_stack_element.element_in_string == 0); XGRAMMAR_DCHECK(new_stack_element.left_utf8_bytes == 0); ExpandEquivalentStackElements(new_stack_element, new_stack_tops, -1, consider_parent); } // Case 2.3. When consider_parent is false, we do nothing and return. return; } auto current_element = grammar_->GetRuleExpr(cur_sequence[cur_stack_element.element_id]); auto stack_element_id = f_add_current_stack_element(); // Step 3. Iterate into sub rules if (current_element.type == RuleExprType::kRuleRef) { ExpandEquivalentStackElements( StackElement(current_element[0], kUnexpandedRuleStartSequenceId, 0, stack_element_id), new_stack_tops, -1, false ); } else { XGRAMMAR_DCHECK( current_element.type == RuleExprType::kCharacterClass || current_element.type == RuleExprType::kByteString || current_element.type == RuleExprType::kCharacterClassStar ); new_stack_tops->push_back(stack_element_id); } // Step 4. Check the next element in the same rule auto exist_in_vector = [](const std::vector& vec, int32_t value) { return std::find(vec.begin(), vec.end(), value) != vec.end(); }; if ((current_element.type == RuleExprType::kCharacterClassStar && cur_stack_element.left_utf8_bytes == 0) || (current_element.type == RuleExprType::kRuleRef && exist_in_vector(grammar_->allow_empty_rule_ids, current_element[0]))) { auto next_stack_element = MoveToNextPosition(cur_stack_element); ExpandEquivalentStackElements(next_stack_element, new_stack_tops, -1, consider_parent); } } bool GrammarMatcherBase::AcceptChar(uint8_t char_value, bool debug_print) { if (debug_print) { XGRAMMAR_LOG(INFO) << "Trying to accept char: " << static_cast(char_value) << " \"" << PrintAsEscapedUTF8(char_value) << "\""; } const auto& prev_stack_tops = stack_tops_history_.GetLatest(); tmp_new_stack_tops_.clear(); for (auto prev_top : prev_stack_tops) { auto cur_stack_element = persistent_stack_[prev_top]; auto accepted = CheckIfAccepted(cur_stack_element, char_value); if (!accepted) { continue; } auto new_stack_element = AdvanceStackElementWithChar(cur_stack_element, char_value); if (new_stack_element == cur_stack_element) { ExpandEquivalentStackElements(new_stack_element, &tmp_new_stack_tops_, prev_top); } else { ExpandEquivalentStackElements(new_stack_element, &tmp_new_stack_tops_); } } if (tmp_new_stack_tops_.empty()) { if (debug_print) { XGRAMMAR_LOG(INFO) << "Character " << static_cast(char_value) << " \"" << PrintAsEscapedUTF8(char_value) << "\" Rejected"; } return false; } stack_tops_history_.PushHistory(tmp_new_stack_tops_); if (debug_print) { XGRAMMAR_LOG(INFO) << "Character: " << static_cast(char_value) << " \"" << PrintAsEscapedUTF8(char_value) << "\" Accepted"; XGRAMMAR_LOG(INFO) << "New stack after acceptance: " << PrintStackState(); } constexpr bool DEBUG_CHECK_WELL_FORMED = false; if (DEBUG_CHECK_WELL_FORMED) { stack_tops_history_.CheckWellFormed(); } return true; } bool GrammarMatcherBase::CanReachEnd() const { const auto& last_stack_tops = stack_tops_history_.GetLatest(); return std::any_of(last_stack_tops.begin(), last_stack_tops.end(), [&](int32_t id) { return persistent_stack_.IsEndOfGrammar(persistent_stack_[id]); }); } void GrammarMatcherBase::RollbackChars(int rollback_cnt) { stack_tops_history_.Rollback(rollback_cnt); } void GrammarMatcherBase::DiscardEarliestChars(int discard_cnt) { stack_tops_history_.DiscardEarliest(discard_cnt); } std::string GrammarMatcherBase::PrintStackState(int steps_before_latest) const { return stack_tops_history_.PrintHistory(steps_before_latest); } void GrammarMatcherBase::PushInitialState( const StackElement& init_stack_element, bool expand_init_stack_element ) { if (init_stack_element == kInvalidStackElement) { // Initialize the stack with the root rule. auto init_stack_element = StackElement( grammar_->GetRootRuleId(), kUnexpandedRuleStartSequenceId, 0, StackElement::kNoParent ); tmp_new_stack_tops_.clear(); ExpandEquivalentStackElements(init_stack_element, &tmp_new_stack_tops_); stack_tops_history_.PushHistory(tmp_new_stack_tops_); } else { if (expand_init_stack_element) { tmp_new_stack_tops_.clear(); ExpandEquivalentStackElements(init_stack_element, &tmp_new_stack_tops_); stack_tops_history_.PushHistory(tmp_new_stack_tops_); } else { stack_tops_history_.PushHistory({persistent_stack_.NewNode(init_stack_element)}); } } } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_matcher_base.h000066400000000000000000000115651500705317600205120ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_matcher_base.h * \brief The base class of GrammarMatcher. It implements a character-based matching automata. */ #ifndef XGRAMMAR_GRAMMAR_MATCHER_BASE_H_ #define XGRAMMAR_GRAMMAR_MATCHER_BASE_H_ #include #include #include #include "grammar_data_structure.h" #include "persistent_stack.h" namespace xgrammar { /*! * \brief The base class of GrammarMatcher. It implements a character-based matching * automata, and supports accepting a character, rolling back by character, etc. */ class GrammarMatcherBase { protected: using RuleExpr = Grammar::Impl::RuleExpr; using RuleExprType = Grammar::Impl::RuleExprType; public: /*! * \brief Construct a GrammarMatcherBase with the given grammar and initial stack element. * \param grammar The grammar to match. * \param init_stack_element The initial stack element. If not specified, the root rule will be * used. * \param expand_init_stack_element Whether to expand the initial stack element to all possible * locations. See ExpandEquivalentStackElements. */ GrammarMatcherBase( const Grammar& grammar, StackElement init_stack_element = kInvalidStackElement, bool expand_init_stack_element = true ) : grammar_(grammar), persistent_stack_(grammar), stack_tops_history_(&persistent_stack_) { PushInitialState(init_stack_element, expand_init_stack_element); } /*! \brief Accept one character. */ bool AcceptChar(uint8_t char_value, bool debug_print = false); /*! \brief Check if the end of the root rule is reached. If so, the stop token can be accepted. */ bool CanReachEnd() const; /*! \brief Rollback the matcher to a previous state by the number of characters. */ void RollbackChars(int rollback_cnt); /*! \brief Discard the earliest history by the number of characters. */ void DiscardEarliestChars(int discard_cnt); /*! \brief Print the stack state. */ std::string PrintStackState(int steps_before_latest = 0) const; protected: /*! * \brief Push an initial stack state according to the given stack element. * \param init_stack_element The initial stack element. If kInvalidStackElement, init the stack * with the root rule. * \param expand_init_stack_element Whether to expand the initial stack element to all equivalent * locations. See ExpandEquivalentStackElements. Only meaningful when init_stack_element is not * kInvalidStackElement. */ void PushInitialState(const StackElement& init_stack_element, bool expand_init_stack_element); // Check if the character is accepted by the current stack element. bool CheckIfAccepted(const StackElement& stack_element, uint8_t char_value) const; /*! * \brief Move to the next position in the current rule, and return the updated stack element. */ StackElement MoveToNextPosition(const StackElement& stack_element); /*! * \brief Return the updated stack element after accepting the character. */ StackElement AdvanceStackElementWithChar(const StackElement& stack_element, uint8_t char_value); /*! * \brief Expand the given stack element to all possible positions approachable in the grammar. * The expanded positions must refers to an element (CharacterClass or CharacterClassStar or * ByteString) in a rule. Push all new positions into new_stack_tops. * \example * A ::= "a" B [a-z]* "c" * B ::= "b" | "" * * Input position: (rule=A, position=B) * Approachable positions: (rule=B, position="b"), (rule=A, position=[a-z]*), * (rule=A, position="c"), since B and [a-z]* can be empty. * \param cur_stack_element The current stack element. * \param new_stack_tops The vector to store the new stack tops. * \param cur_stack_element_id The id in the persistent stack of the current stack element. If the * current stack element does not exist in the persistent stack, pass -1. This is used to avoid * inserting the same stack element again. * \param consider_parent Whether to consider the parent position if the current position is * at the end of the rule. Only used in its self recursion. */ void ExpandEquivalentStackElements( const StackElement& cur_stack_element, std::vector* new_stack_tops, int32_t cur_stack_element_id = -1, bool consider_parent = true ); // The matched grammar. Grammar grammar_; // The tree storing all states PersistentStack persistent_stack_; // The tracked history of stack tops (each stack top refers to a node in the tree). // We store the stack tops in different steps in the history to support rollback. StackTopsHistory stack_tops_history_; // Temporary data for AcceptChar, PushInitialState, etc to store new stacks. // They are stored here to avoid repeated allocation. std::vector tmp_new_stack_tops_; }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_MATCHER_BASE_H_ xgrammar-0.1.19/cpp/grammar_parser.cc000066400000000000000000000506741500705317600175330ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_parser.cc */ #include "grammar_parser.h" #include #include "grammar_builder.h" #include "grammar_data_structure.h" #include "support/encoding.h" #include "support/logging.h" namespace xgrammar { class EBNFParser { public: /*! \brief The logic of parsing the grammar string. */ Grammar Parse(const std::string& ebnf_string, const std::string& root_rule_name); private: using Rule = Grammar::Impl::Rule; using RuleExprType = Grammar::Impl::RuleExprType; // Parsing different parts of the grammar std::string ParseIdentifier(bool accept_empty = false); int32_t ParseCharacterClass(); int32_t ParseString(); int32_t ParseRuleRef(); int32_t ParseElement(); int64_t ParseInteger(); std::pair ParseRepetitionRange(); int32_t ParseElementWithQuantifier(); int32_t ParseLookaheadAssertion(); int32_t ParseSequence(); int32_t ParseChoices(); std::pair ParseTagDispatchElement(); int32_t ParseTagDispatchOrChoices(); Rule ParseRule(); // Helper functions // Helper for ParseElementWithQuantifier int32_t HandleStarQuantifier(int32_t rule_expr_id); int32_t HandlePlusQuantifier(int32_t rule_expr_id); int32_t HandleQuestionQuantifier(int32_t rule_expr_id); int32_t HandleRepetitionRange(int32_t rule_expr_id, int64_t lower, int64_t upper); // When parsing, we first find the names of all rules, and build the mapping from name to rule id. void BuildRuleNameToId(); // Consumes several spaces (newline, space, tab, comment, etc.) void ConsumeSpace(bool allow_newline = true); // Check the validity of a name static bool IsNameChar(TCodepoint c, bool first_char = false); // Reset the parser to the beginning of the string. void ResetStringIterator(const char* cur); // Consume a specified number of characters, and maintain the line and column number. void Consume(int cnt = 1) { for (int i = 0; i < cnt; ++i) { // \n \r \r\n if (Peek() == '\n' || (Peek() == '\r' && Peek(1) != '\n')) { ++cur_line_; cur_column_ = 1; } else { ++cur_column_; } ++cur_; } } // Peek the next character. char Peek(int delta = 0) const { return *(cur_ + delta); } // Report a parsing error with the given message and the line and column number. [[noreturn]] void ReportParseError(const std::string& msg) { XGRAMMAR_LOG(FATAL) << "EBNF parse error at line " + std::to_string(cur_line_) + ", column " + std::to_string(cur_column_) + ": " + msg; } // The grammar builder GrammarBuilder builder_; // A pointer to the current parse position in the string const char* cur_ = nullptr; // The current line and column number int cur_line_ = 1; int cur_column_ = 1; // The current rule name. Help to generate a name for a new rule. std::string cur_rule_name_; // Whether the current element is in parentheses. // A sequence expression cannot contain newline, unless it is in parentheses. bool in_parentheses_ = false; // The name of the root rule std::string root_rule_name_; inline static constexpr int64_t MAX_INTEGER_IN_GRAMMAR = 1e10; }; void EBNFParser::ConsumeSpace(bool allow_newline) { while (Peek() && (Peek() == ' ' || Peek() == '\t' || Peek() == '#' || (allow_newline && (Peek() == '\n' || Peek() == '\r')))) { Consume(); if (Peek(-1) == '#') { auto start = cur_column_ - 1; while (Peek() && Peek() != '\n' && Peek() != '\r') { Consume(); } if (!Peek() || start != 1 /* Reserve \n for inline comment */) { return; } Consume(); if (Peek(-1) == '\r' && Peek() == '\n') { Consume(); } } } } bool EBNFParser::IsNameChar(TCodepoint c, bool first_char) { return c == '_' || c == '-' || c == '.' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (!first_char && c >= '0' && c <= '9'); } // name should be a char string (not a utf8 string) std::string EBNFParser::ParseIdentifier(bool accept_empty) { auto start = cur_; bool first_char = true; while (Peek() && IsNameChar(Peek(), first_char)) { Consume(); first_char = false; } if (start == cur_ && !accept_empty) { ReportParseError("Expect rule name"); } return std::string(start, cur_); } // Character class: // 1. Examples: [a-z] [ab] [a-zA-Z0-9] [^a-z] [测] [\u0123] // 2. The "-" character is treated as a literal character if it is the last or the first (after // the "^"", if present) character within the brackets. E.g. [a-] and [-a] means "a" or "-" // 3. "-" and "]" should be escaped when used as a literal character: // [\-] means - // [\]] means ] // Character class should not contain newlines. int32_t EBNFParser::ParseCharacterClass() { static constexpr TCodepoint kUnknownUpperBound = -4; static const std::unordered_map CUSTOM_ESCAPE_MAP = {{'-', '-'}, {']', ']'}}; std::vector elements; bool is_negated = false; if (Peek() == '^') { is_negated = true; Consume(); } bool past_is_hyphen = false; bool past_is_single_char = false; while (Peek() && Peek() != ']') { if (Peek() == '\r' || Peek() == '\n') { ReportParseError("Character class should not contain newline"); } else if (Peek() == '-' && Peek(1) != ']' && !past_is_hyphen && past_is_single_char) { Consume(); past_is_hyphen = true; past_is_single_char = false; continue; } auto [codepoint, len] = ParseNextUTF8OrEscaped(cur_, CUSTOM_ESCAPE_MAP); if (codepoint == CharHandlingError::kInvalidUTF8) { ReportParseError("Invalid UTF8 sequence"); } if (codepoint == CharHandlingError::kInvalidEscape) { ReportParseError("Invalid escape sequence"); } Consume(len); if (past_is_hyphen) { XGRAMMAR_ICHECK(!elements.empty()); if (elements.back().lower > codepoint) { ReportParseError("Invalid character class: lower bound is larger than upper bound"); } elements.back().upper = codepoint; past_is_hyphen = false; XGRAMMAR_ICHECK(past_is_single_char == false); } else { elements.push_back({codepoint, kUnknownUpperBound}); past_is_single_char = true; } } for (auto& element : elements) { if (element.upper == kUnknownUpperBound) { element.upper = element.lower; } } return builder_.AddCharacterClass(elements, is_negated); } // parse a c style string with utf8 support int32_t EBNFParser::ParseString() { if (Peek() != '\"') { ReportParseError("Expect \" in string literal"); } Consume(); std::vector codepoints; while (Peek() && Peek() != '\"' && Peek() != '\n' && Peek() != '\r') { auto [codepoint, len] = ParseNextUTF8OrEscaped(cur_); if (codepoint == CharHandlingError::kInvalidUTF8) { ReportParseError("Invalid utf8 sequence"); } if (codepoint == CharHandlingError::kInvalidEscape) { ReportParseError("Invalid escape sequence"); } Consume(len); codepoints.push_back(codepoint); } if (Peek() != '\"') { ReportParseError("Expect \" in string literal"); } Consume(); if (codepoints.empty()) { return builder_.AddEmptyStr(); } // convert codepoints to string std::string str; for (auto codepoint : codepoints) { str += PrintAsUTF8(codepoint); } // convert str to int32_t vector std::vector bytes; for (auto c : str) { bytes.push_back(static_cast(static_cast(c))); } return builder_.AddByteString(bytes); } int32_t EBNFParser::ParseRuleRef() { std::string name = ParseIdentifier(); auto rule_id = builder_.GetRuleId(name); if (rule_id == -1) { ReportParseError("Rule \"" + name + "\" is not defined"); } return builder_.AddRuleRef(rule_id); } int32_t EBNFParser::ParseElement() { switch (Peek()) { case '(': { Consume(); ConsumeSpace(); if (Peek() == ')') { // Special case: ( ) Consume(); return builder_.AddEmptyStr(); } auto prev_in_parentheses = in_parentheses_; in_parentheses_ = true; auto rule_expr_id = ParseChoices(); ConsumeSpace(); if (Peek() != ')') { ReportParseError("Expect )"); } Consume(); in_parentheses_ = prev_in_parentheses; return rule_expr_id; } case '[': { Consume(); auto rule_expr_id = ParseCharacterClass(); if (Peek() != ']') { ReportParseError("Expect ]"); } Consume(); return rule_expr_id; } case '\"': { return ParseString(); } default: { if (IsNameChar(Peek(), true)) { return ParseRuleRef(); } ReportParseError( "Expect element, but got character: " + std::string(1, static_cast(Peek())) ); } } } int32_t EBNFParser::HandleStarQuantifier(int32_t rule_expr_id) { Grammar::Impl::RuleExpr rule_expr = builder_.GetRuleExpr(rule_expr_id); if (rule_expr.type == GrammarBuilder::RuleExprType::kCharacterClass) { // We have special handling for character class star, e.g. [a-z]* rule_expr.type = GrammarBuilder::RuleExprType::kCharacterClassStar; // Copy rule expr because the grammar may change during insertion, and rule_expr is in the // grammar, so it may become invalid std::vector rule_expr_data(rule_expr.begin(), rule_expr.end()); return builder_.AddRuleExpr({rule_expr.type, rule_expr_data.data(), rule_expr.data_len}); } else { // For other star quantifiers, we transform it into a rule: // a* --> rule ::= a rule | "" auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_rule_id = builder_.AddEmptyRule(new_rule_name); auto ref_to_new_rule = builder_.AddRuleRef(new_rule_id); auto new_rule_expr_id = builder_.AddChoices( {builder_.AddEmptyStr(), builder_.AddSequence({rule_expr_id, ref_to_new_rule})} ); builder_.UpdateRuleBody(new_rule_id, new_rule_expr_id); // Return the reference to the new rule return builder_.AddRuleRef(new_rule_id); } } int32_t EBNFParser::HandlePlusQuantifier(int32_t rule_expr_id) { // a+ --> rule ::= a rule | a auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_rule_id = builder_.AddEmptyRule(new_rule_name); auto ref_to_new_rule = builder_.AddRuleRef(new_rule_id); auto new_rule_expr_id = builder_.AddChoices({builder_.AddSequence({rule_expr_id, ref_to_new_rule}), rule_expr_id}); builder_.UpdateRuleBody(new_rule_id, new_rule_expr_id); // Return the reference to the new rule return builder_.AddRuleRef(new_rule_id); } int32_t EBNFParser::HandleQuestionQuantifier(int32_t rule_expr_id) { // a? --> rule ::= a | empty auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_rule_expr_id = builder_.AddChoices({builder_.AddEmptyStr(), rule_expr_id}); auto new_rule_id = builder_.AddRule({new_rule_name, new_rule_expr_id}); return builder_.AddRuleRef(new_rule_id); } int64_t EBNFParser::ParseInteger() { if (!isdigit(Peek())) { ReportParseError("Expect integer"); } int64_t num = 0; while (Peek() && isdigit(Peek())) { num = num * 10 + (Peek() - '0'); Consume(); if (num > MAX_INTEGER_IN_GRAMMAR) { ReportParseError( "Integer is too large: parsed " + std::to_string(num) + ", max allowed is " + std::to_string(MAX_INTEGER_IN_GRAMMAR) ); } } return num; } // {x}: Match exactly x occurrences // {x,}: Match at least x occurrences // {x,y}: Match at least x occurrences, at most y occurrences std::pair EBNFParser::ParseRepetitionRange() { Consume(); ConsumeSpace(); int64_t lower = ParseInteger(); ConsumeSpace(); if (Peek() == ',') { Consume(); ConsumeSpace(); if (Peek() == '}') { Consume(); return {lower, -1}; } int64_t upper = ParseInteger(); if (upper < lower) { ReportParseError( "Lower bound is larger than upper bound: " + std::to_string(lower) + " > " + std::to_string(upper) ); } Consume(); return {lower, upper}; } else if (Peek() == '}') { Consume(); return {lower, lower}; } ReportParseError("Expect ',' or '}' in repetition range"); } int32_t EBNFParser::HandleRepetitionRange(int32_t rule_expr_id, int64_t lower, int64_t upper) { // Construct expr expr ... expr (l times) std::vector elements; for (int64_t i = 0; i < lower; ++i) { elements.push_back(rule_expr_id); } // Case 1: {l}: // expr expr ... expr (l times) if (upper == lower) { return builder_.AddSequence(elements); } // Case 2: {l,}: // expr expr ... expr (l times) rest // rest ::= "" | expr rest if (upper == -1) { auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); auto new_rule_id = builder_.AddEmptyRule(new_rule_name); auto ref_to_new_rule = builder_.AddRuleRef(new_rule_id); auto new_rule_expr_id = builder_.AddChoices( {builder_.AddEmptyStr(), builder_.AddSequence({rule_expr_id, ref_to_new_rule})} ); builder_.UpdateRuleBody(new_rule_id, new_rule_expr_id); elements.push_back(builder_.AddRuleRef(new_rule_id)); return builder_.AddSequence(elements); } // Case 3: {l, r} (r - l >= 1) // expr expr ... expr (l times) rest1 // rest1 ::= "" | expr rest2 // rest2 ::= "" | expr rest3 // ... // rest(r - l) ::= "" | expr std::vector rest_rule_ids; for (int64_t i = 0; i < upper - lower; ++i) { auto new_rule_name = builder_.GetNewRuleName(cur_rule_name_); rest_rule_ids.push_back(builder_.AddEmptyRule(new_rule_name)); } for (int64_t i = 0; i < upper - lower - 1; ++i) { auto ref_to_next_rule = builder_.AddRuleRef(rest_rule_ids[i + 1]); auto new_rule_expr_id = builder_.AddChoices( {builder_.AddEmptyStr(), builder_.AddSequence({rule_expr_id, ref_to_next_rule})} ); builder_.UpdateRuleBody(rest_rule_ids[i], new_rule_expr_id); } auto last_rule_expr_id = builder_.AddChoices({builder_.AddEmptyStr(), rule_expr_id}); builder_.UpdateRuleBody(rest_rule_ids.back(), last_rule_expr_id); elements.push_back(builder_.AddRuleRef(rest_rule_ids[0])); return builder_.AddSequence(elements); } int32_t EBNFParser::ParseElementWithQuantifier() { int32_t rule_expr_id = ParseElement(); ConsumeSpace(in_parentheses_); if (Peek() != '*' && Peek() != '+' && Peek() != '?' && Peek() != '{') { return rule_expr_id; } // Handle repetition range if (Peek() == '{') { auto [lower, upper] = ParseRepetitionRange(); return HandleRepetitionRange(rule_expr_id, lower, upper); } // Handle quantifiers Consume(); // We will transform a*, a+, a? into a rule, and return the reference to this rule switch (Peek(-1)) { case '*': // We assume that the star quantifier should be the body of some rule now return HandleStarQuantifier(rule_expr_id); case '+': return HandlePlusQuantifier(rule_expr_id); case '?': return HandleQuestionQuantifier(rule_expr_id); default: XGRAMMAR_LOG(FATAL) << "Unreachable"; } } int32_t EBNFParser::ParseSequence() { std::vector elements; do { elements.push_back(ParseElementWithQuantifier()); ConsumeSpace(in_parentheses_); } while (Peek() && Peek() != '|' && Peek() != ')' && Peek() != '\n' && Peek() != '\r' && (Peek() != '(' || Peek(1) != '=')); return builder_.AddSequence(elements); } int32_t EBNFParser::ParseChoices() { std::vector choices; choices.push_back(ParseSequence()); ConsumeSpace(); while (Peek() == '|') { Consume(); ConsumeSpace(); choices.push_back(ParseSequence()); ConsumeSpace(); } return builder_.AddChoices(choices); } std::pair EBNFParser::ParseTagDispatchElement() { if (Peek() != '(') { ReportParseError("Expect ( in tag dispatch element"); } Consume(); ConsumeSpace(); // Parse tag (a string literal) auto tag_id = ParseString(); if (builder_.GetRuleExpr(tag_id).type == RuleExprType::kEmptyStr) { ReportParseError("Tag cannot be empty"); } ConsumeSpace(); if (Peek() != ',') { ReportParseError("Expect , in tag dispatch element"); } Consume(); ConsumeSpace(); // Parse rule name (should refer to a rule in the grammar) auto rule_name = ParseIdentifier(false); // The rule cannot be the root rule and should be defined in the grammar if (rule_name == root_rule_name_) { ReportParseError("The root rule \"" + rule_name + "\" cannot be used as a tag"); } auto rule_id = builder_.GetRuleId(rule_name); if (rule_id == -1) { ReportParseError("Rule \"" + rule_name + "\" is not defined"); } ConsumeSpace(); if (Peek() != ')') { ReportParseError("Expect ) in tag dispatch element"); } Consume(); return {tag_id, rule_id}; } // TagDispatch: // root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ...) int32_t EBNFParser::ParseTagDispatchOrChoices() { auto prev_cursor = std::make_tuple(cur_, cur_line_, cur_column_, in_parentheses_); auto first_identifier = ParseIdentifier(true); if (first_identifier.empty() || first_identifier != "TagDispatch") { std::tie(cur_, cur_line_, cur_column_, in_parentheses_) = prev_cursor; return ParseChoices(); } // TODO(yixin): Make tagdispatch general if (cur_rule_name_ != root_rule_name_) { ReportParseError("TagDispatch should only be used in the root rule"); } ConsumeSpace(); if (Peek() != '(') { ReportParseError("Expect ( after TagDispatch"); } Consume(); ConsumeSpace(); std::vector> tag_dispatch_list; while (true) { auto tag_dispatch = ParseTagDispatchElement(); tag_dispatch_list.push_back(tag_dispatch); ConsumeSpace(); if (Peek() == ',') { Consume(); ConsumeSpace(); } else if (Peek() == ')') { Consume(); break; } else { ReportParseError("Expect , or ) in macro function TagDispatch"); } } return builder_.AddTagDispatch(tag_dispatch_list); } int32_t EBNFParser::ParseLookaheadAssertion() { if (Peek() != '(' || Peek(1) != '=') { return -1; } Consume(2); auto prev_in_parentheses = in_parentheses_; in_parentheses_ = true; ConsumeSpace(in_parentheses_); auto result = ParseSequence(); ConsumeSpace(in_parentheses_); if (Peek() != ')') { ReportParseError("Expect )"); } Consume(); in_parentheses_ = prev_in_parentheses; return result; } EBNFParser::Rule EBNFParser::ParseRule() { std::string name = ParseIdentifier(); cur_rule_name_ = name; ConsumeSpace(); if (Peek() != ':' || Peek(1) != ':' || Peek(2) != '=') { ReportParseError("Expect ::="); } Consume(3); ConsumeSpace(); auto body_id = ParseTagDispatchOrChoices(); ConsumeSpace(); auto lookahead_id = ParseLookaheadAssertion(); return {name, body_id, lookahead_id}; } void EBNFParser::BuildRuleNameToId() { ConsumeSpace(); while (Peek()) { auto name = ParseIdentifier(true); ConsumeSpace(false); if (Peek() == ':' && Peek(1) == ':' && Peek(2) == '=') { if (name.empty()) { ReportParseError("Expect rule name"); } Consume(3); if (builder_.GetRuleId(name) != -1) { ReportParseError("Rule \"" + name + "\" is defined multiple times"); } builder_.AddEmptyRule(name); } while (Peek() && Peek() != '\n' && Peek() != '\r') { Consume(); } ConsumeSpace(); } } void EBNFParser::ResetStringIterator(const char* cur) { cur_ = cur; cur_line_ = 1; cur_column_ = 1; cur_rule_name_ = ""; in_parentheses_ = false; } Grammar EBNFParser::Parse(const std::string& ebnf_string, const std::string& root_rule_name) { root_rule_name_ = root_rule_name; ResetStringIterator(ebnf_string.c_str()); BuildRuleNameToId(); ResetStringIterator(ebnf_string.c_str()); ConsumeSpace(); while (Peek()) { // Throw error when there are multiple lookahead assertions if (Peek() == '(' && Peek(1) == '=') { ReportParseError("Unexpected lookahead assertion"); } auto new_rule = ParseRule(); builder_.UpdateRuleBody(new_rule.name, new_rule.body_expr_id); // Update the lookahead assertion builder_.AddLookaheadAssertion(new_rule.name, new_rule.lookahead_assertion_id); ConsumeSpace(); } // Check that the root rule is defined if (builder_.GetRuleId(root_rule_name) == -1) { ReportParseError("The root rule with name \"" + root_rule_name + "\" is not found."); } return builder_.Get(root_rule_name); } Grammar ParseEBNF(const std::string& ebnf_string, const std::string& root_rule_name) { EBNFParser parser; return parser.Parse(ebnf_string, root_rule_name); } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_parser.h000066400000000000000000000021631500705317600173630ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_parser.h * \brief The header for the parser of BNF/EBNF grammar into BNF AST. */ #ifndef XGRAMMAR_GRAMMAR_PARSER_H_ #define XGRAMMAR_GRAMMAR_PARSER_H_ #include namespace xgrammar { /*! * \brief This class parses a BNF/EBNF grammar string into an BNF abstract syntax tree (AST). * \details This function accepts the EBNF notation defined in the W3C XML Specification * (https://www.w3.org/TR/xml/#sec-notation), which is a popular standard, with the following * changes: * - Using # as comment mark instead of C-style comments * - Accept C-style unicode escape sequence \u01AB, \U000001AB, \xAB instead of #x0123 * - Rule A-B (match A and not match B) is not supported yet * * See tests/python/serve/json.ebnf for an example. * \param ebnf_string The grammar string. * \param root_rule_name The name of the root rule. Default is "root". * \return The parsed grammar. */ Grammar ParseEBNF(const std::string& ebnf_string, const std::string& root_rule_name = "root"); } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_PARSER_H_ xgrammar-0.1.19/cpp/grammar_serializer.cc000066400000000000000000000162741500705317600204060ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_serializer.cc */ #include "grammar_serializer.h" #include #include "support/encoding.h" namespace xgrammar { std::string GrammarPrinter::PrintRule(const Rule& rule) { std::string res = rule.name + " ::= " + PrintRuleExpr(rule.body_expr_id); if (rule.lookahead_assertion_id != -1) { res += " (=" + PrintRuleExpr(rule.lookahead_assertion_id) + ")"; } return res; } std::string GrammarPrinter::PrintRule(int32_t rule_id) { return PrintRule(grammar_->GetRule(rule_id)); } std::string GrammarPrinter::PrintRuleExpr(const RuleExpr& rule_expr) { std::string result; switch (rule_expr.type) { case RuleExprType::kByteString: return PrintByteString(rule_expr); case RuleExprType::kCharacterClass: return PrintCharacterClass(rule_expr); case RuleExprType::kCharacterClassStar: return PrintCharacterClassStar(rule_expr); case RuleExprType::kEmptyStr: return PrintEmptyStr(rule_expr); case RuleExprType::kRuleRef: return PrintRuleRef(rule_expr); case RuleExprType::kSequence: return PrintSequence(rule_expr); case RuleExprType::kChoices: return PrintChoices(rule_expr); case RuleExprType::kTagDispatch: return PrintTagDispatch(rule_expr); default: XGRAMMAR_LOG(FATAL) << "Unexpected RuleExpr type: " << static_cast(rule_expr.type); } } std::string GrammarPrinter::PrintRuleExpr(int32_t rule_expr_id) { return PrintRuleExpr(grammar_->GetRuleExpr(rule_expr_id)); } std::string GrammarPrinter::PrintByteString(const RuleExpr& rule_expr) { std::string internal_str; internal_str.reserve(rule_expr.data_len); for (int i = 0; i < rule_expr.data_len; ++i) { internal_str += static_cast(rule_expr[i]); } auto codepoints = ParseUTF8(internal_str.c_str(), true); std::string result; for (auto codepoint : codepoints) { result += PrintAsEscapedUTF8(codepoint); } return "\"" + result + "\""; } std::string GrammarPrinter::PrintCharacterClass(const RuleExpr& rule_expr) { static const std::unordered_map kCustomEscapeMap = { {'-', "\\-"}, {']', "\\]"} }; std::string result = "["; bool is_negative = static_cast(rule_expr[0]); if (is_negative) { result += "^"; } for (auto i = 1; i < rule_expr.data_len; i += 2) { result += PrintAsEscapedUTF8(rule_expr[i], kCustomEscapeMap); if (rule_expr[i] == rule_expr[i + 1]) { continue; } result += "-"; result += PrintAsEscapedUTF8(rule_expr[i + 1], kCustomEscapeMap); } result += "]"; return result; } std::string GrammarPrinter::PrintCharacterClassStar(const RuleExpr& rule_expr) { return PrintCharacterClass(rule_expr) + "*"; } std::string GrammarPrinter::PrintEmptyStr(const RuleExpr& rule_expr) { return "\"\""; } std::string GrammarPrinter::PrintRuleRef(const RuleExpr& rule_expr) { return grammar_->GetRule(rule_expr[0]).name; } std::string GrammarPrinter::PrintSequence(const RuleExpr& rule_expr) { std::string result; result += "("; for (int i = 0; i < rule_expr.data_len; ++i) { result += PrintRuleExpr(rule_expr[i]); if (i + 1 != rule_expr.data_len) { result += " "; } } result += ")"; return result; } std::string GrammarPrinter::PrintChoices(const RuleExpr& rule_expr) { std::string result; result += "("; for (int i = 0; i < rule_expr.data_len; ++i) { result += PrintRuleExpr(rule_expr[i]); if (i + 1 != rule_expr.data_len) { result += " | "; } } result += ")"; return result; } std::string GrammarPrinter::PrintTagDispatch(const RuleExpr& rule_expr) { std::string result = "TagDispatch("; for (int i = 0; i < rule_expr.data_len; i += 2) { result += "(" + PrintRuleExpr(rule_expr[i]) + ", " + grammar_->GetRule(rule_expr[i + 1]).name + ")"; if (i + 2 != rule_expr.data_len) { result += ", "; } } result += ")"; return result; } std::string GrammarPrinter::ToString() { std::string result; int num_rules = grammar_->NumRules(); for (auto i = 0; i < num_rules; ++i) { result += PrintRule(grammar_->GetRule(i)) + "\n"; } return result; } std::string GrammarSerializer::Serialize() { picojson::object grammar_json_obj; picojson::array rules_json; for (const auto& rule : grammar_->rules_) { picojson::object rule_json; rule_json["name"] = picojson::value(rule.name); rule_json["body_expr_id"] = picojson::value(static_cast(rule.body_expr_id)); rules_json.push_back(picojson::value(rule_json)); } grammar_json_obj["rules"] = picojson::value(rules_json); picojson::array rule_expr_data_json; for (const auto& data : grammar_->rule_expr_data_) { rule_expr_data_json.push_back(picojson::value(static_cast(data))); } grammar_json_obj["rule_expr_data"] = picojson::value(rule_expr_data_json); picojson::array rule_expr_indptr_json; for (const auto& index_ptr : grammar_->rule_expr_indptr_) { rule_expr_indptr_json.push_back(picojson::value(static_cast(index_ptr))); } grammar_json_obj["rule_expr_indptr"] = picojson::value(rule_expr_indptr_json); auto grammar_json = picojson::value(grammar_json_obj); return grammar_json.serialize(prettify_); } Grammar GrammarDeserializer::Deserialize(std::string json_string) { auto node = std::make_shared(); auto checker = [&](bool condition) { XGRAMMAR_CHECK(condition) << "Failed to deserialize XGrammar object: " << json_string; }; picojson::value serialized_value; std::string err = picojson::parse(serialized_value, json_string); checker(err.empty() && serialized_value.is()); auto serialized_obj = serialized_value.get(); // rules checker(serialized_obj.count("rules") && serialized_obj["rules"].is()); auto rules_array = serialized_obj["rules"].get(); checker(rules_array.size() > 0); for (const auto& rule_value : rules_array) { checker(rule_value.is()); auto rule_obj = rule_value.get(); checker(rule_obj.count("name") && rule_obj["name"].is()); auto name = rule_obj["name"].get(); checker(rule_obj.count("body_expr_id") && rule_obj["body_expr_id"].is()); auto rule_expr = static_cast(rule_obj["body_expr_id"].get()); node->rules_.push_back(Grammar::Impl::Rule({name, rule_expr})); } // rule_expr_data checker( serialized_obj.count("rule_expr_data") && serialized_obj["rule_expr_data"].is() ); auto rule_expr_data_array = serialized_obj["rule_expr_data"].get(); for (const auto& data_json : rule_expr_data_array) { node->rule_expr_data_.push_back(static_cast(data_json.get())); } // rule_expr_indptr checker( serialized_obj.count("rule_expr_indptr") && serialized_obj["rule_expr_indptr"].is() ); auto rule_expr_indptr_array = serialized_obj["rule_expr_indptr"].get(); for (const auto& index_ptr_json : rule_expr_indptr_array) { node->rule_expr_indptr_.push_back(static_cast(index_ptr_json.get())); } return Grammar(node); } } // namespace xgrammar xgrammar-0.1.19/cpp/grammar_serializer.h000066400000000000000000000065651500705317600202520ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar_serializer.h * \brief The header for printing the AST of a BNF grammar. */ #ifndef XGRAMMAR_GRAMMAR_SERIALIZER_H_ #define XGRAMMAR_GRAMMAR_SERIALIZER_H_ #include #include #include "grammar_data_structure.h" namespace xgrammar { /*! * \brief Prints the BNF AST with standard BNF format. */ class GrammarPrinter { private: using Rule = Grammar::Impl::Rule; using RuleExprType = Grammar::Impl::RuleExprType; using RuleExpr = Grammar::Impl::RuleExpr; public: /*! * \brief Constructor. * \param grammar The grammar to print. */ explicit GrammarPrinter(const Grammar& grammar) : grammar_(grammar) {} /*! \brief Print the complete grammar. */ std::string ToString(); /*! \brief Print a rule. */ std::string PrintRule(const Rule& rule); /*! \brief Print a rule corresponding to the given id. */ std::string PrintRule(int32_t rule_id); /*! \brief Print a RuleExpr. */ std::string PrintRuleExpr(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr corresponding to the given id. */ std::string PrintRuleExpr(int32_t rule_expr_id); private: /*! \brief Print a RuleExpr for byte string. */ std::string PrintByteString(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for character class. */ std::string PrintCharacterClass(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for a star quantifier of a character class. */ std::string PrintCharacterClassStar(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for empty string. */ std::string PrintEmptyStr(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for rule reference. */ std::string PrintRuleRef(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for rule_expr sequence. */ std::string PrintSequence(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for rule_expr choices. */ std::string PrintChoices(const RuleExpr& rule_expr); /*! \brief Print a RuleExpr for tag dispatch. */ std::string PrintTagDispatch(const RuleExpr& rule_expr); Grammar grammar_; }; /*! * \brief Serialize the raw representation of the BNF AST to a string with JSON format. Stale for * now. * \sa BNFJSONParser::Parse for parsing the JSON string. * \details JSON format: * { * "rules": [ * {"name": "...", "rule_expr": rule_expr_id}, * {"name": "...", "rule_expr": rule_expr_id}, * ], * "rule_expr_data": [integers...], * "rule_expr_indptr": [integers...], * } */ class GrammarSerializer { public: /*! * \brief Constructor. * \param grammar The grammar to print. */ explicit GrammarSerializer(const Grammar& grammar, bool prettify = true) : grammar_(grammar), prettify_(prettify) {} /*! * \brief Dump the raw representation of the AST to a JSON file. * \param prettify Whether to format the JSON string. If false, all whitespaces will be removed. */ std::string Serialize(); private: Grammar grammar_; bool prettify_; }; /*! * \brief Parse a BNF grammar from the raw representation of the AST in JSON format. Stale for * now. */ class GrammarDeserializer { public: /*! * \brief Parse the JSON string * \param json_string The JSON string. * \return The parsed BNF grammar. */ static Grammar Deserialize(std::string json_string); }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_SERIALIZER_H_ xgrammar-0.1.19/cpp/json_schema_converter.cc000066400000000000000000002652661500705317600211160ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/json_schema_converter.cc */ #include "json_schema_converter.h" #include #include #include #include #include #include #include #include #include #include "ebnf_script_creator.h" #include "regex_converter.h" #include "support/logging.h" #include "support/utils.h" namespace xgrammar { /** * \brief Error for invalid schema: wrong type, invalid keyword, etc. */ struct InvalidSchemaError : public Error { InvalidSchemaError(const std::string& msg) : Error(msg, "InvalidSchemaError") {} }; /** * \brief Error for unsatisfiable schema: conflict in constraints, the whole schema is false, etc. */ struct UnsatisfiableSchemaError : public Error { UnsatisfiableSchemaError(const std::string& msg) : Error(msg, "UnsatisfiableSchemaError") {} }; /*! * \brief Manage the indent and separator for the generation of EBNF grammar. * \param indent The number of spaces for each indent. If it is std::nullopt, there will be no * indent or newline. * \param separator The separator between different elements in json. Examples include "," and ", ". * \param any_whitespace Whether to ignore the indentation restrictions, and allow any whitespace. */ class IndentManager { public: IndentManager(std::optional indent, const std::string& separator, bool any_whitespace) : any_whitespace_(any_whitespace), enable_newline_(indent.has_value()), indent_(indent.value_or(0)), separator_(separator), total_indent_(0), is_first_({true}) {} /*! \brief Enter a new indent level. */ void StartIndent() { total_indent_ += indent_; is_first_.push_back(true); } /*! \brief Exit the current indent level. */ void EndIndent() { total_indent_ -= indent_; is_first_.pop_back(); } /*! * \brief Get the next start separator in the current level. The next separator is escaped and * quoted. * \example * \code * IndentManager indent_manager(2, ", "); * indent_manager.StartIndent(); * indent_manager.StartSeparator(); // get the start separator: "\"\n \"" * indent_manager.MiddleSeparator(); // get the middle separator: "\",\n \"" * indent_manager.EndSeparator(); // get the end separator: "\"\n\"" * indent_manager.EndIndent(); * \endcode */ std::string StartSeparator(); std::string MiddleSeparator(); std::string EndSeparator(); std::string EmptySeparator(); /*! * \brief Get the next separator in the current level. When first called in the current * level, the starting separator will be returned. When called again, the middle separator will be * returned. When called with `is_end=True`, the ending separator will be returned. * \param is_end Get the separator for the end of the current level. * \example * \code * IndentManager indent_manager(2, ", "); * indent_manager.StartIndent(); * indent_manager.GetSep(); // get the start separator: "\"\n \"" * indent_manager.GetSep(); // get the middle separator: "\",\n \"" * indent_manager.GetSep(true); // get the end separator: "\"\n\"" * indent_manager.EndIndent(); * \endcode */ std::string NextSeparator(bool is_end = false); private: bool any_whitespace_; bool enable_newline_; int indent_; std::string separator_; int total_indent_; std::vector is_first_; friend class JSONSchemaConverter; }; std::string IndentManager::StartSeparator() { if (any_whitespace_) { return "[ \\n\\t]*"; } if (!enable_newline_) { return "\"\""; } return "\"\\n" + std::string(total_indent_, ' ') + "\""; } std::string IndentManager::MiddleSeparator() { if (any_whitespace_) { return "[ \\n\\t]* \"" + separator_ + "\" [ \\n\\t]*"; } if (!enable_newline_) { return "\"" + separator_ + "\""; } return "\"" + separator_ + "\\n" + std::string(total_indent_, ' ') + "\""; } std::string IndentManager::EndSeparator() { if (any_whitespace_) { return "[ \\n\\t]*"; } if (!enable_newline_) { return "\"\""; } return "\"\\n" + std::string(total_indent_ - indent_, ' ') + "\""; } std::string IndentManager::EmptySeparator() { if (any_whitespace_) { return "[ \\n\\t]*"; } return "\"\""; } std::string IndentManager::NextSeparator(bool is_end) { if (any_whitespace_) { if (is_first_.back() || is_end) { is_first_.back() = false; return "[ \\n\\t]*"; } else { return "[ \\n\\t]* \"" + separator_ + "\" [ \\n\\t]*"; } } std::string res = ""; if (!is_first_.back() && !is_end) { res += separator_; } is_first_.back() = false; if (enable_newline_) { res += "\\n"; } if (!is_end) { res += std::string(total_indent_, ' '); } else { res += std::string(total_indent_ - indent_, ' '); } return "\"" + res + "\""; } /*! * \brief Convert JSON schema string to EBNF grammar string. The parameters follow * JSONSchemaToEBNF(). * * \note About the representation of json schema in this converter. JSON schema could be two types: * bool (true or false) or dict (a json dict) containing attributes. We use picojson::value to * represent the json schema. */ class JSONSchemaConverter { public: JSONSchemaConverter( const picojson::value& json_schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode ); /*! \brief The root method. Convert the JSON schema to EBNF grammar string. */ std::string Convert(); /*! \brief Generate the regex for integer range. Public for testing. */ static std::string GenerateRangeRegex(std::optional start, std::optional end); /*! \brief Generate the regex for float range. Public for testing. */ static std::string GenerateFloatRangeRegex( std::optional start, std::optional end, int precision ); private: // The name of the root rule inline static const std::string kRootRuleName = "root"; // The name of the basic rules inline static const std::string kBasicAny = "basic_any"; inline static const std::string kBasicInteger = "basic_integer"; inline static const std::string kBasicNumber = "basic_number"; inline static const std::string kBasicString = "basic_string"; inline static const std::string kBasicBoolean = "basic_boolean"; inline static const std::string kBasicNull = "basic_null"; inline static const std::string kBasicArray = "basic_array"; inline static const std::string kBasicObject = "basic_object"; // The name of the helper rules to construct basic rules inline static const std::string kBasicEscape = "basic_escape"; inline static const std::string kBasicStringSub = "basic_string_sub"; /*! \brief Add the basic rules to the rules list and the basic_rules_cache. */ void AddBasicRules(); /*! \brief Add helper rules for the basic rules. */ void AddHelperRules(); /*! \brief Create a rule for the given schema and name, and add it to the basic_rules_cache. */ void CreateBasicRule(const picojson::value& schema, const std::string& name); /*! \brief Get the index for the schema in the cache. Keys that do not effect the validation * will be ignored when finding the corresponding cache rule. */ std::string GetSchemaCacheIndex(const picojson::value& schema); /*! \brief Helpers for GenerateRangeRegex and GenerateFloatRangeRegex */ static std::string MakePatternForDigitRange(char start, char end, int remainingDigits); static std::vector GenerateNumberPatterns(int lower, int upper); static std::string GenerateSubRangeRegex(int lower, int upper); static std::string FormatFloat(double value, int precision); /*! * \brief Create a rule with the given schema and rule name hint. * \returns The name of the rule will be returned. That is not necessarily the same as the * rule_name_hint due to the caching mechanism. */ std::string CreateRuleFromSchema( const picojson::value& schema, const std::string& rule_name_hint ); /*! \brief Get the next separator in the current level from the indent manager. */ std::string NextSeparator(bool is_end = false); /*! \brief Warn if any keyword is existing in the schema but not supported. */ static void WarnUnsupportedKeywords( const picojson::value& schema, const std::vector& keywords, bool verbose = false ); /*! \brief Warn if any keyword is existing in the object but not supported. */ static void WarnUnsupportedKeywords( const picojson::object& schema, const std::vector& keywords, bool verbose = false ); // NOTE: the visit functions should always return the rule body for later constructing the rule. /*! \brief Visit the schema and return the rule body for later constructing the rule. */ std::string VisitSchema(const picojson::value& schema, const std::string& rule_name); /*! \brief Visit a reference schema. */ std::string VisitRef(const picojson::object& schema, const std::string& rule_name); /*! \brief Get the rule from the URI. */ std::string URIToRule(const std::string& uri); /*! \brief Visit a const schema. */ std::string VisitConst(const picojson::object& schema, const std::string& rule_name); /*! \brief Visit an enum schema. */ std::string VisitEnum(const picojson::object& schema, const std::string& rule_name); /*! \brief Convert the JSON string to a printable string that can be shown in BNF. */ std::string JSONStrToPrintableStr(const std::string& json_str); /*! \brief Visit an anyOf schema. */ std::string VisitAnyOf(const picojson::object& schema, const std::string& rule_name); picojson::value FuseAllOfSchema(const std::vector& schemas); /*! \brief Visit an allOf schema. */ std::string VisitAllOf(const picojson::object& schema, const std::string& rule_name); /*! \brief Visit a true schema that can match anything. */ std::string VisitAny(const picojson::value& schema, const std::string& rule_name); /*! \brief Visit an integer schema. */ std::string VisitInteger(const picojson::object& schema, const std::string& rule_name); /*! \brief Visit a number schema. */ std::string VisitNumber(const picojson::object& schema, const std::string& rule_name); /*! \brief Visit a string schema. */ std::string VisitString(const picojson::object& schema, const std::string& rule_name); /*! \brief Visit a boolean schema. */ std::string VisitBoolean(const picojson::object& schema, const std::string& rule_name); /*! \brief Visit a null schema. */ std::string VisitNull(const picojson::object& schema, const std::string& rule_name); struct ArraySpec { std::vector prefix_item_schemas; bool allow_additional_items; picojson::value additional_item_schema; int min_items; int max_items; }; Result ParseArraySchema(const picojson::object& schema); /*! * \brief Visit an array schema. * \example * Schema: * \code * { * "type": "array", * "prefixItems": [ * {"type": "boolean"}, * {"type": "integer"} * ], * "items": { * "type": "string" * } * } * \endcode * Rule (not considering the indent): * \code * root ::= "[" basic_boolean ", " basic_integer (", " basic_string)* "]" * \endcode */ std::string VisitArray(const picojson::object& schema, const std::string& rule_name); /*! * \brief Visit an object schema. * \example * Schema: * \code * { * "type": "object", * "properties": { * "a": {"type": "string"}, * "b": {"type": "integer"} * }, * "required": ["a"], * "additionalProperties": true * } * \endcode * * Rule (not considering the indent): * \code * root ::= "{" "a" ":" basic_string (", " "b" ":" basic_integer)* * (", " basic_string ": " basic_any)* "}" * \endcode * We need special handling when all properties are optional, since the handling of separators * is tricky in this case. E.g. * Schema: * \code * { * "type": "object", * "properties": { * "a": {"type": "string"}, * "b": {"type": "integer"}, * "c": {"type": "boolean"} * }, * "additionalProperties": true * } * \endcode * * Rule (indent=2): * \code * root ::= "{" ("\n " (a root_sub_1 | b root_sub_2 | c root_sub_3 | d root_sub_3) * "\n" | "") "}" * root_sub_1 ::= ",\n " b r2 | r2 * root_sub_2 ::= ",\n " c r3 | r3 * root_sub_3 ::= (",\n " d)* * \endcode */ std::string VisitObject(const picojson::object& schema, const std::string& rule_name); /*! * \brief Visit a type array schema: * \example * \code * { * "type": ["integer", "string"] * } * \endcode * * Method: * - Create a schema for each type in the type array. Copying all other properties. * - Visit each schema and get the rule name. * - Return "(" rule_name_1 | rule_name_2 | ... | rule_name_n ")" */ std::string VisitTypeArray(const picojson::object& schema, const std::string& rule_name); /*! \brief Get the pattern for a property in the object schema. */ std::string GetPropertyPattern( const std::string& prop_name, const picojson::value& prop_schema, const std::string& rule_name, int idx ); /*! \brief Get the pattern for the additional/unevaluated properties in the object schema. */ std::string GetOtherPropertyPattern( const std::string& key_pattern, const picojson::value& prop_schema, const std::string& rule_name, const std::string& rule_name_suffix ); /*! \brief Get the partial rule for the properties when all properties are optional. See the * example in VisitObject(). */ std::string GetPartialRuleForPropertiesAllOptional( const std::vector>& properties, const picojson::value& additional, const std::string& rule_name, const std::string& additional_suffix = "" ); /*! * \brief Get the partial rule for the properties when some properties are required. See the * example in VisitObject(). * * The constructed rule should be: * \code * start_separator (optional_property separator)? (optional_property separator)? ... * first_required_property (separator optional_property)? separator required_property ... * end_separator * \endcode * * i.e. Before the first required property, all properties are in the form * (property separator) ; and after the first required property, all properties are in the form * (separator property) . */ std::string GetPartialRuleForPropertiesContainRequired( const std::vector>& properties, const std::unordered_set& required, const std::string& rule_name ); // The EBNF script creator EBNFScriptCreator ebnf_script_creator_; // The indent manager to get separators std::optional indentManager_; // The root JSON schema picojson::value json_schema_; // Whether to use strict mode in conversion. See JSONSchemaToEBNF(). bool strict_mode_; // The colon separator std::string colon_pattern_; // The cache for basic rules. Mapping from the key of schema returned by GetSchemaCacheIndex() // to the basic rule name. std::unordered_map basic_rules_cache_; // Whether to use any whitespace in the conversion bool any_whitespace_; // The cache for URI to rule. Mapping from the URI to the rule name. std::unordered_map uri_to_rule_cache_; }; JSONSchemaConverter::JSONSchemaConverter( const picojson::value& json_schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode ) : json_schema_(json_schema), strict_mode_(strict_mode), any_whitespace_(any_whitespace) { if (!separators.has_value()) { if (indent == std::nullopt) { separators = std::make_pair(", ", ": "); } else { separators = std::make_pair(",", ": "); } } if (any_whitespace) { separators = std::make_pair(",", ":"); } indentManager_ = IndentManager(indent, separators->first, any_whitespace); if (any_whitespace) { colon_pattern_ = "[ \\n\\t]* \"" + separators->second + "\" [ \\n\\t]*"; } else { colon_pattern_ = "\"" + separators->second + "\""; } AddBasicRules(); } std::string JSONSchemaConverter::Convert() { CreateRuleFromSchema(json_schema_, kRootRuleName); return ebnf_script_creator_.GetScript(); } void JSONSchemaConverter::AddBasicRules() { bool past_strict_mode = strict_mode_; // Allow any field for basic array/obj rules strict_mode_ = false; auto past_indent_manager = indentManager_; if (any_whitespace_) { indentManager_ = IndentManager(std::nullopt, ",", true); } else { indentManager_ = IndentManager(std::nullopt, ", ", false); } AddHelperRules(); CreateBasicRule(picojson::value(true), kBasicAny); basic_rules_cache_[GetSchemaCacheIndex(picojson::value(picojson::object()))] = kBasicAny; CreateBasicRule( picojson::value(picojson::object{{"type", picojson::value("integer")}}), kBasicInteger ); CreateBasicRule( picojson::value(picojson::object{{"type", picojson::value("number")}}), kBasicNumber ); CreateBasicRule( picojson::value(picojson::object{{"type", picojson::value("string")}}), kBasicString ); CreateBasicRule( picojson::value(picojson::object{{"type", picojson::value("boolean")}}), kBasicBoolean ); CreateBasicRule(picojson::value(picojson::object{{"type", picojson::value("null")}}), kBasicNull); CreateBasicRule( picojson::value(picojson::object{{"type", picojson::value("array")}}), kBasicArray ); CreateBasicRule( picojson::value(picojson::object{{"type", picojson::value("object")}}), kBasicObject ); strict_mode_ = past_strict_mode; indentManager_ = past_indent_manager; } void JSONSchemaConverter::AddHelperRules() { ebnf_script_creator_.AddRule( kBasicEscape, "[\"\\\\/bfnrt] | \"u\" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9]" ); ebnf_script_creator_.AddRule( kBasicStringSub, "(\"\\\"\" | [^\"\\\\\\r\\n] " + kBasicStringSub + " | \"\\\\\" " + kBasicEscape + " " + kBasicStringSub + ") (= [ \\n\\t]* [,}\\]:])" ); } void JSONSchemaConverter::CreateBasicRule(const picojson::value& schema, const std::string& name) { std::string rule_name = CreateRuleFromSchema(schema, name); basic_rules_cache_[GetSchemaCacheIndex(schema)] = rule_name; } std::string JSONSchemaConverter::NextSeparator(bool is_end) { return indentManager_->NextSeparator(is_end); } void JSONSchemaConverter::WarnUnsupportedKeywords( const picojson::value& schema, const std::vector& keywords, bool verbose ) { if (schema.is()) { return; } XGRAMMAR_DCHECK(schema.is()) << "Schema should be an object or bool"; WarnUnsupportedKeywords(schema.get(), keywords, verbose); } void JSONSchemaConverter::WarnUnsupportedKeywords( const picojson::object& schema, const std::vector& keywords, bool verbose ) { if (!verbose) { return; } for (const auto& keyword : keywords) { if (schema.find(keyword) != schema.end()) { XGRAMMAR_LOG(WARNING) << "Keyword " << keyword << " is not supported"; } } } std::string JSONSchemaConverter::CreateRuleFromSchema( const picojson::value& schema, const std::string& rule_name_hint ) { std::string idx = GetSchemaCacheIndex(schema); if (basic_rules_cache_.count(idx)) { if (rule_name_hint == kRootRuleName) { // If the rule name is root, we need to define the root rule instead of just using the // cached rule. return ebnf_script_creator_.AddRule(rule_name_hint, basic_rules_cache_[idx]); } return basic_rules_cache_[idx]; } auto rule_name = ebnf_script_creator_.AllocateRuleName(rule_name_hint); std::string rule_content = VisitSchema(schema, rule_name); ebnf_script_creator_.AddRuleWithAllocatedName(rule_name, rule_content); return rule_name; } std::string JSONSchemaConverter::GetSchemaCacheIndex(const picojson::value& schema) { // Keys that do not effect the validation static const std::unordered_set kSkippedKeys = { "title", "default", "description", "examples", "deprecated", "readOnly", "writeOnly", "$comment", "$schema", }; if (schema.is()) { // remove skipped keys and sort key by lexicographical order std::string result = "{"; std::vector> sorted_kv; for (const auto& kv : schema.get()) { if (kSkippedKeys.count(kv.first) == 0) { sorted_kv.push_back(kv); } } std::sort(sorted_kv.begin(), sorted_kv.end(), [](const auto& lhs, const auto& rhs) { return lhs.first < rhs.first; }); int idx = 0; for (const auto& [key, value] : sorted_kv) { if (idx != 0) { result += ","; } ++idx; result += "\"" + key + "\":" + GetSchemaCacheIndex(value); } return result + "}"; } else if (schema.is()) { std::string result = "["; int idx = 0; for (const auto& item : schema.get()) { if (idx != 0) { result += ","; } ++idx; result += GetSchemaCacheIndex(item); } return result + "]"; } // If the object is neither an array nor an object, return it directly return schema.serialize(false); } std::string JSONSchemaConverter::VisitSchema( const picojson::value& schema, const std::string& rule_name ) { if (schema.is()) { XGRAMMAR_CHECK(schema.get()) << "Schema should not be false: it cannot accept any value"; return VisitAny(schema, rule_name); } XGRAMMAR_CHECK(schema.is()) << "Schema should be an object or bool, but got " << schema.serialize(false); WarnUnsupportedKeywords( schema, { "not", "if", "then", "else", "dependentRequired", "dependentSchemas", } ); const auto& schema_obj = schema.get(); if (schema_obj.count("$ref")) { return VisitRef(schema_obj, rule_name); } else if (schema_obj.count("const")) { return VisitConst(schema_obj, rule_name); } else if (schema_obj.count("enum")) { return VisitEnum(schema_obj, rule_name); } else if (schema_obj.count("anyOf") || schema_obj.count("oneOf")) { return VisitAnyOf(schema_obj, rule_name); } else if (schema_obj.count("allOf")) { return VisitAllOf(schema_obj, rule_name); } else if (schema_obj.count("type")) { if (schema_obj.at("type").is()) { return VisitTypeArray(schema_obj, rule_name); } XGRAMMAR_CHECK(schema_obj.at("type").is()) << "Type should be a string"; const std::string& type = schema_obj.at("type").get(); if (type == "integer") { return VisitInteger(schema_obj, rule_name); } else if (type == "number") { return VisitNumber(schema_obj, rule_name); } else if (type == "string") { return VisitString(schema_obj, rule_name); } else if (type == "boolean") { return VisitBoolean(schema_obj, rule_name); } else if (type == "null") { return VisitNull(schema_obj, rule_name); } else if (type == "array") { return VisitArray(schema_obj, rule_name); } else if (type == "object") { return VisitObject(schema_obj, rule_name); } else { XGRAMMAR_LOG(FATAL) << "Unsupported type \"" << type << "\""; } } else if (schema_obj.count("properties") || schema_obj.count("additionalProperties") || schema_obj.count("unevaluatedProperties")) { return VisitObject(schema_obj, rule_name); } else if (schema_obj.count("items") || schema_obj.count("prefixItems") || schema_obj.count("unevaluatedItems")) { return VisitArray(schema_obj, rule_name); } // If no above keyword is detected, we treat it as any return VisitAny(schema, rule_name); } std::string JSONSchemaConverter::VisitRef( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("$ref") && schema.at("$ref").is()) << "Schema $ref should be a string"; auto ref_str = schema.at("$ref").get(); return URIToRule(ref_str); } std::string JSONSchemaConverter::URIToRule(const std::string& uri) { if (uri_to_rule_cache_.count(uri)) { return uri_to_rule_cache_[uri]; } if (uri == "#") { return kRootRuleName; } if (uri.size() < 2 || uri[0] != '#' || uri[1] != '/') { XGRAMMAR_LOG(WARNING) << "URI should either be '#' or start with '#/' but got " << uri; return kBasicAny; } std::vector parts; std::stringstream ss(uri.substr(2)); std::string part; std::string new_rule_name_perfix; while (std::getline(ss, part, '/')) { if (!part.empty()) { parts.push_back(part); } // Update new_rule_name_perfix if (!new_rule_name_perfix.empty()) { new_rule_name_perfix += "_"; } // filter out non-alpha characters for (const auto& c : part) { if (std::isalpha(c) || c == '_' || c == '-' || c == '.') { new_rule_name_perfix += c; } } } picojson::value current = json_schema_; for (const auto& part : parts) { XGRAMMAR_CHECK(current.is() && current.contains(part)) << "Cannot find field " << part << " in " << current.serialize(false); current = current.get(part); } auto new_rule_name = ebnf_script_creator_.AllocateRuleName(new_rule_name_perfix); uri_to_rule_cache_[uri] = new_rule_name; auto body = VisitSchema(current, new_rule_name); ebnf_script_creator_.AddRuleWithAllocatedName(new_rule_name, body); return new_rule_name; } std::string JSONSchemaConverter::VisitConst( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("const")); // TODO(yixin): Customize serialize to support indent logics return "\"" + JSONStrToPrintableStr(schema.at("const").serialize()) + "\""; } std::string JSONSchemaConverter::VisitEnum( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("enum")); std::string result = ""; int idx = 0; for (auto value : schema.at("enum").get()) { if (idx != 0) { result += " | "; } ++idx; result += "(\"" + JSONStrToPrintableStr(value.serialize()) + "\")"; } return result; } std::string JSONSchemaConverter::JSONStrToPrintableStr(const std::string& json_str) { static const std::vector> kReplaceMapping = { {"\\", "\\\\"}, {"\"", "\\\""} }; std::string result = json_str; for (const auto& [k, v] : kReplaceMapping) { size_t pos = 0; while ((pos = result.find(k, pos)) != std::string::npos) { result.replace(pos, k.length(), v); pos += v.length(); } } return result; } std::string JSONSchemaConverter::VisitAnyOf( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("anyOf") || schema.count("oneOf")); std::string result = ""; int idx = 0; auto anyof_schema = schema.count("anyOf") ? schema.at("anyOf") : schema.at("oneOf"); XGRAMMAR_CHECK(anyof_schema.is()) << "anyOf or oneOf must be an array"; for (auto anyof_schema : anyof_schema.get()) { if (idx != 0) { result += " | "; } result += CreateRuleFromSchema(anyof_schema, rule_name + "_case_" + std::to_string(idx)); ++idx; } return result; } picojson::value JSONSchemaConverter::FuseAllOfSchema(const std::vector& schemas) { picojson::object fused_schema; XGRAMMAR_LOG(WARNING) << "Support for allOf with multiple options is still ongoing"; return picojson::value(fused_schema); } std::string JSONSchemaConverter::VisitAllOf( const picojson::object& schema, const std::string& rule_name ) { // We support common usecases of AllOf, but not all, because it's impossible to support all // cases with CFG XGRAMMAR_CHECK(schema.count("allOf")); XGRAMMAR_CHECK(schema.at("allOf").is()) << "allOf must be an array"; auto all_array = schema.at("allOf").get(); // Case 1: allOf is a single schema if (all_array.size() == 1) { return VisitSchema(all_array[0], rule_name + "_case_0"); } // Case 2: allOf is a list of schemas, we fuse them into a single schema auto fused_schema = FuseAllOfSchema(all_array); return VisitSchema(fused_schema, rule_name); } std::string JSONSchemaConverter::VisitAny( const picojson::value& schema, const std::string& rule_name ) { // Note integer is a subset of number, so we don't need to add integer here return kBasicNumber + " | " + kBasicString + " | " + kBasicBoolean + " | " + kBasicNull + " | " + kBasicArray + " | " + kBasicObject; } std::string JSONSchemaConverter::MakePatternForDigitRange( char start, char end, int remainingDigits ) { std::ostringstream oss; if (start == end) { oss << start; } else { oss << "[" << start << "-" << end << "]"; } if (remainingDigits > 0) { oss << "\\d{" << remainingDigits << "}"; } return oss.str(); } std::vector JSONSchemaConverter::GenerateNumberPatterns(int lower, int upper) { std::vector patterns; int lower_len = static_cast(std::to_string(lower).size()); int upper_len = static_cast(std::to_string(upper).size()); for (int len = lower_len; len <= upper_len; ++len) { const int digit_min = static_cast(std::pow(10, len - 1)); const int digit_max = static_cast(std::pow(10, len)) - 1; int start = (len == lower_len) ? lower : digit_min; int end = (len == upper_len) ? upper : digit_max; std::string start_str = std::to_string(start); std::string end_str = std::to_string(end); if (len == 1) { patterns.push_back(MakePatternForDigitRange(start_str[0], end_str[0], 0)); continue; } int prefix = 0; while (prefix < len && start_str[prefix] == end_str[prefix]) { prefix++; } if (prefix == len) { patterns.push_back(start_str); continue; } // Generate common prefix pattern if only last digit differs for start/end if (prefix > 0 && prefix >= len - 2) { std::string common_part = start_str.substr(0, prefix); patterns.push_back( common_part + MakePatternForDigitRange(start_str[prefix], end_str[prefix], len - prefix - 1) ); continue; } if (len == lower_len && len == upper_len) { if (start == digit_max) { XGRAMMAR_ICHECK(start == end); patterns.push_back(start_str); } else if (start == digit_min) { if (end == digit_max) { patterns.push_back("[1-9]\\d{" + std::to_string(len - 1) + "}"); } else { for (size_t i = 0; i < end_str.size(); i++) { if (i == 0) { // First digit: range from 1 to end[0]-1 if (end_str[0] > '1') { patterns.push_back( MakePatternForDigitRange('1', static_cast(end_str[0] - 1), len - 1) ); } } else { // Fix first i digits to end[0..i-1], then range from 0 to end[i]-1 std::string prefix = end_str.substr(0, i); if (end_str[i] > '0') { patterns.push_back( prefix + MakePatternForDigitRange('0', static_cast(end_str[i] - 1), len - i - 1) ); } } } patterns.push_back(end_str); } } else if (end == digit_max) { for (size_t i = 0; i < start_str.size(); i++) { if (i == 0) { // First digit: range from start[0]+1 to 9 if (start_str[0] < '9') { patterns.push_back( MakePatternForDigitRange(static_cast(start_str[0] + 1), '9', len - 1) ); } } else { // Fix first i digits to start[0..i-1], then range from start[i]+1 to 9 std::string prefix = start_str.substr(0, i); if (start_str[i] < '9') { patterns.push_back( prefix + MakePatternForDigitRange(static_cast(start_str[i] + 1), '9', len - i - 1) ); } } } patterns.push_back(start_str); } else { // Handle middle range between first digits if they differ by more than 1 char start_first_digit = start_str[0]; char end_first_digit = end_str[0]; if (end_first_digit - start_first_digit > 1) { patterns.push_back(MakePatternForDigitRange( static_cast(start_first_digit + 1), static_cast(end_first_digit - 1), len - 1 )); } // Patterns starting from start for (size_t i = 0; i < start_str.size(); i++) { if (i == 0) { std::string prefix = start_str.substr(0, 1); if (start_str[1] < '9') { patterns.push_back( prefix + MakePatternForDigitRange(static_cast(start_str[1] + 1), '9', len - 2) ); } } else { std::string prefix = start_str.substr(0, i); if (start_str[i] < '9') { patterns.push_back( prefix + MakePatternForDigitRange(static_cast(start_str[i] + 1), '9', len - i - 1) ); } } } patterns.push_back(start_str); // Patterns starting from end for (size_t i = 0; i < end_str.size(); i++) { if (i == 0) { std::string prefix = end_str.substr(0, 1); if (end_str[1] > '0') { patterns.push_back( prefix + MakePatternForDigitRange('0', static_cast(end_str[1] - 1), len - 2) ); } } else { std::string prefix = end_str.substr(0, i); if (end_str[i] > '0') { patterns.push_back( prefix + MakePatternForDigitRange('0', static_cast(end_str[i] - 1), len - i - 1) ); } } } patterns.push_back(end_str); } } else if (len == lower_len && len != upper_len) { XGRAMMAR_ICHECK(end == digit_max); if (start == digit_min) { patterns.push_back("[1-9]\\d{" + std::to_string(len - 1) + "}"); } else { for (size_t i = 0; i < start_str.size(); i++) { if (i == 0) { if (start_str[0] < '9') { patterns.push_back( MakePatternForDigitRange(static_cast(start_str[0] + 1), '9', len - 1) ); } } else { std::string prefix = start_str.substr(0, i); if (start_str[i] < '9') { patterns.push_back( prefix + MakePatternForDigitRange(static_cast(start_str[i] + 1), '9', len - i - 1) ); } } } patterns.push_back(start_str); } } else if (len != lower_len && len == upper_len) { XGRAMMAR_ICHECK(start == digit_min); if (end == digit_max) { patterns.push_back("[1-9]\\d{" + std::to_string(len - 1) + "}"); } else { for (size_t i = 0; i < end_str.size(); i++) { if (i == 0) { if (end_str[0] > '1') { patterns.push_back( MakePatternForDigitRange('1', static_cast(end_str[0] - 1), len - 1) ); } } else { std::string prefix = end_str.substr(0, i); if (end_str[i] > '0') { patterns.push_back( prefix + MakePatternForDigitRange('0', static_cast(end_str[i] - 1), len - i - 1) ); } } } patterns.push_back(end_str); } } // len != lower_len && len != upper_len else { patterns.push_back("[1-9]\\d{" + std::to_string(len - 1) + "}"); } } return patterns; } std::string JSONSchemaConverter::GenerateSubRangeRegex(int lower, int upper) { std::vector patterns = GenerateNumberPatterns(lower, upper); std::ostringstream oss; for (size_t i = 0; i < patterns.size(); ++i) { if (i > 0) { oss << "|"; } oss << patterns[i]; } return "(" + oss.str() + ")"; } std::string JSONSchemaConverter::GenerateRangeRegex( std::optional start, std::optional end ) { std::vector parts; std::ostringstream result; // If start and end undefined - match any integer if (!start && !end) { return "^-?\\d+$"; } // Only start defined - match numbers >= start if (start && !end) { if (start.value() <= 0) { if (start.value() < 0) { parts.push_back("-" + GenerateSubRangeRegex(-(-start.value()), 1)); } parts.push_back("0"); parts.push_back("[1-9]\\d*"); } else { std::string start_str = std::to_string(start.value()); int len = static_cast(start_str.length()); if (len == 1) { parts.push_back(MakePatternForDigitRange(start_str[0], '9', 0)); parts.push_back("[1-9]\\d*"); } else { parts.push_back(start_str); // Handle numbers of same length for (size_t i = 0; i < start_str.size(); i++) { if (i == 0) { // First digit: range from start[0]+1 to 9 if (start_str[0] < '9') { parts.push_back( MakePatternForDigitRange(static_cast(start_str[0] + 1), '9', len - 1) ); } } else { // Fix first i digits to start[0..i-1], then range from start[i]+1 to 9 std::string prefix = start_str.substr(0, i); if (start_str[i] < '9') { parts.push_back( prefix + MakePatternForDigitRange(static_cast(start_str[i] + 1), '9', len - i - 1) ); } } } parts.push_back("[1-9]\\d{" + std::to_string(len) + ",}"); } } } // Only end defined - match numbers <= end if (!start && end) { if (end.value() >= 0) { parts.push_back("-[1-9]\\d*"); parts.push_back("0"); if (end.value() > 0) { parts.push_back(GenerateSubRangeRegex(1, end.value())); } } else { std::string end_str = std::to_string(-end.value()); int len = static_cast(end_str.length()); if (len == 1) { parts.push_back("-" + MakePatternForDigitRange(end_str[0], '9', 0)); parts.push_back("-[1-9]\\d*"); } else { parts.push_back(std::to_string(end.value())); // Handle -123 exactly for (size_t i = 0; i < end_str.size(); i++) { if (i == 0) { if (end_str[0] > '1') { parts.push_back( "-" + MakePatternForDigitRange('1', static_cast(end_str[0] - 1), len - 1) ); } } else { std::string prefix = end_str.substr(0, i); if (end_str[i] > '0') { parts.push_back( "-" + prefix + MakePatternForDigitRange('0', static_cast(end_str[i] - 1), len - i - 1) ); } } } parts.push_back("-[1-9]\\d{" + std::to_string(len) + ",}"); } } } if (start && end) { int range_start = start.value(); int range_end = end.value(); if (range_start > range_end) { return "^()$"; // Invalid input } if (range_start < 0) { int neg_start = range_start; int neg_end = std::min(-1, range_end); parts.push_back("-" + GenerateSubRangeRegex(-neg_end, -neg_start)); } if (range_start <= 0 && range_end >= 0) { parts.push_back("0"); } if (range_end > 0) { int pos_start = std::max(1, range_start); parts.push_back(GenerateSubRangeRegex(pos_start, range_end)); } } result << "^("; for (size_t i = 0; i < parts.size(); ++i) { if (i > 0) { result << "|"; } result << parts[i]; } result << ")$"; return result.str(); } std::string JSONSchemaConverter::FormatFloat(double value, int precision = 6) { std::ostringstream oss; oss << std::fixed << std::setprecision(precision) << value; std::string result = oss.str(); // Remove trailing zeros after decimal point size_t decimalPos = result.find('.'); if (decimalPos != std::string::npos) { size_t lastNonZero = result.find_last_not_of('0'); if (lastNonZero != std::string::npos && lastNonZero > decimalPos) { result.erase(lastNonZero + 1); } else if (lastNonZero == decimalPos) { result.erase(decimalPos); } } return result; } std::string JSONSchemaConverter::GenerateFloatRangeRegex( std::optional start, std::optional end, int precision = 6 ) { if ((start && end) && (start.value() > end.value())) { return "^()$"; // Invalid input } if (!start && !end) { return "^-?\\d+(\\.\\d{1," + std::to_string(precision) + "})?$"; } std::vector parts; int startInt = 0; int endInt = 0; double startFrac = 0.0; double endFrac = 0.0; bool isStartNegative = false; bool isEndNegative = false; if (start) { isStartNegative = start.value() < 0; startInt = static_cast(floor(start.value())); startFrac = start.value() - startInt; } if (end) { isEndNegative = end.value() < 0; endInt = static_cast(floor(end.value())); endFrac = end.value() - endInt; } // Only start defined - match numbers >= start if (start && !end) { std::string startIntStr = FormatFloat(start.value(), precision); parts.push_back(startIntStr); // fractional parts > startFrac with same integer part (for positive) // fractional parts < startFrac with same integer part (for negative) if (startFrac > 0.0) { size_t dotPos = startIntStr.find('.'); if (dotPos != std::string::npos) { std::string intPartStr = startIntStr.substr(0, dotPos); std::string fracPartStr = startIntStr.substr(dotPos + 1); if (!fracPartStr.empty()) { for (size_t i = 0; i < fracPartStr.length(); i++) { if (i == 0) { if (isStartNegative) { for (char d = '0'; d < fracPartStr[0]; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } else { for (char d = fracPartStr[0] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } } else { std::string prefix = fracPartStr.substr(0, i); if (isStartNegative) { if (fracPartStr[i] > '0') { for (char d = '0'; d < fracPartStr[i]; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } else { for (char d = fracPartStr[i] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } } } } } // For all integers > startInt if (startInt < INT_MAX - 1) { std::string intRangeRegex = GenerateRangeRegex(startInt + 1, std::nullopt); intRangeRegex = intRangeRegex.substr(1, intRangeRegex.length() - 2); parts.push_back(intRangeRegex + "(\\.\\d{1," + std::to_string(precision) + "})?"); } } // Only end defined - match numbers <= end else if (!start && end) { std::string endIntStr = FormatFloat(end.value(), precision); parts.push_back(endIntStr); // fractional parts < endFrac with same integer part (for positive) // fractional parts > endFrac with same integer part (for negative) if (endFrac > 0.0) { size_t dotPos = endIntStr.find('.'); if (dotPos != std::string::npos) { std::string intPartStr = endIntStr.substr(0, dotPos); std::string fracPartStr = endIntStr.substr(dotPos + 1); if (!fracPartStr.empty()) { for (size_t i = 0; i < fracPartStr.length(); i++) { if (i == 0) { if (isEndNegative) { for (char d = fracPartStr[0] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } else { for (char d = '0'; d < fracPartStr[0]; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } } else { if (isEndNegative) { std::string prefix = fracPartStr.substr(0, i); for (char d = fracPartStr[i] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } else if (fracPartStr[i] > '0') { std::string prefix = fracPartStr.substr(0, i); for (char d = '0'; d < fracPartStr[i]; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } } } } } // For all integers < endInt if (endInt > INT_MIN + 1) { std::string intRangeRegex = GenerateRangeRegex(std::nullopt, endInt - 1); intRangeRegex = intRangeRegex.substr(1, intRangeRegex.length() - 2); parts.push_back(intRangeRegex + "(\\.\\d{1," + std::to_string(precision) + "})?"); } } // start and end both defined else if (start && end) { // same integer part if (startInt == endInt) { if (startFrac == 0.0 && endFrac == 0.0) { parts.push_back(std::to_string(startInt)); } else { std::string startStr = FormatFloat(start.value(), precision); parts.push_back(startStr); std::string endStr = FormatFloat(end.value(), precision); if (startStr != endStr) { parts.push_back(endStr); } if (startFrac < endFrac) { size_t startDotPos = startStr.find('.'); size_t endDotPos = endStr.find('.'); if (startDotPos != std::string::npos && endDotPos != std::string::npos) { std::string intPart = startStr.substr(0, startDotPos); std::string startFracPart = startStr.substr(startDotPos + 1); std::string endFracPart = endStr.substr(endDotPos + 1); size_t diffPos = 0; size_t minLength = std::min(startFracPart.length(), endFracPart.length()); while (diffPos < minLength && startFracPart[diffPos] == endFracPart[diffPos]) { diffPos++; } if (diffPos < minLength) { char startDigit = startFracPart[diffPos]; char endDigit = endFracPart[diffPos]; if (endDigit > startDigit + 1) { std::string prefix = startFracPart.substr(0, diffPos); for (char d = startDigit + 1; d < endDigit; d++) { parts.push_back( intPart + "\\." + prefix + d + "\\d{0," + std::to_string(precision - diffPos - 1) + "}" ); } } if (diffPos + 1 < startFracPart.length()) { std::string prefix = startFracPart.substr(0, diffPos + 1); for (size_t i = diffPos + 1; i < startFracPart.length(); i++) { std::string currentPrefix = startFracPart.substr(0, i); char currentDigit = startFracPart[i]; for (char d = currentDigit + 1; d <= '9'; d++) { parts.push_back( intPart + "\\." + currentPrefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } if (diffPos + 1 < endFracPart.length()) { std::string prefix = endFracPart.substr(0, diffPos + 1); for (size_t i = diffPos + 1; i < endFracPart.length(); i++) { if (endFracPart[i] > '0') { std::string currentPrefix = endFracPart.substr(0, i); char currentDigit = endFracPart[i]; for (char d = '0'; d < currentDigit; d++) { parts.push_back( intPart + "\\." + currentPrefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } } } } } } } // Different integer parts else { std::string startStr = FormatFloat(start.value(), precision); parts.push_back(startStr); std::string endStr = FormatFloat(end.value(), precision); if (startStr != endStr) { parts.push_back(endStr); } if (endInt > startInt + 1) { std::string intRangeRegex = GenerateRangeRegex(startInt + 1, endInt - 1); intRangeRegex = intRangeRegex.substr(1, intRangeRegex.length() - 2); parts.push_back(intRangeRegex + "(\\.\\d{1," + std::to_string(precision) + "})?"); } if (startFrac > 0.0) { size_t dotPos = startStr.find('.'); if (dotPos != std::string::npos) { std::string intPartStr = startStr.substr(0, dotPos); std::string fracPartStr = startStr.substr(dotPos + 1); if (!fracPartStr.empty()) { for (size_t i = 0; i < fracPartStr.length(); i++) { if (i == 0) { if (isStartNegative) { for (char d = '0'; d < fracPartStr[0]; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } else { for (char d = fracPartStr[0] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } } else { std::string prefix = fracPartStr.substr(0, i); if (isStartNegative) { if (fracPartStr[i] > '0') { for (char d = '0'; d < fracPartStr[i]; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } else { for (char d = fracPartStr[i] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } } } } } else { parts.push_back(std::to_string(startInt) + "\\.\\d{1," + std::to_string(precision) + "}"); } if (endFrac > 0.0) { size_t dotPos = endStr.find('.'); if (dotPos != std::string::npos) { std::string intPartStr = endStr.substr(0, dotPos); std::string fracPartStr = endStr.substr(dotPos + 1); if (!fracPartStr.empty()) { for (size_t i = 0; i < fracPartStr.length(); i++) { if (i == 0) { if (isEndNegative) { for (char d = fracPartStr[0] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } else { for (char d = '0'; d < fracPartStr[0]; d++) { parts.push_back( intPartStr + "\\." + d + "\\d{0," + std::to_string(precision - 1) + "}" ); } } } else { if (isEndNegative) { std::string prefix = fracPartStr.substr(0, i); for (char d = fracPartStr[i] + 1; d <= '9'; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } else if (fracPartStr[i] > '0') { std::string prefix = fracPartStr.substr(0, i); for (char d = '0'; d < fracPartStr[i]; d++) { parts.push_back( intPartStr + "\\." + prefix + d + "\\d{0," + std::to_string(precision - i - 1) + "}" ); } } } } } } } else { parts.push_back(std::to_string(endInt) + "\\.\\d{1," + std::to_string(precision) + "}"); } } } std::ostringstream result; result << "^("; for (size_t i = 0; i < parts.size(); ++i) { if (i > 0) { result << "|"; } result << parts[i]; } result << ")$"; return result.str(); } std::string JSONSchemaConverter::VisitInteger( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("type")); XGRAMMAR_CHECK(schema.at("type").get() == "integer"); WarnUnsupportedKeywords( schema, { "multipleOf", } ); std::string range_regex = ""; if (schema.count("minimum") || schema.count("maximum") || schema.count("exclusiveMinimum") || schema.count("exclusiveMaximum")) { std::optional start, end; if (schema.count("minimum")) { XGRAMMAR_CHECK(schema.at("minimum").is() || schema.at("minimum").is()) << "minimum must be a number"; double start_double = schema.at("minimum").get(); XGRAMMAR_CHECK(start_double == static_cast(start_double)) << "minimum must be an integer"; start = static_cast(start_double); } if (schema.count("exclusiveMinimum")) { XGRAMMAR_CHECK( schema.at("exclusiveMinimum").is() || schema.at("exclusiveMinimum").is() ) << "exclusiveMinimum must be a number"; double start_double = schema.at("exclusiveMinimum").get(); XGRAMMAR_CHECK(start_double == static_cast(start_double)) << "exclusiveMinimum must be an integer"; start = static_cast(start_double); } if (schema.count("maximum")) { XGRAMMAR_CHECK(schema.at("maximum").is() || schema.at("maximum").is()) << "maximum must be a number"; double end_double = schema.at("maximum").get(); XGRAMMAR_CHECK(end_double == static_cast(end_double)) << "maximum must be an integer"; end = static_cast(end_double); } if (schema.count("exclusiveMaximum")) { XGRAMMAR_CHECK( schema.at("exclusiveMaximum").is() || schema.at("exclusiveMaximum").is() ) << "exclusiveMaximum must be a number"; double end_double = schema.at("exclusiveMaximum").get(); XGRAMMAR_CHECK(end_double == static_cast(end_double)) << "exclusiveMaximum must be an integer"; end = static_cast(end_double); } XGRAMMAR_CHECK(!(start && end) || *start <= *end) << "Invalid range, start value greater than end value"; range_regex = GenerateRangeRegex(start, end); } if (!range_regex.empty()) { std::string converted_regex = RegexToEBNF(range_regex, false); return converted_regex; // not " " for numbers } return "(\"0\" | \"-\"? [1-9] [0-9]*)"; } std::string JSONSchemaConverter::VisitNumber( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("type")); XGRAMMAR_CHECK(schema.at("type").get() == "number"); WarnUnsupportedKeywords( schema, { "multipleOf", } ); std::string range_regex = ""; if (schema.count("minimum") || schema.count("maximum") || schema.count("exclusiveMinimum") || schema.count("exclusiveMaximum")) { std::optional start, end; if (schema.count("minimum")) { XGRAMMAR_CHECK(schema.at("minimum").is() || schema.at("minimum").is()) << "minimum must be a number"; start = schema.at("minimum").get(); } if (schema.count("exclusiveMinimum")) { XGRAMMAR_CHECK( schema.at("exclusiveMinimum").is() || schema.at("exclusiveMinimum").is() ) << "exclusiveMinimum must be a number"; start = schema.at("exclusiveMinimum").get(); } if (schema.count("maximum")) { XGRAMMAR_CHECK(schema.at("maximum").is() || schema.at("maximum").is()) << "maximum must be a number"; end = schema.at("maximum").get(); } if (schema.count("exclusiveMaximum")) { XGRAMMAR_CHECK( schema.at("exclusiveMaximum").is() || schema.at("exclusiveMaximum").is() ) << "exclusiveMaximum must be a number"; end = schema.at("exclusiveMaximum").get(); } XGRAMMAR_CHECK(!(start && end) || *start <= *end) << "Invalid range, start value greater than end value"; range_regex = GenerateFloatRangeRegex(start, end); } if (!range_regex.empty()) { std::string converted_regex = RegexToEBNF(range_regex, false); return converted_regex; } return "(\"0\" | \"-\"? [1-9] [0-9]*) (\".\" [0-9]+)? ([eE] [+-]? [0-9]+)?"; } std::string JSONSchemaConverter::VisitString( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("type")); XGRAMMAR_CHECK(schema.at("type").get() == "string"); if (schema.count("format")) { std::string format = schema.at("format").get(); if (format == "email") { // refer to RFC 5321 and RFC 5322, but skipping `address-literal` at // RFC 5321 section 4.1.2 currently std::string atext = "[\\w!#$%&'*+/=?^`{|}~-]"; std::string dot_string = "(" + atext + "+(\\." + atext + "+)*)"; std::string quoted_string = "\\\\\"(\\\\[\\x20-\\x7E]|[\\x20\\x21\\x23-\\x5B\\x5D-\\x7E])*\\\\\""; std::string domain = "([A-Za-z0-9]([\\-A-Za-z0-9]*[A-Za-z0-9])?)((\\.[A-Za-z0-9][\\-A-Za-z0-9]*[A-Za-z0-9])*)"; std::string email_regex_pattern = "^(" + dot_string + "|" + quoted_string + ")@" + domain + "$"; std::string email_ebnf = RegexToEBNF(email_regex_pattern, false); return "\"\\\"\" " + email_ebnf + " \"\\\"\""; } if (format == "date") { // refer to RFC 3339, section 5.6 std::string date_regex_pattern = "^(\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2]\\d|3[01]))$"; std::string date_ebnf = RegexToEBNF(date_regex_pattern, false); return "\"\\\"\" " + date_ebnf + " \"\\\"\""; } if (format == "time") { // refer to RFC 3339, section 5.6 std::string time_regex_pattern = "^([01]\\d|2[0-3]):[0-5]\\d:([0-5]\\d|60)(\\.\\d+)?(Z|[+-]([01]\\d|2[0-3]):[0-5]\\d)$"; std::string time_ebnf = RegexToEBNF(time_regex_pattern, false); return "\"\\\"\" " + time_ebnf + " \"\\\"\""; } if (format == "date-time") { // refer to RFC 3339, section 5.6 std::string date_time_regex_pattern = "^(\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2]\\d|3[01]))T([01]\\d|2[0-3]):([0-5]\\d|60):[" "0-5]\\d(\\.\\d+)?(Z|[+-]([01]\\d|2[0-3]):[0-5]\\d)$"; std::string date_time_ebnf = RegexToEBNF(date_time_regex_pattern, false); return "\"\\\"\" " + date_time_ebnf + " \"\\\"\""; } if (format == "duration") { // refer to RFC 3339, Appendix A std::string duration_regex_pattern = "^P((\\d+D|\\d+M(\\d+D)?|\\d+Y(\\d+M(\\d+D)?)?)(T(\\d+S|\\d+M(\\d+S)?|\\d+H(\\d+M(\\d+S)?" ")?))?|T(\\d+S|\\d+M(\\d+S)?|\\d+H(\\d+M(\\d+S)?)?)|\\d+W)$"; std::string duration_ebnf = RegexToEBNF(duration_regex_pattern, false); return "\"\\\"\" " + duration_ebnf + " \"\\\"\""; } if (format == "ipv4") { // refer to RFC 2673, section 3.2 std::string decbyte = "(25[0-5]|2[0-4]\\d|[0-1]?\\d?\\d)"; std::string ipv4_regex_pattern = "^(" + decbyte + "\\.){3}" + decbyte + "$"; std::string ipv4_ebnf = RegexToEBNF(ipv4_regex_pattern, false); return "\"\\\"\" " + ipv4_ebnf + " \"\\\"\""; } if (format == "ipv6") { // refer to RFC 3986, section 3.3.2 std::string ipv6_regex_pattern = "(" "([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|" // 1:2:3:4:5:6:7:8 "([0-9a-fA-F]{1,4}:){1,7}:|" // 1:: 1:2:3:4:5:6:7:: "([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|" // 1::8 1:2:3:4:5:6::8 // 1:2:3:4:5:6::8 "([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|" // 1::7:8 1:2:3:4:5::7:8 // 1:2:3:4:5::8 "([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|" // 1::6:7:8 1:2:3:4::6:7:8 // 1:2:3:4::8 "([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|" // 1::5:6:7:8 1:2:3::5:6:7:8 // 1:2:3::8 "([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|" // 1::4:5:6:7:8 1:2::4:5:6:7:8 // 1:2::8 "[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|" // 1::3:4:5:6:7:8 1::3:4:5:6:7:8 1::8 ":((:[0-9a-fA-F]{1,4}){1,7}|:)|" // ::2:3:4:5:6:7:8 ::2:3:4:5:6:7:8 ::8 :: "::(ffff(:0{1,4}){0,1}:){0,1}" "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}" "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|" // ::255.255.255.255 ::ffff:255.255.255.255 // ::ffff:0:255.255.255.255 (IPv4-mapped // IPv6 addresses and IPv4-translated // addresses) "([0-9a-fA-F]{1,4}:){1,4}:" "((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}" "(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])" // 2001:db8:3:4::192.0.2.33 // 64:ff9b::192.0.2.33 (IPv4-Embedded IPv6 // Address) ")"; std::string ipv6_ebnf = RegexToEBNF(ipv6_regex_pattern, false); return "\"\\\"\" " + ipv6_ebnf + " \"\\\"\""; } if (format == "hostname") { // refer to RFC 1123, section 2.1 std::string hostname_regex_pattern = "^([a-z0-9]([a-z0-9-]*[a-z0-9])?)(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$"; std::string hostname_ebnf = RegexToEBNF(hostname_regex_pattern, false); return "\"\\\"\" " + hostname_ebnf + " \"\\\"\""; } if (format == "uuid") { // refer to RFC 4122, section 3 std::string uuid_regex_pattern = "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$"; std::string uuid_ebnf = RegexToEBNF(uuid_regex_pattern, false); return "\"\\\"\" " + uuid_ebnf + " \"\\\"\""; } if (format == "uri") { // refer to RFC 3986, Appendix A, but skipping IP-literal and IPv4address currently std::string schema = "[a-zA-Z][a-zA-Z+\\.-]*"; std::string pchar = "([\\w\\.~!$&'()*+,;=:@-]|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string query_fragment_char = "([\\w\\.~!$&'()*+,;=:@/\\?-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string query = "(\\?" + query_fragment_char + ")?"; std::string fragment = "(#" + query_fragment_char + ")?"; std::string path_abempty = "(/" + pchar + "*)*"; std::string path_absolute_rootless_empty = "/?(" + pchar + "+(/" + pchar + "*)*)?"; std::string userinfo = "([\\w\\.~!$&'()*+,;=:-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string host = "([\\w\\.~!$&'()*+,;=-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string authority = "(" + userinfo + "@)?" + host + "(:\\d*)?"; std::string hier_part = "(//" + authority + path_abempty + "|" + path_absolute_rootless_empty + ")"; std::string uri_regex_pattern = "^" + schema + ":" + hier_part + query + fragment + "$"; std::string uri_ebnf = RegexToEBNF(uri_regex_pattern, false); return "\"\\\"\" " + uri_ebnf + " \"\\\"\""; } if (format == "uri-reference") { // refer to RFC 3986, Appendix A, but skipping IP-literal and IPv4address currently std::string pchar = "([\\w\\.~!$&'()*+,;=:@-]|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string query_fragment_char = "([\\w\\.~!$&'()*+,;=:@/\\?-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string query = "(\\?" + query_fragment_char + ")?"; std::string fragment = "(#" + query_fragment_char + ")?"; std::string path_abempty = "(/" + pchar + "*)*"; std::string path_absolute = "/(" + pchar + "+(/" + pchar + "*)*)?"; std::string segment_nz_nc = "([\\w\\.~!$&'()*+,;=@-]|%[0-9A-Fa-f][0-9A-Fa-f])+"; std::string path_noscheme = segment_nz_nc + "(/" + pchar + "*)*"; std::string userinfo = "([\\w\\.~!$&'()*+,;=:-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string host = "([\\w\\.~!$&'()*+,;=-]|%[0-9A-Fa-f][0-9A-Fa-f])*"; std::string authority = "(" + userinfo + "@)?" + host + "(:\\d*)?"; std::string relative_part = "(//" + authority + path_abempty + "|" + path_absolute + "|" + path_noscheme + ")?"; std::string uri_reference_regex_pattern = "^" + relative_part + query + fragment + "$"; std::string uri_reference_ebnf = RegexToEBNF(uri_reference_regex_pattern, false); return "\"\\\"\" " + uri_reference_ebnf + " \"\\\"\""; } if (format == "uri-template") { // refer to RFC 6570, section 2 std::string literals = "([\\x21\\x23-\\x24\\x26\\x28-\\x3B\\x3D\\x3F-\\x5B\\x5D\\x5F\\x61-\\x7A\\x7E]" "|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string op = "[+#\\./;\\?&=,!@|]"; std::string varchar = "(\\w|%[0-9A-Fa-f][0-9A-Fa-f])"; std::string varname = varchar + "(\\.?" + varchar + ")*"; std::string varspec = varname + "(:[1-9]\\d?\\d?\\d?|\\*)?"; std::string variable_list = varspec + "(," + varspec + ")*"; std::string expression = "\\{(" + op + ")?" + variable_list + "\\}"; std::string uri_template_regex_pattern = "^(" + literals + "|" + expression + ")*$"; std::string uri_template_ebnf = RegexToEBNF(uri_template_regex_pattern, false); return "\"\\\"\" " + uri_template_ebnf + " \"\\\"\""; } if (format == "json-pointer") { // refer to RFC 6901, section 3 std::string json_pointer_regex_pattern = "^(/([\\x00-\\x2E]|[\\x30-\\x7D]|[\\x7F-\\U0010FFFF]|~[01])*)*$"; std::string json_pointer_ebnf = RegexToEBNF(json_pointer_regex_pattern, false); return "\"\\\"\" " + json_pointer_ebnf + " \"\\\"\""; } if (format == "relative-json-pointer") { // refer to draft-handrews-relative-json-pointer-01, section 3 std::string relative_json_pointer_regex_pattern = "^(0|[1-9][0-9]*)(#|(/([\\x00-\\x2E]|[\\x30-\\x7D]|[\\x7F-\\U0010FFFF]|~[01])*)*)$"; std::string relative_json_pointer_ebnf = RegexToEBNF(relative_json_pointer_regex_pattern, false); return "\"\\\"\" " + relative_json_pointer_ebnf + " \"\\\"\""; } } if (schema.count("pattern")) { if (schema.count("minLength") || schema.count("maxLength") || schema.count("format")) { XGRAMMAR_LOG(WARNING) << "Specifying pattern and minLength/maxLength/format is not " << "supported yet, ignoring minLength/maxLength/format"; } std::string regex_pattern = schema.at("pattern").get(); std::string converted_regex = RegexToEBNF(regex_pattern, false); return "\"\\\"\" " + converted_regex + " \"\\\"\""; } if (schema.count("minLength") || schema.count("maxLength")) { int min_length = schema.count("minLength") ? schema.at("minLength").get() : 0; int max_length = schema.count("maxLength") ? schema.at("maxLength").get() : -1; XGRAMMAR_CHECK(max_length == -1 || min_length <= max_length) << "In string schema, minLength " << min_length << " is greater than " << "maxLength " << max_length; std::string range_part = "{" + std::to_string(min_length) + "," + (max_length == -1 ? "" : std::to_string(max_length)) + "}"; return "\"\\\"\" " + std::string("[^\"\\\\\\r\\n]") + range_part + " \"\\\"\""; } return "[\"] " + kBasicStringSub; } std::string JSONSchemaConverter::VisitBoolean( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("type")); XGRAMMAR_CHECK(schema.at("type").get() == "boolean"); return "\"true\" | \"false\""; } std::string JSONSchemaConverter::VisitNull( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.count("type")); XGRAMMAR_CHECK(schema.at("type").get() == "null"); return "\"null\""; } Result JSONSchemaConverter::ParseArraySchema( const picojson::object& schema ) { XGRAMMAR_DCHECK( (schema.count("type") && schema.at("type").get() == "array") || schema.count("prefixItems") || schema.count("items") || schema.count("unevaluatedItems") ); WarnUnsupportedKeywords(schema, {"uniqueItems", "contains", "minContains", "maxContains"}); std::vector prefix_item_schemas; bool allow_additional_items = true; picojson::value additional_item_schema; int min_items = 0; int max_items = -1; if (schema.count("prefixItems")) { if (!schema.at("prefixItems").is()) { return Result::Err( std::make_shared("prefixItems must be an array") ); } prefix_item_schemas = schema.at("prefixItems").get(); for (const auto& item : prefix_item_schemas) { if (item.is()) { if (!item.get()) { return Result::Err( std::make_shared("prefixItems contains false") ); } } else if (!item.is()) { return Result::Err(std::make_shared( "prefixItems must be an array of objects or booleans" )); } } } if (schema.count("items")) { auto items_value = schema.at("items"); if (!items_value.is() && !items_value.is()) { return Result::Err( std::make_shared("items must be a boolean or an object") ); } if (items_value.is() && !items_value.get()) { allow_additional_items = false; } else { allow_additional_items = true; additional_item_schema = items_value; } } else if (schema.count("unevaluatedItems")) { auto unevaluated_items_value = schema.at("unevaluatedItems"); if (!unevaluated_items_value.is() && !unevaluated_items_value.is()) { return Result::Err( std::make_shared("unevaluatedItems must be a boolean or an object") ); } if (unevaluated_items_value.is() && !unevaluated_items_value.get()) { allow_additional_items = false; } else { allow_additional_items = true; additional_item_schema = unevaluated_items_value; } } else if (!strict_mode_) { allow_additional_items = true; additional_item_schema = picojson::value(true); } else { allow_additional_items = false; } if (schema.count("minItems")) { if (!schema.at("minItems").is()) { return Result::Err( std::make_shared("minItems must be an integer") ); } min_items = std::max(0, static_cast(schema.at("minItems").get())); } if (schema.count("minContains")) { if (!schema.at("minContains").is()) { return Result::Err( std::make_shared("minContains must be an integer") ); } min_items = std::max(min_items, static_cast(schema.at("minContains").get())); } if (schema.count("maxItems")) { if (!schema.at("maxItems").is() || schema.at("maxItems").get() < 0) { return Result::Err( std::make_shared("maxItems must be a non-negative integer") ); } max_items = schema.at("maxItems").get(); } // Check if the schema is unsatisfiable if (max_items != -1 && min_items > max_items) { return Result::Err(std::make_shared( "minItems is greater than maxItems: " + std::to_string(min_items) + " > " + std::to_string(max_items) )); } if (max_items != -1 && max_items < static_cast(prefix_item_schemas.size())) { return Result::Err(std::make_shared( "maxItems is less than the number of prefixItems: " + std::to_string(max_items) + " < " + std::to_string(prefix_item_schemas.size()) )); } if (!allow_additional_items) { // [len, len] must be in [min, max] if (static_cast(prefix_item_schemas.size()) < min_items) { return Result::Err(std::make_shared( "minItems is greater than the number of prefixItems, but additional items are not " "allowed: " + std::to_string(min_items) + " > " + std::to_string(prefix_item_schemas.size()) )); } if (max_items != -1 && static_cast(prefix_item_schemas.size()) > max_items) { return Result::Err(std::make_shared( "maxItems is less than the number of prefixItems, but additional items are not " "allowed: " + std::to_string(max_items) + " < " + std::to_string(prefix_item_schemas.size()) )); } } return Result::Ok(ArraySpec{ prefix_item_schemas, allow_additional_items, additional_item_schema, min_items, max_items }); } std::string JSONSchemaConverter::VisitArray( const picojson::object& schema, const std::string& rule_name ) { auto array_spec_result = ParseArraySchema(schema); if (array_spec_result.IsErr()) { XGRAMMAR_LOG(FATAL) << array_spec_result.UnwrapErr()->what(); } auto array_spec = std::move(array_spec_result).Unwrap(); indentManager_->StartIndent(); auto start_separator = indentManager_->StartSeparator(); auto mid_separator = indentManager_->MiddleSeparator(); auto end_separator = indentManager_->EndSeparator(); auto empty_separator = indentManager_->EmptySeparator(); std::vector item_rule_names; std::string additional_rule_name; // 1. Handle prefix items if (array_spec.prefix_item_schemas.size() > 0) { for (int i = 0; i < static_cast(array_spec.prefix_item_schemas.size()); ++i) { XGRAMMAR_DCHECK( array_spec.prefix_item_schemas[i].is() || array_spec.prefix_item_schemas[i].is() ); item_rule_names.push_back(CreateRuleFromSchema( array_spec.prefix_item_schemas[i], rule_name + "_item_" + std::to_string(i) )); } } // 2. Handle additional items if (array_spec.allow_additional_items) { additional_rule_name = CreateRuleFromSchema(array_spec.additional_item_schema, rule_name + "_additional"); } indentManager_->EndIndent(); // 3. Construct the result with given format // clang-format off /* * prefix empty, additional items not allowed: [empty_separator] * prefix empty, additional items allowed: * if min == 0, max == 0: * [empty_separator] * if min == 0, max > 0: * ([start_separator additional_rule_name (mid_separator additional_rule_name){0, max - 1}) end_separator] | [empty_separator] * if min > 0: * ([start_separator additional_rule_name (mid_separator additional_rule_name){min - 1, max - 1}) end_separator] * prefix non-empty, additional items not allowed: [start_separator item0 mid_separator item1 end_separator] * prefix non-empty, additional items allowed: * [start_separator item0 mid_separator item1 (mid_separator additional_rule_name){max(0, min - len(prefix)), max - len(prefix)} end_separator] */ // clang-format on std::string result; const std::string& left_bracket = EBNFScriptCreator::Str("["); const std::string& right_bracket = EBNFScriptCreator::Str("]"); if (array_spec.prefix_item_schemas.empty()) { auto empty_part = EBNFScriptCreator::Concat({left_bracket, empty_separator, right_bracket}); if (!array_spec.allow_additional_items) { return empty_part; } else if (array_spec.min_items == 0 && array_spec.max_items == 0) { return empty_part; } else if (array_spec.min_items == 0 && array_spec.max_items != 0) { return EBNFScriptCreator::Or( {EBNFScriptCreator::Concat( {left_bracket, start_separator, additional_rule_name, EBNFScriptCreator::Repeat( EBNFScriptCreator::Concat({mid_separator, additional_rule_name}), 0, array_spec.max_items == -1 ? -1 : array_spec.max_items - 1 ), end_separator, right_bracket} ), empty_part} ); } else { XGRAMMAR_DCHECK(array_spec.min_items > 0); return EBNFScriptCreator::Concat( {left_bracket, start_separator, additional_rule_name, EBNFScriptCreator::Repeat( EBNFScriptCreator::Concat({mid_separator, additional_rule_name}), array_spec.min_items - 1, array_spec.max_items == -1 ? -1 : array_spec.max_items - 1 ), end_separator, right_bracket} ); } } else { std::vector prefix_part; for (int i = 0; i < static_cast(item_rule_names.size()); ++i) { if (i > 0) { prefix_part.push_back(mid_separator); } prefix_part.push_back(item_rule_names[i]); } auto prefix_part_str = EBNFScriptCreator::Concat(prefix_part); if (!array_spec.allow_additional_items) { return EBNFScriptCreator::Concat( {left_bracket, start_separator, prefix_part_str, end_separator, right_bracket} ); } else { int min_items = std::max(0, array_spec.min_items - static_cast(item_rule_names.size())); return EBNFScriptCreator::Concat( {left_bracket, start_separator, prefix_part_str, EBNFScriptCreator::Repeat( EBNFScriptCreator::Concat({mid_separator, additional_rule_name}), min_items, array_spec.max_items == -1 ? -1 : array_spec.max_items - static_cast(item_rule_names.size()) ), end_separator, right_bracket} ); } } } std::string JSONSchemaConverter::GetPropertyPattern( const std::string& prop_name, const picojson::value& prop_schema, const std::string& rule_name, int idx ) { // the outer quote is for the string in EBNF grammar, and the inner quote is for // the string in JSON std::string key = "\"\\\"" + prop_name + "\\\"\""; std::string value = CreateRuleFromSchema(prop_schema, rule_name + "_prop_" + std::to_string(idx)); return key + " " + colon_pattern_ + " " + value; } std::string JSONSchemaConverter::GetOtherPropertyPattern( const std::string& key_pattern, const picojson::value& prop_schema, const std::string& rule_name, const std::string& rule_name_suffix ) { std::string value = CreateRuleFromSchema(prop_schema, rule_name + "_" + rule_name_suffix); return key_pattern + " " + colon_pattern_ + " " + value; } std::string JSONSchemaConverter::GetPartialRuleForPropertiesAllOptional( const std::vector>& properties, const picojson::value& additional, const std::string& rule_name, const std::string& additional_suffix ) { XGRAMMAR_CHECK(properties.size() >= 1); std::string first_sep = NextSeparator(); std::string mid_sep = NextSeparator(); std::string last_sep = NextSeparator(true); std::string res = ""; std::vector prop_patterns; int idx = 0; for (const auto& [prop_name, prop_schema] : properties) { prop_patterns.push_back(GetPropertyPattern(prop_name, prop_schema, rule_name, idx)); ++idx; } std::vector rule_names(properties.size(), ""); // construct the last rule std::string additional_prop_pattern; if (!additional.is() || additional.get()) { additional_prop_pattern = GetOtherPropertyPattern(kBasicString, additional, rule_name, additional_suffix); std::string last_rule_body = "(" + mid_sep + " " + additional_prop_pattern + ")*"; std::string last_rule_name = rule_name + "_part_" + std::to_string(static_cast(properties.size()) - 1); last_rule_name = ebnf_script_creator_.AddRule(last_rule_name, last_rule_body); rule_names.back() = last_rule_name; } else { rule_names.back() = "\"\""; } // construct 0~(len(properties) - 2) rules for (int i = properties.size() - 2; i >= 0; --i) { const std::string& prop_pattern = prop_patterns[i + 1]; const std::string& last_rule_name = rule_names[i + 1]; std::string cur_rule_body = last_rule_name + " | " + mid_sep + " " + prop_pattern + " " + last_rule_name; std::string cur_rule_name = rule_name + "_part_" + std::to_string(i); cur_rule_name = ebnf_script_creator_.AddRule(cur_rule_name, cur_rule_body); rule_names[i] = cur_rule_name; } // construct the root rule for (int i = 0; i < static_cast(properties.size()); ++i) { if (i != 0) { res += " | "; } res += "(" + prop_patterns[i] + " " + rule_names[i] + ")"; } if (!additional.is() || additional.get()) { res += " | " + additional_prop_pattern + " " + rule_names.back(); } // add separators and the empty string option res = first_sep + " (" + res + ") " + last_sep; return res; } std::string JSONSchemaConverter::GetPartialRuleForPropertiesContainRequired( const std::vector>& properties, const std::unordered_set& required, const std::string& rule_name ) { // Find the index of the first required property int first_required_idx = properties.size(); for (int i = 0; i < static_cast(properties.size()); ++i) { if (required.count(properties[i].first)) { first_required_idx = i; break; } } XGRAMMAR_CHECK(first_required_idx < static_cast(properties.size())); std::string res = NextSeparator(); // Handle the properties before the first required property for (int i = 0; i < first_required_idx; ++i) { const auto& [prop_name, prop_schema] = properties[i]; XGRAMMAR_CHECK(!prop_schema.is() || prop_schema.get()); std::string property_pattern = GetPropertyPattern(prop_name, prop_schema, rule_name, i); res += " (" + property_pattern + " " + NextSeparator() + ")?"; } // Handle the first required property const auto& [prop_name, prop_schema] = properties[first_required_idx]; std::string property_pattern = GetPropertyPattern(prop_name, prop_schema, rule_name, first_required_idx); res += " " + property_pattern; // Handle the properties after the first required property for (int i = first_required_idx + 1; i < static_cast(properties.size()); ++i) { const auto& [prop_name, prop_schema] = properties[i]; XGRAMMAR_CHECK(!prop_schema.is() || prop_schema.get()); std::string property_pattern = GetPropertyPattern(prop_name, prop_schema, rule_name, i); if (required.count(prop_name)) { res += " " + NextSeparator() + " " + property_pattern; } else { res += " (" + NextSeparator() + " " + property_pattern + ")?"; } } return res; } std::string JSONSchemaConverter::VisitObject( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK( (schema.count("type") && schema.at("type").get() == "object") || schema.count("properties") || schema.count("additionalProperties") || schema.count("unevaluatedProperties") ); WarnUnsupportedKeywords( schema, { "patternProperties", "minProperties", "maxProperties", "propertyNames", } ); std::string result = "\"{\""; // could_be_empty will be set to True when the rule could be "{}". We will handle this case at // last, and handle non-empty cases before that. bool could_be_empty = false; indentManager_->StartIndent(); // 1. Handle properties std::vector> properties; if (schema.count("properties")) { XGRAMMAR_CHECK(schema.at("properties").is()) << "properties must be an object"; auto properties_obj = schema.at("properties").get(); for (const auto& key : properties_obj.ordered_keys()) { properties.push_back({key, properties_obj.at(key)}); } } std::unordered_set required; if (schema.count("required")) { XGRAMMAR_CHECK(schema.at("required").is()) << "required must be an array"; for (const auto& required_prop : schema.at("required").get()) { required.insert(required_prop.get()); } } // 2. Find additional properties picojson::value additional_property = picojson::value(false); std::string additional_suffix = ""; if (schema.count("additionalProperties") && (!schema.at("additionalProperties").is() || schema.at("additionalProperties").get())) { additional_property = schema.at("additionalProperties"); additional_suffix = "addl"; } if (schema.count("additionalProperties") == 0) { picojson::value unevaluated = schema.count("unevaluatedProperties") ? schema.at("unevaluatedProperties") : picojson::value(!strict_mode_); if (!unevaluated.is() || unevaluated.get()) { additional_property = unevaluated; additional_suffix = "uneval"; } } bool is_all_properties_optional = std::all_of(properties.begin(), properties.end(), [&](const auto& prop) { return required.count(prop.first) == 0; }); if (is_all_properties_optional && properties.size() > 0) { // 3.1 Case 1: properties are defined and all properties are optional result += " " + GetPartialRuleForPropertiesAllOptional( properties, additional_property, rule_name, additional_suffix ); could_be_empty = true; } else if (properties.size() > 0) { // 3.2 Case 2: properties are defined and some properties are required result += " " + GetPartialRuleForPropertiesContainRequired(properties, required, rule_name); if (!additional_property.is() || additional_property.get()) { std::string other_property_pattern = GetOtherPropertyPattern(kBasicString, additional_property, rule_name, additional_suffix); result += " (" + NextSeparator() + " " + other_property_pattern + ")*"; } result += " " + NextSeparator(true); } else if (!additional_property.is() || additional_property.get()) { // 3.3 Case 3: no properties are defined and additional properties are allowed std::string other_property_pattern = GetOtherPropertyPattern(kBasicString, additional_property, rule_name, additional_suffix); result += " " + NextSeparator() + " " + other_property_pattern + " ("; result += NextSeparator() + " " + other_property_pattern + ")* "; result += NextSeparator(true); could_be_empty = true; } indentManager_->EndIndent(); result += " \"}\""; if (could_be_empty) { // result = (result) | {} auto rest = "\"{\" " + std::string(any_whitespace_ ? "[ \\n\\t]* " : "") + "\"}\""; result = "(" + result + ") | " + rest; } return result; } std::string JSONSchemaConverter::VisitTypeArray( const picojson::object& schema, const std::string& rule_name ) { XGRAMMAR_CHECK(schema.at("type").is()); auto type_array = schema.at("type").get(); picojson::object schema_copy = schema; if (type_array.size() == 0) { schema_copy.erase("type"); return VisitSchema(picojson::value(schema_copy), rule_name); } std::string result; for (const auto& type : type_array) { XGRAMMAR_CHECK(type.is()) << "type must be a string or an array of strings, but got " << type; if (!result.empty()) { result += " | "; } schema_copy["type"] = type; result += CreateRuleFromSchema( picojson::value(schema_copy), rule_name + "_" + type.get() ); } return result; } std::string JSONSchemaToEBNF( const std::string& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode ) { picojson::value schema_value; std::string err = picojson::parse(schema_value, schema); XGRAMMAR_CHECK(err.empty()) << "Failed to parse JSON: " << err << ". The JSON string is:" << schema; return JSONSchemaToEBNF(schema_value, any_whitespace, indent, separators, strict_mode); } std::string JSONSchemaToEBNF( const picojson::value& schema, bool any_whitespace, std::optional indent, std::optional> separators, bool strict_mode ) { JSONSchemaConverter converter(schema, any_whitespace, indent, separators, strict_mode); return converter.Convert(); } // Wrapper function for testing std::string GenerateRangeRegex(std::optional start, std::optional end) { return JSONSchemaConverter::GenerateRangeRegex(start, end); } std::string GenerateFloatRangeRegex(std::optional start, std::optional end) { return JSONSchemaConverter::GenerateFloatRangeRegex(start, end, 6); } } // namespace xgrammar xgrammar-0.1.19/cpp/json_schema_converter.h000066400000000000000000000063561500705317600207510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/json_schema_converter.h * \brief Convert a JSON schema string to EBNF grammar string. */ #ifndef XGRAMMAR_JSON_SCHEMA_CONVERTER_H_ #define XGRAMMAR_JSON_SCHEMA_CONVERTER_H_ #include #include #include #include namespace xgrammar { /*! * \brief Convert JSON schema string to EBNF grammar string. * \param schema The JSON schema string. * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be * in one line. Default: 2. * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the * indent is not -1, and {", ", ": "} otherwise. This follows the convention in python * json.dumps(). Default: std::nullopt. * \param strict_mode Whether to use strict mode. In strict * mode, the generated grammar will not allow properties and items that is not specified in the * schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. * * This helps LLM to generate accurate output in the grammar-guided generation with JSON * schema. Default: true. * \returns The EBNF grammar string. */ std::string JSONSchemaToEBNF( const std::string& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true ); /*! * \brief Convert JSON schema string to EBNF grammar string. * \param schema The JSON schema object. * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be * in one line. Default: 2. * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the * indent is not -1, and {", ", ": "} otherwise. This follows the convention in python * json.dumps(). Default: std::nullopt. \param strict_mode Whether to use strict mode. In strict * mode, the generated grammar will not allow properties and items that is not specified in the * schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. * * This helps LLM to generate accurate output in the grammar-guided generation with JSON * schema. Default: true. * \returns The EBNF grammar string. */ std::string JSONSchemaToEBNF( const picojson::value& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true ); /*! * \brief Generate regex pattern for integer/float range. * \param start The start of the range (inclusive). If null assume negative infinity. * \param end The end of the range (inclusive). If null assume infinity. * \returns The regex pattern that matches integers/floats in the given range. */ std::string GenerateRangeRegex(std::optional start, std::optional end); std::string GenerateFloatRangeRegex(std::optional start, std::optional end); } // namespace xgrammar #endif // XGRAMMAR_JSON_SCHEMA_CONVERTER_H_ xgrammar-0.1.19/cpp/nanobind/000077500000000000000000000000001500705317600157765ustar00rootroot00000000000000xgrammar-0.1.19/cpp/nanobind/CMakeLists.txt000066400000000000000000000025121500705317600205360ustar00rootroot00000000000000find_package( Python COMPONENTS Interpreter Development.Module REQUIRED ) find_package(nanobind CONFIG REQUIRED) # Compile this source file seperately. Nanobind suggests to optimize bindings code for size, but # this source file contains mostly core logic. See notes about size optimizations here: # https://nanobind.readthedocs.io/en/latest/api_cmake.html#command:nanobind_add_module add_library(python_methods STATIC) target_sources(python_methods PRIVATE python_methods.cc) target_link_libraries(python_methods PUBLIC xgrammar) # Any code that uses nanobind directly lives here nanobind_add_module(xgrammar_bindings LTO nanobind.cc) target_link_libraries(xgrammar_bindings PRIVATE python_methods) if(DEFINED SKBUILD_PROJECT_NAME) # Building wheel through scikit-build-core set(LIB_OUTPUT_DIRECTORY xgrammar) else() set(LIB_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/python/xgrammar) endif() set_target_properties(xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIRECTORY}) set_target_properties( xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY_DEBUG ${LIB_OUTPUT_DIRECTORY} ) set_target_properties( xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY_RELEASE ${LIB_OUTPUT_DIRECTORY} ) set_target_properties( xgrammar_bindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY_REL_WITH_DEB_INFO ${LIB_OUTPUT_DIRECTORY} ) xgrammar-0.1.19/cpp/nanobind/nanobind.cc000066400000000000000000000235011500705317600200760ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/nanobind/nanobind.cc */ #include #include #include #include #include #include #include #include #include "../grammar_functor.h" #include "../json_schema_converter.h" #include "../regex_converter.h" #include "../testing.h" #include "python_methods.h" namespace nb = nanobind; using namespace xgrammar; std::vector CommonEncodedVocabType( const nb::typed> encoded_vocab ) { std::vector encoded_vocab_strs; encoded_vocab_strs.reserve(encoded_vocab.size()); for (const auto& token : encoded_vocab) { if (nb::bytes result; nb::try_cast(token, result)) { encoded_vocab_strs.emplace_back(result.c_str()); } else if (nb::str result; nb::try_cast(token, result)) { encoded_vocab_strs.emplace_back(result.c_str()); } else { throw nb::type_error("Expected str or bytes for encoded_vocab"); } } return encoded_vocab_strs; } std::vector TokenizerInfo_GetDecodedVocab(const TokenizerInfo& tokenizer) { const auto& decoded_vocab = tokenizer.GetDecodedVocab(); std::vector py_result; py_result.reserve(decoded_vocab.size()); for (const auto& item : decoded_vocab) { py_result.emplace_back(nanobind::bytes(item.c_str())); } return py_result; } NB_MODULE(xgrammar_bindings, m) { auto pyTokenizerInfo = nb::class_(m, "TokenizerInfo"); pyTokenizerInfo .def( "__init__", [](TokenizerInfo* out, const nb::typed> encoded_vocab, int vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space) { new (out) TokenizerInfo{TokenizerInfo_Init( CommonEncodedVocabType(encoded_vocab), vocab_type, vocab_size, std::move(stop_token_ids), add_prefix_space )}; }, nb::arg("encoded_vocab"), nb::arg("vocab_type"), nb::arg("vocab_size").none(), nb::arg("stop_token_ids").none(), nb::arg("add_prefix_space") ) .def_prop_ro("vocab_type", &TokenizerInfo_GetVocabType) .def_prop_ro("vocab_size", &TokenizerInfo::GetVocabSize) .def_prop_ro("add_prefix_space", &TokenizerInfo::GetAddPrefixSpace) .def_prop_ro("decoded_vocab", &TokenizerInfo_GetDecodedVocab) .def_prop_ro("stop_token_ids", &TokenizerInfo::GetStopTokenIds) .def_prop_ro("special_token_ids", &TokenizerInfo::GetSpecialTokenIds) .def("dump_metadata", &TokenizerInfo::DumpMetadata) .def_static( "from_vocab_and_metadata", [](const nb::typed> encoded_vocab, const std::string& metadata) { return TokenizerInfo::FromVocabAndMetadata( CommonEncodedVocabType(encoded_vocab), metadata ); } ) .def_static("_detect_metadata_from_hf", &TokenizerInfo::DetectMetadataFromHF); auto pyGrammar = nb::class_(m, "Grammar"); pyGrammar.def("to_string", &Grammar::ToString) .def_static("from_ebnf", &Grammar::FromEBNF) .def_static( "from_json_schema", &Grammar::FromJSONSchema, nb::arg("schema"), nb::arg("any_whitespace"), nb::arg("indent").none(), nb::arg("separators").none(), nb::arg("strict_mode"), nb::arg("print_converted_ebnf"), nb::call_guard() ) .def_static("from_regex", &Grammar::FromRegex, nb::call_guard()) .def_static( "from_structural_tag", &Grammar_FromStructuralTag, nb::call_guard() ) .def_static("builtin_json_grammar", &Grammar::BuiltinJSONGrammar) .def_static("union", &Grammar::Union, nb::call_guard()) .def_static("concat", &Grammar::Concat, nb::call_guard()); auto pyCompiledGrammar = nb::class_(m, "CompiledGrammar"); pyCompiledGrammar.def_prop_ro("grammar", &CompiledGrammar::GetGrammar) .def_prop_ro("tokenizer_info", &CompiledGrammar::GetTokenizerInfo) .def_prop_ro("memory_size_bytes", &CompiledGrammar::MemorySizeBytes); auto pyGrammarCompiler = nb::class_(m, "GrammarCompiler"); pyGrammarCompiler.def(nb::init()) .def( "compile_json_schema", &GrammarCompiler::CompileJSONSchema, nb::call_guard(), nb::arg("schema"), nb::arg("any_whitespace"), nb::arg("indent").none(), nb::arg("separators").none(), nb::arg("strict_mode") ) .def( "compile_builtin_json_grammar", &GrammarCompiler::CompileBuiltinJSONGrammar, nb::call_guard() ) .def( "compile_structural_tag", &GrammarCompiler_CompileStructuralTag, nb::call_guard() ) .def( "compile_regex", &GrammarCompiler::CompileRegex, nb::call_guard() ) .def( "compile_grammar", &GrammarCompiler::CompileGrammar, nb::call_guard() ) .def("clear_cache", &GrammarCompiler::ClearCache) .def("get_cache_size_bytes", &GrammarCompiler::GetCacheSizeBytes) .def_prop_ro("cache_limit_bytes", &GrammarCompiler::CacheLimitBytes); auto pyGrammarMatcher = nb::class_(m, "GrammarMatcher"); pyGrammarMatcher .def( nb::init>, bool, int>(), nb::arg("compiled_grammar"), nb::arg("override_stop_tokens").none(), nb::arg("terminate_without_stop_token"), nb::arg("max_rollback_tokens") ) .def("accept_token", &GrammarMatcher::AcceptToken, nb::call_guard()) .def( "fill_next_token_bitmask", &GrammarMatcher_FillNextTokenBitmask, nb::call_guard() ) .def( "find_jump_forward_string", &GrammarMatcher::FindJumpForwardString, nb::call_guard() ) .def("rollback", &GrammarMatcher::Rollback, nb::call_guard()) .def("is_terminated", &GrammarMatcher::IsTerminated) .def("reset", &GrammarMatcher::Reset, nb::call_guard()) .def_prop_ro("max_rollback_tokens", &GrammarMatcher::GetMaxRollbackTokens) .def_prop_ro("stop_token_ids", &GrammarMatcher::GetStopTokenIds) .def( "_debug_accept_string", &GrammarMatcher::_DebugAcceptString, nb::call_guard() ) .def( "_debug_accept_string", [](GrammarMatcher& self, const nb::bytes& input_str, bool debug_print) { return self._DebugAcceptString(input_str.c_str(), debug_print); }, nb::call_guard() ); auto pyTestingModule = m.def_submodule("testing"); pyTestingModule .def( "_json_schema_to_ebnf", nb::overload_cast< const std::string&, bool, std::optional, std::optional>, bool>(&JSONSchemaToEBNF), nb::arg("schema"), nb::arg("any_whitespace"), nb::arg("indent").none(), nb::arg("separators").none(), nb::arg("strict_mode") ) .def("_regex_to_ebnf", &RegexToEBNF) .def("_ebnf_to_grammar_no_normalization", &_EBNFToGrammarNoNormalization) .def("_get_masked_tokens_from_bitmask", &Testing_DebugGetMaskedTokensFromBitmask) .def("_is_single_token_bitmask", &Testing_IsSingleTokenBitmask) .def("_get_allow_empty_rule_ids", &GetAllowEmptyRuleIds) .def( "_generate_range_regex", [](std::optional start, std::optional end) { std::string result = GenerateRangeRegex(start, end); result.erase(std::remove(result.begin(), result.end(), '\0'), result.end()); return result; }, nb::arg("start").none(), nb::arg("end").none() ) .def( "_generate_float_regex", [](std::optional start, std::optional end) { std::string result = GenerateFloatRangeRegex(start, end); result.erase(std::remove(result.begin(), result.end(), '\0'), result.end()); return result; }, nb::arg("start").none(), nb::arg("end").none() ); auto pyGrammarFunctorModule = pyTestingModule.def_submodule("grammar_functor"); pyGrammarFunctorModule.def("structure_normalizer", &StructureNormalizer::Apply) .def("byte_string_fuser", &ByteStringFuser::Apply) .def("rule_inliner", &RuleInliner::Apply) .def("dead_code_eliminator", &DeadCodeEliminator::Apply) .def("lookahead_assertion_analyzer", &LookaheadAssertionAnalyzer::Apply); auto pyKernelsModule = m.def_submodule("kernels"); pyKernelsModule.def( "apply_token_bitmask_inplace_cpu", &Kernels_ApplyTokenBitmaskInplaceCPU, nb::arg("logits_ptr"), nb::arg("logits_shape"), nb::arg("bitmask_ptr"), nb::arg("bitmask_shape"), nb::arg("vocab_size"), nb::arg("indices").none(), nb::call_guard() ); } xgrammar-0.1.19/cpp/nanobind/python_methods.cc000066400000000000000000000112421500705317600213510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/nanobind/python_methods.cc */ #include "python_methods.h" #include #include #include #include #include #include "../grammar_data_structure.h" #include "../support/logging.h" namespace xgrammar { TokenizerInfo TokenizerInfo_Init( const std::vector& encoded_vocab, int vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) { XGRAMMAR_CHECK(vocab_type == 0 || vocab_type == 1 || vocab_type == 2) << "Invalid vocab type: " << vocab_type; return TokenizerInfo( encoded_vocab, static_cast(vocab_type), vocab_size, stop_token_ids, add_prefix_space ); } int TokenizerInfo_GetVocabType(const TokenizerInfo& tokenizer) { return static_cast(tokenizer.GetVocabType()); } bool GrammarMatcher_FillNextTokenBitmask( GrammarMatcher& matcher, intptr_t token_bitmask_ptr, std::vector shape, int32_t index, bool debug_print ) { XGRAMMAR_CHECK(shape.size() == 1 || shape.size() == 2) << "token_bitmask tensor must be 1D or 2D"; DLTensor bitmask_dltensor{ reinterpret_cast(token_bitmask_ptr), DLDevice{kDLCPU, 0}, static_cast(shape.size()), GetBitmaskDLType(), shape.data(), nullptr, 0 }; return matcher.FillNextTokenBitmask(&bitmask_dltensor, index, debug_print); } std::vector Testing_DebugGetMaskedTokensFromBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ) { XGRAMMAR_CHECK(shape.size() == 1 || shape.size() == 2) << "token_bitmask tensor must be 1D or 2D"; DLTensor bitmask_dltensor{ reinterpret_cast(token_bitmask_ptr), DLDevice{kDLCPU, 0}, static_cast(shape.size()), GetBitmaskDLType(), shape.data(), nullptr, 0 }; std::vector result; _DebugGetMaskedTokensFromBitmask(&result, bitmask_dltensor, vocab_size, index); return result; } std::pair Testing_IsSingleTokenBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ) { XGRAMMAR_CHECK(shape.size() == 1 || shape.size() == 2) << "token_bitmask tensor must be 1D or 2D"; DLTensor bitmask_dltensor{ reinterpret_cast(token_bitmask_ptr), DLDevice{kDLCPU, 0}, static_cast(shape.size()), GetBitmaskDLType(), shape.data(), nullptr, 0 }; return _IsSingleTokenBitmask(bitmask_dltensor, vocab_size, index); } void Kernels_ApplyTokenBitmaskInplaceCPU( intptr_t logits_ptr, std::pair logits_shape, intptr_t bitmask_ptr, std::pair bitmask_shape, int vocab_size, std::optional> indices ) { std::array logits_shape_arr = {logits_shape.first, logits_shape.second}; std::array bitmask_shape_arr = {bitmask_shape.first, bitmask_shape.second}; DLTensor logits_dltensor{ reinterpret_cast(logits_ptr), DLDevice{kDLCPU, 0}, 2, DLDataType{kDLFloat, 32, 1}, logits_shape_arr.data(), nullptr, 0 }; DLTensor bitmask_dltensor{ reinterpret_cast(bitmask_ptr), DLDevice{kDLCPU, 0}, 2, GetBitmaskDLType(), bitmask_shape_arr.data(), nullptr, 0 }; ApplyTokenBitmaskInplaceCPU(&logits_dltensor, bitmask_dltensor, vocab_size, indices); } std::vector GetAllowEmptyRuleIds(const CompiledGrammar& compiled_grammar) { return compiled_grammar.GetGrammar()->allow_empty_rule_ids; } Grammar Grammar_FromStructuralTag( const std::vector>& tags, const std::vector& triggers ) { std::vector tags_objects; tags_objects.reserve(tags.size()); for (const auto& tag : tags) { tags_objects.emplace_back( StructuralTagItem{std::get<0>(tag), std::get<1>(tag), std::get<2>(tag)} ); } return Grammar::FromStructuralTag(tags_objects, triggers); } CompiledGrammar GrammarCompiler_CompileStructuralTag( GrammarCompiler& compiler, const std::vector>& tags, const std::vector& triggers ) { std::vector tags_objects; tags_objects.reserve(tags.size()); for (const auto& tag : tags) { tags_objects.emplace_back( StructuralTagItem{std::get<0>(tag), std::get<1>(tag), std::get<2>(tag)} ); } return compiler.CompileStructuralTag(tags_objects, triggers); } } // namespace xgrammar xgrammar-0.1.19/cpp/nanobind/python_methods.h000066400000000000000000000036561500705317600212250ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/nanobind/python_methods.h * \brief The header for the support of grammar-guided generation. */ #ifndef XGRAMMAR_NANOBIND_PYTHON_METHODS_H_ #define XGRAMMAR_NANOBIND_PYTHON_METHODS_H_ #include #include #include #include #include #include namespace xgrammar { TokenizerInfo TokenizerInfo_Init( const std::vector& encoded_vocab, int vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ); int TokenizerInfo_GetVocabType(const TokenizerInfo& tokenizer); bool GrammarMatcher_FillNextTokenBitmask( GrammarMatcher& matcher, intptr_t token_bitmask_ptr, std::vector shape, int32_t index, bool debug_print ); std::vector Testing_DebugGetMaskedTokensFromBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ); std::pair Testing_IsSingleTokenBitmask( intptr_t token_bitmask_ptr, std::vector shape, int32_t vocab_size, int32_t index ); void Kernels_ApplyTokenBitmaskInplaceCPU( intptr_t logits_ptr, std::pair logits_shape, intptr_t bitmask_ptr, std::pair bitmask_shape, int vocab_size, std::optional> indices ); std::vector GetAllowEmptyRuleIds(const CompiledGrammar& compiled_grammar); Grammar Grammar_FromStructuralTag( const std::vector>& tags, const std::vector& triggers ); CompiledGrammar GrammarCompiler_CompileStructuralTag( GrammarCompiler& compiler, const std::vector>& tags, const std::vector& triggers ); } // namespace xgrammar #endif // XGRAMMAR_NANOBIND_PYTHON_METHODS_H_ xgrammar-0.1.19/cpp/persistent_stack.h000066400000000000000000000410461500705317600177510ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/persistent_stack.h * \brief The header for the definition of the persistent stack and stack elements. */ #ifndef XGRAMMAR_PERSISTENT_STACK_H_ #define XGRAMMAR_PERSISTENT_STACK_H_ #include #include #include #include #include "grammar_data_structure.h" #include "grammar_serializer.h" namespace xgrammar { /*! \brief Specifies a stack element. It is mainly a position in a rule. */ struct StackElement { /*! \brief The rule's id. Used for debug purposes. */ int32_t rule_id = -1; /*! \brief Which choice in this rule is selected. */ int32_t sequence_id = -1; /*! \brief Which element of the choice sequence is to be visited. When the current sequence is * a tag dispatch rule, this element id the currently visited node. */ int32_t element_id = -1; /*! \brief The number of left utf8 bytes in the current element. Used when the element is * a character class or a character class star. */ int32_t left_utf8_bytes = 0; /*! \brief The next position to match in the current byte string. Used when the element is * a byte string. */ int32_t element_in_string = 0; /*! \brief The id of the parent node in the PersistentStack. */ int32_t parent_id = -1; /*! \brief The reference count of this StackElement. If reduces to zero, the node will be * removed from the StackElementBuffer. */ int reference_count = 0; /*! \brief A parent_id value of kNoParent means this StackElement is the root of the tree. */ static constexpr int32_t kNoParent = -1; constexpr StackElement() = default; constexpr StackElement( int32_t rule_id, int32_t sequence_id, int32_t element_id, int32_t parent_id = kNoParent ) : rule_id(rule_id), sequence_id(sequence_id), element_id(element_id), parent_id(parent_id) {} // The element is invalid when sequence_id is -1. bool IsInvalid() const { return sequence_id == -1; } bool operator==(const StackElement& other) const { return rule_id == other.rule_id && sequence_id == other.sequence_id && element_id == other.element_id && parent_id == other.parent_id && left_utf8_bytes == other.left_utf8_bytes && element_in_string == other.element_in_string; } }; /*! \brief A special value for invalid StackElement. */ inline constexpr StackElement kInvalidStackElement(-1, -1, -1, -1); /*! \brief A buffer to manage all StackElements. */ class StackElementBuffer { public: /*! * \brief Allocate a new StackElement. with given initial value. * \returns The id of the allocated node. */ int32_t Allocate(StackElement stack_element) { int32_t id; if (free_nodes_.empty()) { buffer_.emplace_back(); id = static_cast(buffer_.size()) - 1; } else { id = free_nodes_.back(); XGRAMMAR_DCHECK(buffer_[id].IsInvalid()); free_nodes_.pop_back(); } stack_element.reference_count = 0; buffer_[id] = stack_element; return id; } /*! \brief Free the StackElement with the given id. */ void Free(int32_t id) { XGRAMMAR_DCHECK(!buffer_[id].IsInvalid()); buffer_[id] = kInvalidStackElement; free_nodes_.push_back(id); } /*! \brief Get the capacity of the buffer. */ size_t Capacity() const { return buffer_.size(); } /*! \brief Get the number of allocated nodes. */ size_t Size() const { XGRAMMAR_DCHECK(buffer_.size() >= free_nodes_.size()); return buffer_.size() - free_nodes_.size(); } /*! \brief Get the StackElement with the given id. */ StackElement& operator[](int32_t id) { XGRAMMAR_DCHECK(id >= 0 && id < static_cast(buffer_.size())); XGRAMMAR_DCHECK(!buffer_[id].IsInvalid()); return buffer_[id]; } const StackElement& operator[](int32_t id) const { XGRAMMAR_DCHECK(id >= 0 && id < static_cast(buffer_.size())); XGRAMMAR_DCHECK(!buffer_[id].IsInvalid()); return buffer_[id]; } void Reset() { buffer_.clear(); free_nodes_.clear(); } friend class PersistentStack; private: /*! \brief The buffer to store all StackElements. */ std::vector buffer_; /*! \brief A stack to store all free node ids. */ std::vector free_nodes_; }; /*! * \brief A tree structure to store all stacks. Every stack contains several StackElements, and * is represented as a path from the root to a leaf node. */ class PersistentStack { public: /*! \brief Construct a PersistentStack associated with the given grammar. */ PersistentStack(const Grammar& grammar) : grammar_(grammar) {} /*! * \brief Create a new node with the given StackElement. The reference count of the new node * is zero. * * \note Later, this node should either be pointed by some child rule, or become a stack top * node (so it will be pointed to by an attached pointer) to be maintained in the * reference-counting based memory management. */ int32_t NewNode(const StackElement& stack_element) { auto id = node_buffer_.Allocate(stack_element); if (stack_element.parent_id != StackElement::kNoParent) { XGRAMMAR_DCHECK( stack_element.parent_id < static_cast(node_buffer_.Capacity()) && !node_buffer_[stack_element.parent_id].IsInvalid() ); node_buffer_[stack_element.parent_id].reference_count++; } return id; } /*! * \brief Check if the given StackElement points to the end of the grammar. For a stack element, * if its rule id is the root rule id, and the element id equals to the length of the sequence it * refers to, it would be the end of the grammar. */ bool IsEndOfGrammar(const StackElement& stack_element) const; /*! \brief Attach an additional reference to the node with the given id. */ void AttachRefTo(int32_t id) { XGRAMMAR_DCHECK(id != StackElement::kNoParent); node_buffer_[id].reference_count++; } /*! \brief Remove a reference to the node with the given id. If the reference count becomes zero, * free the node and recursively all its ancestors with zero reference count. */ void RemoveRefTo(int32_t id) { XGRAMMAR_DCHECK(id != StackElement::kNoParent); auto cur_node = id; while (cur_node != StackElement::kNoParent) { node_buffer_[cur_node].reference_count--; if (node_buffer_[cur_node].reference_count != 0) { break; } auto next_node = node_buffer_[cur_node].parent_id; node_buffer_.Free(cur_node); cur_node = next_node; } } /*! \brief Get the StackElement with the given id. */ const StackElement& operator[](int32_t id) const { XGRAMMAR_DCHECK(id != StackElement::kNoParent); XGRAMMAR_DCHECK(!node_buffer_[id].IsInvalid()); return node_buffer_[id]; } /*! \brief Print the given stack_element to a string. */ std::string PrintStackElement(const StackElement& stack_element) const; /*! \brief Print the stack_element associated with the given id to a string. */ std::string PrintStackElement(int32_t id) const; /*! \brief Print the stack with the given top id to a string. */ std::string PrintStackByTopId(int32_t top_id) const; /*! * \brief Check the well-formedness of the tree and the associated buffer. For debug purpose. * \details This function checks the following properties: * 1. Every node is pointed directly or indirectly by a outside pointer. * 2. Every node's reference count is consistent with the actual reference count. * 3. All ids and stack elements are valid. * 4. If a node in the buffer is free, it should be equal to kInvalidStackElement. */ void CheckWellFormed(const std::vector& outside_pointers) const; /*! \brief Reset the tree and the associated buffer. */ void Reset() { node_buffer_.Reset(); } private: /*! \brief The grammar associated with this PersistentStack. */ Grammar grammar_; /*! \brief The buffer to store all StackElements. */ StackElementBuffer node_buffer_; }; /*! * \brief A class to maintain the stack tops and its history to support rollback. * \details This class helps to maintain nodes by automatically maintaining the attached references. * If a node is not existing in any stack in the history record, it will be freed. * * It can store up to the previous max_rollback_tokens + 1 steps of history, and thus supports * rolling back up to max_rollback_tokens steps. */ class StackTopsHistory { public: /*! * \param tree The PersistentStack to be associated with. Possibly modify the tree by * attaching and removing references to the stack top nodes. * \param max_rollback_tokens The maximum number of rollback tokens to be supported. */ StackTopsHistory(PersistentStack* tree) : persistent_stack_(tree) {} /*! * \brief Push a new history record consisting a list of stack tops. These nodes will be recorded * as existing in a stack (by attaching a reference to them). * \param stack_tops The stack tops to be pushed. * \param drop_old Whether to drop the oldest history record if the history size exceeds the * limit. If the history is dropped, node that do not exist in any stack any more will be freed. */ void PushHistory(const std::vector& stack_tops) { stack_tops_history_.push_back(stack_tops); for (auto id : stack_tops) { persistent_stack_->AttachRefTo(id); } } /*! \brief Roll back to several previous steps. Possibly frees node that do not exist in any stack * any more. */ void Rollback(int rollback_steps) { XGRAMMAR_DCHECK(rollback_steps < static_cast(stack_tops_history_.size())) << "The number of requested rollback tokens is greater than or equal to the current " "history " << "size: " << rollback_steps << " vs " << stack_tops_history_.size() << "."; while (rollback_steps--) { PopLatest(); } } /*! \brief Discard the earliest several steps. Possibly frees node that do not exist in any stack * any more. */ void DiscardEarliest(int discard_steps) { XGRAMMAR_DCHECK(discard_steps < static_cast(stack_tops_history_.size())) << "The number of requested discard steps is greater than or equal to the current " "history " << "size: " << discard_steps << " vs " << stack_tops_history_.size() << "."; while (discard_steps--) { PopEarliest(); } } /*! \brief Get the latest stack tops. */ const std::vector& GetLatest() const { return stack_tops_history_.back(); } /*! * \brief Print one history record. * \param steps_ago The number of steps behind the latest record. 0 means the * latest record. */ std::string PrintHistory(int steps_ago = 0) const; /*! \brief Get the number of history records. */ int Size() const { return stack_tops_history_.size(); } /*! \brief Check the well-formedness of the tree and the associated buffer. */ void CheckWellFormed() const; /*! \brief Reset the history and the associated node tree. */ void Reset() { stack_tops_history_.clear(); persistent_stack_->Reset(); } private: /*! \brief Pop the oldest history record. Possibly frees node that do not exist in any stack any * more. */ void PopEarliest() { const auto& old_stack_tops = stack_tops_history_.front(); for (auto id : old_stack_tops) { persistent_stack_->RemoveRefTo(id); } stack_tops_history_.pop_front(); } /*! \brief Pop the latest history record. Possibly frees node that do not exist in any stack any * more. */ void PopLatest() { const auto& new_stack_tops = stack_tops_history_.back(); for (auto id : new_stack_tops) { persistent_stack_->RemoveRefTo(id); } stack_tops_history_.pop_back(); } /*! \brief Modifiable pointer to the PersistentStack. */ PersistentStack* persistent_stack_; /*! \brief The history of stack tops. */ std::deque> stack_tops_history_; }; inline bool PersistentStack::IsEndOfGrammar(const StackElement& stack_element) const { if (stack_element.parent_id != StackElement::kNoParent) { return false; } auto seq_expr = grammar_->GetRuleExpr(stack_element.sequence_id); if (seq_expr.type == Grammar::Impl::RuleExprType::kTagDispatch) { return stack_element.element_id != -1; } else { return seq_expr.size() == stack_element.element_id; } } inline std::string PersistentStack::PrintStackElement(int32_t id) const { return "id: " + std::to_string(id) + ", " + PrintStackElement(node_buffer_[id]); } inline std::string PersistentStack::PrintStackElement(const StackElement& stack_element) const { std::stringstream ss; ss << "StackElement: rule " << stack_element.rule_id; if (stack_element.rule_id != -1) { ss << ": " << grammar_->GetRule(stack_element.rule_id).name; } ss << ", sequence " << stack_element.sequence_id << ": " << GrammarPrinter(grammar_).PrintRuleExpr(stack_element.sequence_id); ss << ", element id: " << stack_element.element_id; auto sequence = grammar_->GetRuleExpr(stack_element.sequence_id); if (sequence.type != Grammar::Impl::RuleExprType::kTagDispatch && stack_element.element_id < static_cast(sequence.size())) { auto element = grammar_->GetRuleExpr(sequence[stack_element.element_id]); if (element.type == Grammar::Impl::RuleExprType::kByteString) { ss << ", element in string: " << stack_element.element_in_string; } else if (element.type == Grammar::Impl::RuleExprType::kCharacterClass || element.type == Grammar::Impl::RuleExprType::kCharacterClassStar) { ss << ", left utf8 bytes: " << stack_element.left_utf8_bytes; } } ss << ", parent id: " << stack_element.parent_id << ", ref count: " << stack_element.reference_count; return ss.str(); } inline std::string PersistentStack::PrintStackByTopId(int32_t top_id) const { std::stringstream ss; std::vector stack; for (auto cur_id = top_id; cur_id != StackElement::kNoParent; cur_id = node_buffer_[cur_id].parent_id) { stack.push_back(cur_id); } ss << "{\n"; for (auto it = stack.rbegin(); it != stack.rend(); ++it) { ss << PrintStackElement(*it) << "\n"; } ss << "}"; return ss.str(); } inline void PersistentStack::CheckWellFormed(const std::vector& outside_pointers) const { const auto& buffer = node_buffer_.buffer_; std::unordered_set free_nodes_set( node_buffer_.free_nodes_.begin(), node_buffer_.free_nodes_.end() ); int buffer_size = static_cast(buffer.size()); std::vector new_reference_counter(buffer_size, 0); std::vector visited(buffer_size, false); std::queue visit_queue; for (auto id : outside_pointers) { XGRAMMAR_CHECK(id >= 0 && id < buffer_size); XGRAMMAR_CHECK(!buffer[id].IsInvalid()); new_reference_counter[id]++; if (visited[id] == false) { visited[id] = true; visit_queue.push(id); } } while (!visit_queue.empty()) { auto cur_id = visit_queue.front(); visit_queue.pop(); const auto& stack_element = buffer[cur_id]; if (stack_element.parent_id != StackElement::kNoParent) { XGRAMMAR_CHECK(stack_element.parent_id >= 0 && stack_element.parent_id < buffer_size); XGRAMMAR_CHECK(!buffer[stack_element.parent_id].IsInvalid()); new_reference_counter[stack_element.parent_id]++; if (visited[stack_element.parent_id] == false) { visited[stack_element.parent_id] = true; visit_queue.push(stack_element.parent_id); } } } for (int i = 0; i < static_cast(buffer.size()); ++i) { if (free_nodes_set.count(i)) { XGRAMMAR_CHECK(buffer[i].IsInvalid()); XGRAMMAR_CHECK(visited[i] == false); } else { XGRAMMAR_CHECK(visited[i] == true); XGRAMMAR_CHECK(!buffer[i].IsInvalid()); XGRAMMAR_CHECK(new_reference_counter[i] == buffer[i].reference_count) << "Reference counters unmatch for node #" << i << ": Updated " << new_reference_counter[i] << ", Original " << buffer[i].reference_count; } } } inline std::string StackTopsHistory::PrintHistory(int steps_ago) const { const auto& latest_tops = stack_tops_history_[static_cast(stack_tops_history_.size()) - 1 - steps_ago]; std::stringstream ss; ss << "Num of stacks: " << latest_tops.size() << std::endl; int cnt = 0; for (auto id : latest_tops) { ss << "Stack #" << cnt << ": " << persistent_stack_->PrintStackByTopId(id) << "\n"; ++cnt; } return ss.str(); } inline void StackTopsHistory::CheckWellFormed() const { std::vector outside_pointers; for (const auto& stack_tops : stack_tops_history_) { outside_pointers.insert(outside_pointers.end(), stack_tops.begin(), stack_tops.end()); } persistent_stack_->CheckWellFormed(outside_pointers); } } // namespace xgrammar #endif // XGRAMMAR_PERSISTENT_STACK_H_ xgrammar-0.1.19/cpp/regex_converter.cc000066400000000000000000000267341500705317600177320ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/regex_converter.cc */ #include "regex_converter.h" #include #include #include #include #include "support/encoding.h" #include "support/logging.h" #include "support/utils.h" namespace xgrammar { /*! * \brief Convert a regex to EBNF. * \details The implementation refers to the regex described in * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions */ class RegexConverter { public: explicit RegexConverter(const std::string& regex) : regex_(regex) { regex_codepoints_ = ParseUTF8(regex_.c_str(), false); if (regex_codepoints_[0] == kInvalidUTF8) { XGRAMMAR_LOG(FATAL) << "The regex is not a valid UTF-8 string."; XGRAMMAR_UNREACHABLE(); } regex_codepoints_.push_back(0); // Add a null terminator } std::string Convert(); private: /** * \brief Add a segment string to the result EBNF string. It especially adds a space if needed * and add_space is true. */ void AddEBNFSegment(const std::string& element); [[noreturn]] void RaiseError(const std::string& message); void RaiseWarning(const std::string& message); std::string HandleCharacterClass(); std::string HandleRepetitionRange(); std::string HandleCharEscape(); std::string HandleEscape(); std::string HandleEscapeInCharClass(); /** * \brief Handle group modifier. The general format is "(?" + modifier + content + ")". E.g. * "(?:abc)" is a non-capturing group. */ void HandleGroupModifier(); std::string regex_; std::vector regex_codepoints_; TCodepoint* start_; TCodepoint* current_; TCodepoint* end_; std::string result_ebnf_; int parenthesis_level_ = 0; }; void RegexConverter::AddEBNFSegment(const std::string& element) { if (!result_ebnf_.empty()) { result_ebnf_ += ' '; } result_ebnf_ += element; } void RegexConverter::RaiseError(const std::string& message) { XGRAMMAR_LOG(FATAL) << "Regex parsing error at position " << current_ - start_ + 1 << ": " << message; XGRAMMAR_UNREACHABLE(); } void RegexConverter::RaiseWarning(const std::string& message) { XGRAMMAR_LOG(WARNING) << "Regex parsing warning at position " << current_ - start_ + 1 << ": " << message; } std::string RegexConverter::HandleCharacterClass() { std::string char_class = "["; ++current_; if (*current_ == ']') { RaiseError("Empty character class is not allowed in regex."); } while (*current_ != ']' && current_ != end_) { if (*current_ == '\\') { char_class += HandleEscapeInCharClass(); } else { char_class += PrintAsUTF8(*current_); ++current_; } } if (current_ == end_) { RaiseError("Unclosed '['"); } char_class += ']'; ++current_; return char_class; } // {x}: Match exactly x occurrences of the preceding regular expression. // {x,} // {x,y} std::string RegexConverter::HandleRepetitionRange() { std::string result = "{"; ++current_; if (!isdigit(*current_)) { RaiseError("Invalid repetition count."); } while (isdigit(*current_)) { result += static_cast(*current_); ++current_; } if (*current_ != ',' && *current_ != '}') { RaiseError("Invalid repetition count."); } result += static_cast(*current_); ++current_; if (current_[-1] == '}') { // Matches {x} return result; } if (!isdigit(*current_) && *current_ != '}') { RaiseError("Invalid repetition count."); } while (isdigit(*current_)) { result += static_cast(*current_); ++current_; } if (*current_ != '}') { RaiseError("Invalid repetition count."); } result += '}'; ++current_; return result; } std::string RegexConverter::HandleCharEscape() { // clang-format off static const std::unordered_map CUSTOM_ESCAPE_MAP = { {'^', '^'}, {'$', '$'}, {'.', '.'}, {'*', '*'}, {'+', '+'}, {'?', '?'}, {'\\', '\\'}, {'(', '('}, {')', ')'}, {'[', '['}, {']', ']'}, {'{', '{'}, {'}', '}'}, {'|', '|'}, {'/', '/'}, {'-', '-'} }; // clang-format on if (end_ - current_ < 2 || (current_[1] == 'u' && end_ - current_ < 5) || (current_[1] == 'x' && end_ - current_ < 4) || (current_[1] == 'c' && end_ - current_ < 3)) { RaiseError("Escape sequence is not finished."); } auto [codepoint, len] = xgrammar::HandleEscape(current_, CUSTOM_ESCAPE_MAP); if (codepoint != CharHandlingError::kInvalidEscape) { current_ += len; return PrintAsEscapedUTF8(codepoint); } else if (current_[1] == 'u' && current_[2] == '{') { current_ += 3; int len = 0; TCodepoint value = 0; while (HexCharToInt(current_[len]) != -1 && len <= 6) { value = value * 16 + HexCharToInt(current_[len]); ++len; } if (len == 0 || len > 6 || current_[len] != '}') { RaiseError("Invalid Unicode escape sequence."); } current_ += len + 1; return PrintAsEscapedUTF8(value); } else if (current_[1] == 'c') { current_ += 2; if (!std::isalpha(*current_)) { RaiseError("Invalid control character escape sequence."); } ++current_; return PrintAsEscapedUTF8((*(current_ - 1)) % 32); } else { RaiseWarning( "Escape sequence '\\" + PrintAsEscapedUTF8(current_[1]) + "' is not recognized. The character itself will be matched" ); current_ += 2; return PrintAsEscapedUTF8(current_[-1]); } } std::string RegexConverter::HandleEscapeInCharClass() { if (end_ - current_ < 2) { RaiseError("Escape sequence is not finished."); } if (current_[1] == 'd') { current_ += 2; return "0-9"; } else if (current_[1] == 'D') { current_ += 2; return R"(\x00-\x2F\x3A-\U0010FFFF)"; } else if (current_[1] == 'w') { current_ += 2; return "a-zA-Z0-9_"; } else if (current_[1] == 'W') { current_ += 2; return R"(\x00-\x2F\x3A-\x40\x5B-\x5E\x60\x7B-\U0010FFFF)"; } else if (current_[1] == 's') { current_ += 2; return R"(\f\n\r\t\v\u0020\u00a0)"; } else if (current_[1] == 'S') { current_ += 2; return R"(\x00-\x08\x0E-\x1F\x21-\x9F\xA1-\U0010FFFF)"; } else { auto res = HandleCharEscape(); if (res == "]" || res == "-") { return "\\" + res; } else { return res; } } } std::string RegexConverter::HandleEscape() { // clang-format off static const std::unordered_map CUSTOM_ESCAPE_MAP = { {'^', '^'}, {'$', '$'}, {'.', '.'}, {'*', '*'}, {'+', '+'}, {'?', '?'}, {'\\', '\\'}, {'(', '('}, {')', ')'}, {'[', '['}, {']', ']'}, {'{', '{'}, {'}', '}'}, {'|', '|'}, {'/', '/'} }; // clang-format on if (end_ - current_ < 2) { RaiseError("Escape sequence is not finished."); } if (current_[1] == 'd') { current_ += 2; return "[0-9]"; } else if (current_[1] == 'D') { current_ += 2; return "[^0-9]"; } else if (current_[1] == 'w') { current_ += 2; return "[a-zA-Z0-9_]"; } else if (current_[1] == 'W') { current_ += 2; return "[^a-zA-Z0-9_]"; } else if (current_[1] == 's') { current_ += 2; return R"([\f\n\r\t\v\u0020\u00a0])"; } else if (current_[1] == 'S') { current_ += 2; return R"([^[\f\n\r\t\v\u0020\u00a0])"; } else if ((current_[1] >= '1' && current_[1] <= '9') || current_[1] == 'k') { RaiseError("Backreference is not supported yet."); } else if (current_[1] == 'p' || current_[1] == 'P') { RaiseError("Unicode character class escape sequence is not supported yet."); } else if (current_[1] == 'b' || current_[1] == 'B') { RaiseError("Word boundary is not supported yet."); } else { return "\"" + HandleCharEscape() + "\""; } } void RegexConverter::HandleGroupModifier() { if (current_ == end_) { RaiseError("Group modifier is not finished."); } if (*current_ == ':') { // Non-capturing group. ++current_; } else if (*current_ == '=' || *current_ == '!') { // Positive or negative lookahead. RaiseError("Lookahead is not supported yet."); } else if (*current_ == '<' && current_ + 1 != end_ && (current_[1] == '=' || current_[1] == '!')) { // Positive or negative lookbehind. RaiseError("Lookbehind is not supported yet."); } else if (*current_ == '<') { ++current_; while (current_ != end_ && isalpha(*current_)) { ++current_; } if (current_ == end_ || *current_ != '>') { RaiseError("Invalid named capturing group."); } // Just ignore the named of the group. ++current_; } else { // Group modifier flag. RaiseError("Group modifier flag is not supported yet."); } } std::string RegexConverter::Convert() { start_ = regex_codepoints_.data(); current_ = start_; end_ = start_ + regex_codepoints_.size() - 1; while (current_ != end_) { if (*current_ == '^') { if (current_ != start_) { RaiseWarning( "'^' should be at the start of the regex, but found in the middle. It is ignored." ); } ++current_; } else if (*current_ == '$') { if (current_ != end_ - 1) { RaiseWarning( "'$' should be at the end of the regex, but found in the middle. It is ignored." ); } ++current_; } else if (*current_ == '[') { AddEBNFSegment(HandleCharacterClass()); } else if (*current_ == '(') { ++current_; ++parenthesis_level_; AddEBNFSegment("("); if (current_ != end_ && *current_ == '?') { ++current_; HandleGroupModifier(); } } else if (*current_ == ')') { if (parenthesis_level_ == 0) { RaiseError("Unmatched ')'"); } // Special case: if the previous character is '|', add an empty string to the result. if (current_ != start_ && current_[-1] == '|') { AddEBNFSegment("\"\""); } --parenthesis_level_; AddEBNFSegment(")"); ++current_; } else if (*current_ == '*' || *current_ == '+' || *current_ == '?') { result_ebnf_ += static_cast(*current_); ++current_; if (current_ != end_ && *current_ == '?') { // Ignore the non-greedy modifier because our grammar handles all repetition numbers // non-deterministically. ++current_; } if (current_ != end_ && (*current_ == '{' || *current_ == '*' || *current_ == '+' || *current_ == '?')) { RaiseError("Two consecutive repetition modifiers are not allowed."); } } else if (*current_ == '{') { result_ebnf_ += HandleRepetitionRange(); if (current_ != end_ && *current_ == '?') { // Still ignore the non-greedy modifier. ++current_; } if (current_ != end_ && (*current_ == '{' || *current_ == '*' || *current_ == '+' || *current_ == '?')) { RaiseError("Two consecutive repetition modifiers are not allowed."); } } else if (*current_ == '|') { AddEBNFSegment("|"); ++current_; } else if (*current_ == '\\') { AddEBNFSegment(HandleEscape()); } else if (*current_ == '.') { AddEBNFSegment(R"([\u0000-\U0010FFFF])"); ++current_; } else { // Non-special characters are matched literally. AddEBNFSegment("\"" + PrintAsEscapedUTF8(*current_) + "\""); ++current_; } } if (parenthesis_level_ != 0) { RaiseError("The parenthesis is not closed."); } return result_ebnf_; } std::string RegexToEBNF(const std::string& regex, bool with_rule_name) { RegexConverter converter(regex); if (with_rule_name) { return "root ::= " + converter.Convert() + "\n"; } else { return converter.Convert(); } } } // namespace xgrammar xgrammar-0.1.19/cpp/regex_converter.h000066400000000000000000000007211500705317600175600ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/regex_converter.h * \brief Convert a regex string to EBNF grammar string. */ #ifndef XGRAMMAR_REGEX_CONVERTER_H_ #define XGRAMMAR_REGEX_CONVERTER_H_ #include namespace xgrammar { /*! * \brief Convert a regex string to EBNF grammar string. */ std::string RegexToEBNF(const std::string& regex, bool with_rule_name = true); } // namespace xgrammar #endif // XGRAMMAR_REGEX_CONVERTER_H_ xgrammar-0.1.19/cpp/structural_tag.cc000066400000000000000000000052101500705317600175560ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/structural_tag.cc */ #include "structural_tag.h" #include #include #include #include "grammar_functor.h" #include "support/logging.h" namespace xgrammar { Grammar StructuralTagToGrammar( const std::vector& tags, const std::vector& triggers ) { // Step 1: handle triggers. Triggers should not be mutually inclusive std::vector sorted_triggers(triggers.begin(), triggers.end()); std::sort(sorted_triggers.begin(), sorted_triggers.end()); for (int i = 0; i < static_cast(sorted_triggers.size()) - 1; ++i) { XGRAMMAR_CHECK( sorted_triggers[i + 1].size() < sorted_triggers[i].size() || std::string_view(sorted_triggers[i + 1]).substr(0, sorted_triggers[i].size()) != sorted_triggers[i] ) << "Triggers should not be mutually inclusive, but " << sorted_triggers[i] << " is a prefix of " << sorted_triggers[i + 1]; } // Step 2: For each tag, find the trigger that is a prefix of the tag.begin // Convert the schema to grammar at the same time std::vector schema_grammars; schema_grammars.reserve(tags.size()); for (const auto& tag : tags) { auto schema_grammar = Grammar::FromJSONSchema(tag.schema, true); schema_grammars.push_back(schema_grammar); } std::vector>> tag_groups(triggers.size()); for (int it_tag = 0; it_tag < static_cast(tags.size()); ++it_tag) { const auto& tag = tags[it_tag]; bool found = false; for (int it_trigger = 0; it_trigger < static_cast(sorted_triggers.size()); ++it_trigger) { const auto& trigger = sorted_triggers[it_trigger]; if (trigger.size() <= tag.begin.size() && std::string_view(tag.begin).substr(0, trigger.size()) == trigger) { tag_groups[it_trigger].push_back(std::make_pair(tag, schema_grammars[it_tag])); found = true; break; } } XGRAMMAR_CHECK(found) << "Tag " << tag.begin << " does not match any trigger"; } // Step 3: Combine the tags to form a grammar // root ::= TagDispatch((trigger1, rule1), (trigger2, rule2), ...) // Suppose tag1 and tag2 matches trigger1, then // rule1 ::= (tag1.begin[trigger1.size():] + ToEBNF(tag1.schema) + tag1.end) | // (tag2.begin[trigger1.size():] + ToEBNF(tag2.schema) + tag2.end) | ... // // Suppose tag3 matches trigger2, then // rule2 ::= (tag3.begin[trigger2.size():] + ToEBNF(tag3.schema) + tag3.end) // // ... return StructuralTagGrammarCreator::Apply(sorted_triggers, tag_groups); } } // namespace xgrammar xgrammar-0.1.19/cpp/structural_tag.h000066400000000000000000000007601500705317600174250ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/structural_tag.h * \brief The header for the definition of the structural tag. */ #ifndef XGRAMMAR_STRUCTURAL_TAG_H_ #define XGRAMMAR_STRUCTURAL_TAG_H_ #include #include #include namespace xgrammar { Grammar StructuralTagToGrammar( const std::vector& tags, const std::vector& triggers ); } // namespace xgrammar #endif // XGRAMMAR_STRUCTURAL_TAG_H_ xgrammar-0.1.19/cpp/support/000077500000000000000000000000001500705317600157225ustar00rootroot00000000000000xgrammar-0.1.19/cpp/support/container.h000066400000000000000000000072211500705317600200570ustar00rootroot00000000000000#ifndef XGRAMMAR_SUPPORT_CONTAINER_H_ #define XGRAMMAR_SUPPORT_CONTAINER_H_ #include #include "logging.h" namespace xgrammar { namespace details { template class NodePool { public: NodePool() = default; void Reserve(int n) { node_pool_.reserve(n); } [[nodiscard]] int Allocate() { if (free_list_.empty()) { int node = Size(); node_pool_.emplace_back(); return node; } else { int node = free_list_.back(); free_list_.pop_back(); return node; } } void Deallocate(int node) { free_list_.push_back(node); } void Clear() { node_pool_.clear(); free_list_.clear(); } Node& operator[](int node) { XGRAMMAR_DCHECK(0 <= node && node < Size()); return node_pool_[node]; } int Size() const { return static_cast(node_pool_.size()); } private: std::vector node_pool_; std::vector free_list_; }; } // namespace details template class List { private: struct Node { int prev; int next; Value value; }; public: struct iterator { public: iterator(int n, List& c) : node_(n), list_(&c) { XGRAMMAR_DCHECK(0 <= node_ && node_ < list_->node_pool_.Size()); } iterator& operator++() { node_ = GetNode().next; return *this; } iterator operator++(int) { iterator tmp = *this; ++*this; return tmp; } Value& operator*() const { return GetNode().value; } Value* operator->() const { return &GetNode().value; } bool operator==(const iterator& rhs) const { XGRAMMAR_DCHECK(list_ == rhs.list_) << "compare different container is UB"; return node_ == rhs.node_; // compare different container is UB } bool operator!=(const iterator& rhs) const { XGRAMMAR_DCHECK(list_ == rhs.list_) << "compare different container is UB"; return node_ != rhs.node_; // compare different container is UB } int Index() const { return node_; } private: friend class List; Node& GetNode() const { return list_->node_pool_[node_]; } int node_; List* list_; }; List(int reserved = 0) { node_pool_.Reserve(reserved); InitGuard(); } iterator PushBack(const Value& value) { int node = node_pool_.Allocate(); XGRAMMAR_DCHECK(0 < node && node < node_pool_.Size()); node_pool_[node].value = value; LinkBefore(node, 0); return iterator(node, *this); } void MoveBack(int node) { XGRAMMAR_DCHECK(0 < node && node < node_pool_.Size()); Unlink(node); LinkBefore(node, 0); } iterator Erase(iterator it) { int node = it.Index(); XGRAMMAR_DCHECK(0 < node && node < node_pool_.Size()); int next = node_pool_[node].next; Unlink(node); node_pool_.Deallocate(node); return iterator(next, *this); } void Clear() { node_pool_.Clear(); InitGuard(); } iterator begin() { return iterator(node_pool_[0].next, *this); } iterator end() { return iterator(0, *this); } private: void InitGuard() { int node_id = node_pool_.Allocate(); XGRAMMAR_DCHECK(node_id == 0) << "node 0 should be reserved as guard node"; node_pool_[0].prev = 0; node_pool_[0].next = 0; } void LinkBefore(int node, int next) { int prev = node_pool_[next].prev; node_pool_[node].prev = prev; node_pool_[node].next = next; node_pool_[prev].next = node; node_pool_[next].prev = node; } void Unlink(int node) { int prev = node_pool_[node].prev; int next = node_pool_[node].next; node_pool_[prev].next = next; node_pool_[next].prev = prev; } details::NodePool node_pool_; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_CONTAINER_H_ xgrammar-0.1.19/cpp/support/cpptrace.h000066400000000000000000000020361500705317600176750ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/cpptrace.h * \details This file is an encapsulation of the cpptrace library. It helps debugging. This file * takes effect only when XGRAMMAR_ENABLE_CPPTRACE is set to 1, and only support Linux and * RelWithDebugInfo or Debug build. */ #ifndef XGRAMMAR_SUPPORT_CPPTRACE_H_ #define XGRAMMAR_SUPPORT_CPPTRACE_H_ #if XGRAMMAR_ENABLE_CPPTRACE == 1 #include #endif #include namespace xgrammar { #if XGRAMMAR_ENABLE_CPPTRACE == 1 // Flag to check if cpptrace feature is enabled static constexpr bool CPPTRACE_ENABLED = true; inline void PrintTrace() { cpptrace::generate_trace().print(); } inline std::string GetTraceString() { return cpptrace::generate_trace().to_string(true); } #else static constexpr bool CPPTRACE_ENABLED = false; // Provide empty implementation when cpptrace is disabled inline void PrintTrace() {} inline std::string GetTraceString() { return ""; } #endif } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_CPPTRACE_H_ xgrammar-0.1.19/cpp/support/csr_array.h000066400000000000000000000222121500705317600200570ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/csr_array.h */ #ifndef XGRAMMAR_SUPPORT_CSR_ARRAY_H_ #define XGRAMMAR_SUPPORT_CSR_ARRAY_H_ #include #include #include #include #include "logging.h" #include "utils.h" namespace xgrammar { // TODO(yixin): consider renaming to CompactVector /*! * \brief This class implements a Compressed Sparse Row (CSR) array data structure. It stores * a 2D array in a compressed format, where each row can have a variable number of elements, and * all rows are stored contiguously in memory. The inserted row is immutable. * * \note Inserting new rows into the CSRArray will invalidate the existing Row objects. * * \tparam DataType The type of elements stored in the CSRArray. * * \details * The CSRArray stores elements of type DataType in a compressed format, * where each row can have a variable number of elements. It uses two vectors: * - data_: stores all elements contiguously * - indptr_: stores the starting index of each row in data_. Its last element is the size of data_ * representing the ending index. * * This structure allows efficient storage and access for sparse data. */ template class CSRArray { public: /*! \brief Default constructor. */ CSRArray() = default; /****************** Accessors ******************/ /*! \brief Get the number of rows in the CSRArray. */ int32_t Size() const { return static_cast(indptr_.size()) - 1; } friend std::size_t MemorySize(const CSRArray& arr) { return MemorySize(arr.data_) + MemorySize(arr.indptr_); } /*! * \brief Struct representing a row in the CSRArray. */ struct Row { /*! \brief Pointer to the data of the row. */ const DataType* data; /*! \brief Length of the row data. */ int32_t data_len; /*! * \brief Access an element in the row. * \param i Index of the element to access. * \return Reference to the element at index i. */ const DataType& operator[](int32_t i) const { XGRAMMAR_DCHECK(i >= 0 && i < data_len) << "Index " << i << " of the CSRArray Row is out of bound"; return data[i]; } /*! \brief Get the beginning iterator of the row. */ const DataType* begin() const { return data; } /*! \brief Get the end iterator of the row. */ const DataType* end() const { return data + data_len; } /*! \brief Get the size of the row. */ int32_t size() const { return data_len; } friend std::ostream& operator<<(std::ostream& os, const Row& row) { os << "["; for (auto i = 0; i < row.data_len; ++i) { if (i > 0) { os << ", "; } os << row[i]; } os << "]"; return os; } }; /*! * \brief Access a row in the CSRArray. * \param i Index of the row to access. * \return Row struct representing the i-th row. */ Row operator[](int32_t i) const; /****************** Modifiers ******************/ /*! * \brief Insert a new row of data into the CSRArray. * \param data Pointer to the data to be inserted. * \param data_len Length of the data to be inserted. * \return The index of the newly inserted row. */ int32_t Insert(const DataType* new_data, int32_t new_data_len); /*! * \brief Insert a new row of data into the CSRArray from a vector. * \param data Vector containing the data to be inserted. * \return The index of the newly inserted row. */ int32_t Insert(const std::vector& new_data); /*! * \brief Insert a new row of data into the CSRArray from a Row struct. * \param row The Row struct containing the data to be inserted. * \return The index of the newly inserted row. */ int32_t Insert(const Row& row) { return Insert(row.data, row.data_len); } /*! * \brief Insert a new row of non-contiguous data into the CSRArray. This method inserts a * single element followed by a sequence of elements. This is useful in the GrammarExpr data * structure. * \param data_1 The first element to be inserted. * \param data_2 Pointer to the remaining data to be inserted. * \param data_2_len Length of the remaining data to be inserted. * \return The index of the newly inserted row. */ int32_t InsertNonContiguous(DataType data_1, const DataType* data_2, int32_t data_2_len); /****************** Internal Accessors ******************/ /*! \brief Get a pointer to the underlying data array. */ const DataType* data() const { return data_.data(); } /*! \brief Get a pointer to the underlying index pointer array. */ const int32_t* indptr() const { return indptr_.data(); } /****************** Serialization ******************/ /*! * \brief Serialize the CSRArray to a JSON string. * \return A JSON value representation of the CSRArray. */ picojson::value Serialize() const; /*! * \brief Deserialize a JSON string to create a CSRArray. * \param v The JSON value to deserialize. * \return A new CSRArray object created from the deserialized data. * \throws xgrammar::InternalError if the JSON parsing fails or if the required fields are * missing. */ static CSRArray Deserialize(const picojson::value& v); friend std::ostream& operator<<(std::ostream& os, const CSRArray& csr_array) { os << "CSRArray(["; for (auto i = 0; i < csr_array.Size(); ++i) { if (i > 0) { os << ", "; } os << csr_array[i]; } os << "])"; return os; } private: /*! \brief Vector storing all elements contiguously. */ std::vector data_; /*! \brief Vector storing the starting index of each row in data_. */ std::vector indptr_{0}; }; template inline typename CSRArray::Row CSRArray::operator[](int32_t i) const { XGRAMMAR_DCHECK(i >= 0 && i < Size()) << "CSRArray index " << i << " is out of bound"; int32_t start = indptr_[i]; int32_t end = indptr_[i + 1]; return {data_.data() + start, end - start}; } template inline int32_t CSRArray::Insert(const DataType* new_data, int32_t new_data_len) { // TODO(yixin): whether to add a additional data_len // If the new data is already in the CSRArray, we need to copy it to the new memory location. if (new_data >= data_.data() && new_data < data_.data() + data_.size()) { std::vector new_data_copied(new_data, new_data + new_data_len); data_.insert(data_.end(), new_data_copied.begin(), new_data_copied.end()); } else { data_.insert(data_.end(), new_data, new_data + new_data_len); } indptr_.push_back(static_cast(data_.size())); return static_cast(indptr_.size()) - 2; } template inline int32_t CSRArray::Insert(const std::vector& new_data) { data_.insert(data_.end(), new_data.begin(), new_data.end()); indptr_.push_back(static_cast(data_.size())); return static_cast(indptr_.size()) - 2; } template inline int32_t CSRArray::InsertNonContiguous( DataType data_1, const DataType* data_2, int32_t data_2_len ) { if (data_2 >= data_.data() && data_2 < data_.data() + data_.size()) { std::vector new_data_copied(data_2, data_2 + data_2_len); data_.push_back(data_1); data_.insert(data_.end(), new_data_copied.begin(), new_data_copied.end()); } else { data_.push_back(data_1); data_.insert(data_.end(), data_2, data_2 + data_2_len); } indptr_.push_back(static_cast(data_.size())); return static_cast(indptr_.size()) - 2; } template inline picojson::value CSRArray::Serialize() const { // Serialize data_ picojson::array data_json; for (const auto& item : data_) { data_json.push_back(picojson::value(static_cast(item))); } // Serialize indptr_ picojson::array indptr_json; for (const auto& item : indptr_) { indptr_json.push_back(picojson::value(static_cast(item))); } // Serialize the object picojson::object obj; obj["data"] = picojson::value(data_json); obj["indptr"] = picojson::value(indptr_json); return picojson::value(obj); } template inline CSRArray CSRArray::Deserialize(const picojson::value& v) { XGRAMMAR_CHECK(v.is()) << "Failed to deserialize CSRArray: expected a JSON object"; picojson::object obj = v.get(); XGRAMMAR_CHECK(obj.find("data") != obj.end() && obj["data"].is()) << "Failed to parse data in CSRArray"; XGRAMMAR_CHECK(obj.find("indptr") != obj.end() && obj["indptr"].is()) << "Failed to parse indptr in CSRArray"; CSRArray csr_array; // Deserialize data_ csr_array.data_.clear(); for (const auto& item : obj["data"].get()) { csr_array.data_.push_back(static_cast(item.get())); } // Deserialize indptr_ csr_array.indptr_.clear(); for (const auto& item : obj["indptr"].get()) { csr_array.indptr_.push_back(static_cast(item.get())); } return csr_array; } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_CSR_ARRAY_H_ xgrammar-0.1.19/cpp/support/dynamic_bitset.h000066400000000000000000000171421500705317600210760ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/dynamic_bitset.h * \brief The header for utilities used in grammar-guided generation. */ #ifndef XGRAMMAR_SUPPORT_DYNAMIC_BITSET_H_ #define XGRAMMAR_SUPPORT_DYNAMIC_BITSET_H_ #include #include #include #include // For __popcnt #ifdef _MSC_VER #include #endif #include "logging.h" namespace xgrammar { /*! * \brief A bitset whose length is specified at runtime. Note the size cannot be changed after * construction. * \details The buffer of the bitset is a uint32_t array. There are two uses for this class: * - When passing nullptr to data, it maintains an internal buffer for the bitset. * - When passing a pointer to a buffer with enough size, it uses the external buffer for the * bitset. * \details Part of the implementation is adopted from Boost::dynamic_bitset. */ class DynamicBitset { public: /*! * \brief Calculate the minimal size of the uint32_t buffer for the bitset with the given size. * \param element_size The size of the bitset. * \return The minimal buffer size. */ static int GetBufferSize(int element_size) { return (element_size + 31) / 32; } /*! * \brief Construct a empty bitset. This object should be assigned to a valid bitset before using. */ DynamicBitset() : size_(0), buffer_size_(0), data_(nullptr), is_internal_(true) {} /*! * \brief Construct a bitset with the given size. * \param size The size of the bitset. * \param data The buffer for the bitset. If nullptr, the bitset will maintain an internal buffer. */ DynamicBitset(int size, uint32_t* data = nullptr) : size_(size), buffer_size_(GetBufferSize(size)) { if (data == nullptr) { internal_buffer_.resize(buffer_size_, 0); data_ = internal_buffer_.data(); is_internal_ = true; } else { data_ = data; is_internal_ = false; } } /*! \brief Copy assignment. */ DynamicBitset& operator=(const DynamicBitset& other) { XGRAMMAR_DCHECK(is_internal_ || size_ >= other.size_) << "Expanding bitset size is not allowed when the " "memory of the bitset is externally managed"; size_ = other.size_; buffer_size_ = other.buffer_size_; if (is_internal_) { internal_buffer_.reserve(buffer_size_); data_ = internal_buffer_.data(); } if (data_ != other.data_) { std::memcpy(data_, other.data_, buffer_size_ * sizeof(uint32_t)); } return *this; } /*! \brief Move assignment. */ DynamicBitset& operator=(DynamicBitset&& other) { size_ = other.size_; buffer_size_ = other.buffer_size_; is_internal_ = other.is_internal_; if (is_internal_) { internal_buffer_ = std::move(other.internal_buffer_); data_ = internal_buffer_.data(); } else { data_ = other.data_; } return *this; } /*! \brief Get the value of the bit at the given index. */ bool operator[](int index) const { XGRAMMAR_DCHECK(data_ && index >= 0 && index < size_); return (data_[index / 32] >> (index % 32)) & 1; } /*! \brief Get the size of the bitset. */ int Size() const { return size_; } /*! \brief Set the whole bitset to true. */ void Set() { XGRAMMAR_DCHECK(data_); std::memset(data_, 0xFF, buffer_size_ * sizeof(uint32_t)); } /*! \brief Set the bit at the given index to the given value. */ void Set(int index, bool value = true) { XGRAMMAR_DCHECK(data_ && index >= 0 && index < size_); if (value) { data_[index / 32] |= 1 << (index % 32); } else { data_[index / 32] &= ~(1 << (index % 32)); } } /*! \brief Set the whole bitset to false. */ void Reset() { XGRAMMAR_DCHECK(data_); std::memset(data_, 0, buffer_size_ * sizeof(uint32_t)); } /*! \brief Set the bit at the given index to false. */ void Reset(int index) { Set(index, false); } /*! \brief Perform a bitwise OR operation between the current bitset and another bitset. */ DynamicBitset& operator|=(const DynamicBitset& other) { XGRAMMAR_DCHECK(buffer_size_ <= other.buffer_size_); for (int i = 0; i < buffer_size_; ++i) { data_[i] |= other.data_[i]; } return *this; } int FindFirstOne() const { return DoFindOneFrom(0); } int FindNextOne(int pos) const { if (pos >= size_ - 1 || size_ == 0) return -1; ++pos; int blk = pos / BITS_PER_BLOCK; int ind = pos % BITS_PER_BLOCK; uint32_t fore = data_[blk] >> ind; int result = fore ? pos + LowestBit(fore) : DoFindOneFrom(blk + 1); return result < size_ ? result : -1; } int FindFirstZero() const { return DoFindZeroFrom(0); } int FindNextZero(int pos) const { if (pos >= size_ - 1 || size_ == 0) return -1; ++pos; int blk = pos / BITS_PER_BLOCK; int ind = pos % BITS_PER_BLOCK; uint32_t fore = (~data_[blk]) >> ind; int result = fore ? pos + LowestBit(fore) : DoFindZeroFrom(blk + 1); return result < size_ ? result : -1; } int Count() const { int count = 0; for (int i = 0; i < buffer_size_; ++i) { count += PopCount(data_[i]); } return count; } bool All() const { if (size_ == 0) return true; // Check all complete blocks except the last one for (int i = 0; i < buffer_size_ - 1; ++i) { if (data_[i] != ~static_cast(0)) { return false; } } // For the last block, create a mask for valid bits only int remaining_bits = size_ % BITS_PER_BLOCK; uint32_t last_block_mask = remaining_bits ? (static_cast(1) << remaining_bits) - 1 : ~static_cast(0); return (data_[buffer_size_ - 1] & last_block_mask) == last_block_mask; } static constexpr int BITS_PER_BLOCK = 32; friend std::size_t MemorySize(const DynamicBitset& bitset) { return bitset.buffer_size_ * sizeof(bitset.data_[0]); } private: static int LowestBit(uint32_t value) { #ifdef __GNUC__ return __builtin_ctz(value); #else // __GNUC__ // From https://stackoverflow.com/a/757266 static const int MultiplyDeBruijnBitPosition[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; return MultiplyDeBruijnBitPosition[((uint32_t)((value & -value) * 0x077CB531U)) >> 27]; #endif // __GNUC__ } static int PopCount(uint32_t value) { #ifdef __GNUC__ return __builtin_popcount(value); #elif defined(_MSC_VER) return __popcnt(value); #else XGRAMMAR_LOG(FATAL) << "PopCount is not supported on this platform"; #endif } int DoFindZeroFrom(int first_block) const { int position = -1; for (int i = first_block; i < buffer_size_; ++i) { if (data_[i] != ~static_cast(0)) { position = i; break; } } if (position == -1) return -1; return position * BITS_PER_BLOCK + LowestBit(~data_[position]); } int DoFindOneFrom(int first_block) const { int position = -1; for (int i = first_block; i < buffer_size_; ++i) { if (data_[i] != 0) { position = i; break; } } if (position == -1) return -1; return position * BITS_PER_BLOCK + LowestBit(data_[position]); } // The size of the bitset. int size_; // The size of the buffer. int buffer_size_; // The buffer for the bitset. uint32_t* data_; // The internal buffer. It is empty if not needed. std::vector internal_buffer_; // Whether the buffer is internally managed. bool is_internal_; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_DYNAMIC_BITSET_H_ xgrammar-0.1.19/cpp/support/encoding.cc000066400000000000000000000161101500705317600200160ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/encoding.cc */ #include "encoding.h" #include #include "logging.h" namespace xgrammar { std::string PrintAsUTF8(TCodepoint codepoint) { XGRAMMAR_ICHECK(codepoint <= 0x10FFFF) << "Invalid codepoint: " << codepoint; std::string utf8; if (codepoint <= 0x7F) { // 1-byte sequence utf8 += static_cast(codepoint); } else if (codepoint <= 0x7FF) { // 2-byte sequence utf8 += static_cast(0xC0 | ((codepoint >> 6) & 0x1F)); utf8 += static_cast(0x80 | (codepoint & 0x3F)); } else if (codepoint <= 0xFFFF) { // 3-byte sequence utf8 += static_cast(0xE0 | ((codepoint >> 12) & 0x0F)); utf8 += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); utf8 += static_cast(0x80 | (codepoint & 0x3F)); } else { // 4-byte sequence utf8 += static_cast(0xF0 | ((codepoint >> 18) & 0x07)); utf8 += static_cast(0x80 | ((codepoint >> 12) & 0x3F)); utf8 += static_cast(0x80 | ((codepoint >> 6) & 0x3F)); utf8 += static_cast(0x80 | (codepoint & 0x3F)); } return utf8; } std::string PrintAsEscapedUTF8( TCodepoint codepoint, const std::unordered_map& additional_escape_map ) { static const std::unordered_map kCodepointToEscape = { {'\'', "\\\'"}, {'\"', "\\\""}, {'\?', "\\?"}, {'\\', "\\\\"}, {'\a', "\\a"}, {'\b', "\\b"}, {'\f', "\\f"}, {'\n', "\\n"}, {'\r', "\\r"}, {'\t', "\\t"}, {'\v', "\\v"}, {'\0', "\\0"}, {'\x1B', "\\e"} }; if (auto it = additional_escape_map.find(codepoint); it != additional_escape_map.end()) { return it->second; } if (auto it = kCodepointToEscape.find(codepoint); it != kCodepointToEscape.end()) { return it->second; } if (codepoint >= 0x20 && codepoint <= 0x7E) { return std::string({static_cast(codepoint)}); } // convert codepoint to hex char prefix = codepoint <= 0xFF ? 'x' : codepoint <= 0xFFFF ? 'u' : 'U'; int width = codepoint <= 0xFF ? 2 : codepoint <= 0xFFFF ? 4 : 8; std::stringstream ss; ss << std::setfill('0') << std::setw(width) << std::hex << codepoint; auto hex = ss.str(); return std::string("\\") + prefix + hex; } std::string PrintAsEscapedUTF8(uint8_t raw_char) { return PrintAsEscapedUTF8(static_cast(raw_char)); } std::string PrintAsEscapedUTF8(std::string raw_str) { std::string res; auto codepoints = ParseUTF8(raw_str.c_str(), true); for (auto c : codepoints) { res += PrintAsEscapedUTF8(c); } return res; } std::tuple HandleUTF8FirstByte(uint8_t byte) { static const std::array kFirstByteMask = {0x00, 0x7F, 0x1F, 0x0F, 0x07}; // clang-format off static const std::array kUtf8Bytes = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1, }; // clang-format on auto num_bytes = kUtf8Bytes[static_cast(byte)]; if (num_bytes == -1) { return {false, 0, 0}; } return {true, num_bytes, byte & kFirstByteMask[num_bytes]}; } /*! * \brief Parse the first codepoint in a UTF-8 string. * \param utf8 The UTF-8 string. * \return The codepoint and new pointer. If the UTF-8 string is invalid, and the error policy is * kReturnInvalid, the function returns (CharHandlingError::kInvalidUTF8, input char pointer). */ std::pair ParseNextUTF8(const char* utf8, bool return_byte_on_error) { auto [accepted, num_bytes, res] = HandleUTF8FirstByte(utf8[0]); if (accepted) { for (int i = 1; i < num_bytes; ++i) { if (utf8[i] == 0 || (static_cast(utf8[i]) & 0xC0) != 0x80) { // invalid utf8 accepted = false; break; } res = (res << 6) | (static_cast(utf8[i]) & 0x3F); } } if (!accepted) { // invalid utf8 if (return_byte_on_error) { return {static_cast(utf8[0]), utf8 + 1}; } else { return {CharHandlingError::kInvalidUTF8, utf8}; } } return {res, utf8 + num_bytes}; } std::vector ParseUTF8(const char* utf8, bool return_byte_on_error) { std::vector codepoints; while (*utf8 != 0) { TCodepoint codepoint; std::tie(codepoint, utf8) = ParseNextUTF8(utf8, return_byte_on_error); if (codepoint == CharHandlingError::kInvalidUTF8) { return {codepoint}; } codepoints.push_back(codepoint); } return codepoints; } std::pair ParseNextUTF8OrEscaped( const char* utf8, const std::unordered_map& additional_escape_map ) { static const std::unordered_map kEscapeToCodepoint = { {'\'', '\''}, {'\"', '\"'}, {'?', '\?'}, {'\\', '\\'}, {'a', '\a'}, {'b', '\b'}, {'f', '\f'}, {'n', '\n'}, {'r', '\r'}, {'t', '\t'}, {'v', '\v'}, {'0', '\0'}, {'e', '\x1B'} }; if (utf8[0] != '\\') { auto result = ParseNextUTF8(utf8, false); return {result.first, result.second - utf8}; } if (auto it = additional_escape_map.find(utf8[1]); it != additional_escape_map.end()) { return {it->second, 2}; } if (auto it = kEscapeToCodepoint.find(utf8[1]); it != kEscapeToCodepoint.end()) { return {it->second, 2}; } if (utf8[1] == 'x') { // arbitrary length hex int len = 0; TCodepoint codepoint = 0; int32_t digit; while ((digit = HexCharToInt(utf8[2 + len])) != -1) { codepoint = codepoint * 16 + digit; ++len; } if (len == 0) { return {CharHandlingError::kInvalidEscape, 0}; } return {codepoint, len + 2}; } else if (utf8[1] == 'u' || utf8[1] == 'U') { // 4- or 8-digit hex int len = utf8[1] == 'u' ? 4 : 8; TCodepoint codepoint = 0; for (int i = 0; i < len; ++i) { auto digit = HexCharToInt(utf8[i + 2]); if (digit == -1) { return {CharHandlingError::kInvalidEscape, 0}; } codepoint = codepoint * 16 + digit; } return {codepoint, len + 2}; } else { return {CharHandlingError::kInvalidEscape, 0}; } } } // namespace xgrammar xgrammar-0.1.19/cpp/support/encoding.h000066400000000000000000000124251500705317600176650ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/encoding.h * \brief Encoding and decoding from/to UTF-8 and escape sequence to/from codepoints. */ #ifndef XGRAMMAR_SUPPORT_ENCODING_H_ #define XGRAMMAR_SUPPORT_ENCODING_H_ // TODO(yixin): enhance performance #include #include #include #include namespace xgrammar { /*! \brief Represents a unicode codepoint. */ using TCodepoint = int32_t; /*! * \brief Handle the utf-8 first byte. * \returns (is_valid, total_number_of_bytes, initial_codepoint). */ std::tuple HandleUTF8FirstByte(uint8_t byte); /*! * \brief Print a codepoint to a UTF-8 string. * \param codepoint The codepoint. * \return The UTF-8 string. */ std::string PrintAsUTF8(TCodepoint codepoint); /*! * \brief Print a codepoint to a escaped string. If the codepoint is not printable, it will be * escaped. By default the function support escape sequences in C ("\n", "\t", "\u0123"). User can * specify more escape sequences using additional_escape_map. * \param codepoint The codepoint. * \param additional_escape_map A map from codepoint to escape sequence. If the codepoint is in the * map, it will be escaped using the corresponding escape sequence. e.g. {{'-', "\\-"}}. \return The * printable string. */ std::string PrintAsEscapedUTF8( TCodepoint codepoint, const std::unordered_map& additional_escape_map = {} ); /*! * \brief Print the given char to a escaped string that can be printed. * \return The escaped string. */ std::string PrintAsEscapedUTF8(uint8_t raw_char); /*! * \brief Print the given string to a escaped string that can be printed. * \return The escaped string. */ std::string PrintAsEscapedUTF8(std::string raw_str); inline int HexCharToInt(char c) { if (c >= '0' && c <= '9') { return c - '0'; } else if (c >= 'a' && c <= 'f') { return c - 'a' + 10; } else if (c >= 'A' && c <= 'F') { return c - 'A' + 10; } else { return -1; } } /*! * \brief Represents an error when handling characters. Will be returned as a special TCodepoint * value. */ enum CharHandlingError : TCodepoint { /*! \brief The UTF-8 string is invalid. */ kInvalidUTF8 = -10, /*! \brief The escape sequence is invalid. */ kInvalidEscape = -11, }; /*! * \brief Parse all codepoints in a UTF-8 string. * \param utf8 The UTF-8 string. * \return All codepoints. If the UTF-8 string is invalid, and the error policy is * kReturnInvalid, the function returns {CharHandlingError::kInvalidUTF8}. */ std::vector ParseUTF8(const char* utf8, bool return_byte_on_error = false); template std::pair HandleEscape( const CharType* data, const std::unordered_map& additional_escape_map = {} ); /*! * \brief Parse the first codepoint from a UTF-8 string. Also checks escape sequences and converts * the escaped char to its original value. * \param utf8 The UTF-8 string or the escape sequence. * \param additional_escape_map A map from escape sequence to codepoint. If the escape sequence is * in the map, it will be converted to the corresponding codepoint. e.g. {{"\\-", '-'}}. * \return The codepoint and the new pointer. If the UTF-8 string or the escape sequence is * invalid, and the error policy is kReturnInvalid, the function returns * (CharHandlingError::kInvalidUTF8, input char pointer). */ std::pair ParseNextUTF8OrEscaped( const char* utf8, const std::unordered_map& additional_escape_map = {} ); // Template implementation template std::pair HandleEscape( const CharType* data, const std::unordered_map& additional_escape_map ) { static const std::unordered_map kEscapeToCodepoint = { {'\'', '\''}, {'\"', '\"'}, {'?', '\?'}, {'\\', '\\'}, {'a', '\a'}, {'b', '\b'}, {'f', '\f'}, {'n', '\n'}, {'r', '\r'}, {'t', '\t'}, {'v', '\v'}, {'0', '\0'}, {'e', '\x1B'} }; if (data[0] != '\\') { return {CharHandlingError::kInvalidEscape, 0}; } if (auto it = additional_escape_map.find(static_cast(data[1])); it != additional_escape_map.end()) { return {it->second, 2}; } if (auto it = kEscapeToCodepoint.find(static_cast(data[1])); it != kEscapeToCodepoint.end()) { return {it->second, 2}; } if (data[1] == 'x') { // arbitrary length hex int len = 0; TCodepoint codepoint = 0; int32_t digit; while ((digit = HexCharToInt(data[2 + len])) != -1) { codepoint = codepoint * 16 + digit; ++len; } if (len == 0) { return {CharHandlingError::kInvalidEscape, 0}; } return {codepoint, len + 2}; } else if (data[1] == 'u' || data[1] == 'U') { // 4- or 8-digit hex int len = data[1] == 'u' ? 4 : 8; TCodepoint codepoint = 0; for (int i = 0; i < len; ++i) { auto digit = HexCharToInt(data[i + 2]); if (digit == -1) { return {CharHandlingError::kInvalidEscape, 0}; } codepoint = codepoint * 16 + digit; } return {codepoint, len + 2}; } else { return {CharHandlingError::kInvalidEscape, 0}; } } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_ENCODING_H_ xgrammar-0.1.19/cpp/support/int_set.h000066400000000000000000000042371500705317600175460ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/int_set.h * \brief The header for utilities used in grammar-guided generation. */ #ifndef XGRAMMAR_SUPPORT_INT_SET_H_ #define XGRAMMAR_SUPPORT_INT_SET_H_ #include #include #include #include namespace xgrammar { /*! * \brief Let lhs be the union of lhs and rhs. Suppose that both sets are sorted. * \note No additional vectors are allocated, and the time complexity is O(n) */ inline void IntsetUnion(std::vector* lhs, const std::vector& rhs) { int original_lhs_size = lhs->size(); int rhs_size = rhs.size(); lhs->resize(original_lhs_size + rhs_size); auto it_lhs = lhs->rbegin() + rhs_size; auto it_rhs = rhs.rbegin(); auto it_result = lhs->rbegin(); while (it_lhs != lhs->rend() && it_rhs != rhs.rend()) { if (*it_lhs > *it_rhs) { *it_result = *it_lhs; ++it_lhs; } else if (*it_lhs < *it_rhs) { *it_result = *it_rhs; ++it_rhs; } else { *it_result = *it_lhs; ++it_lhs; ++it_rhs; } ++it_result; } while (it_rhs != rhs.rend()) { *it_result = *it_rhs; ++it_result; ++it_rhs; } auto last = std::unique(lhs->begin(), lhs->end()); lhs->erase(last, lhs->end()); } /*! * \brief Let lhs be the intersection of lhs and rhs. Suppose that both sets are sorted. * \note No additional vector is allocated, and the time complexity is O(n). * \note Support the case where lhs is the universal set by setting lhs to {-1}. The result will be * rhs then. */ inline void IntsetIntersection(std::vector* lhs, const std::vector& rhs) { if (lhs->size() == 1 && (*lhs)[0] == -1) { *lhs = rhs; return; } auto it_lhs = lhs->begin(); auto it_rhs = rhs.begin(); auto it_result = lhs->begin(); while (it_lhs != lhs->end() && it_rhs != rhs.end()) { if (*it_lhs < *it_rhs) { ++it_lhs; } else if (*it_lhs > *it_rhs) { ++it_rhs; } else { *it_result = *it_lhs; ++it_lhs; ++it_rhs; ++it_result; } } lhs->erase(it_result, lhs->end()); } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_INT_SET_H_ xgrammar-0.1.19/cpp/support/logging.cc000066400000000000000000000010111500705317600176500ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/logging.cc */ #include "logging.h" namespace xgrammar { #if XGRAMMAR_LOG_CUSTOMIZE == 0 LogFatal::Entry& LogFatal::GetEntry() { static thread_local LogFatal::Entry result; return result; } const char* LogMessage::level_strings_[] = { ": ", // XGRAMMAR_LOG_LEVEL_INFO ": Debug: ", // XGRAMMAR_LOG_LEVEL_DEBUG ": Warning: ", // XGRAMMAR_LOG_LEVEL_WARNING }; #endif // XGRAMMAR_LOG_CUSTOMIZE } // namespace xgrammar xgrammar-0.1.19/cpp/support/logging.h000066400000000000000000000153341500705317600175270ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/logging.h * \brief A logging library that supports logging at different levels. */ #ifndef XGRAMMAR_SUPPORT_LOGGING_H_ #define XGRAMMAR_SUPPORT_LOGGING_H_ #include #include #include #include #include /*! * \brief Whether or not customize the logging output. * If log customize is enabled, the user must implement * xgrammar::LogFatalImpl and xgrammar::LogMessageImpl. */ #ifndef XGRAMMAR_LOG_CUSTOMIZE #define XGRAMMAR_LOG_CUSTOMIZE 0 #endif namespace xgrammar { /*! * \brief Base error class in XGrammar. */ class Error : public std::runtime_error { public: explicit Error(const std::string& msg, const std::string& type = "") : std::runtime_error(msg), type_(type) { if (!type_.empty()) { full_message_ = type_ + ": " + std::runtime_error::what(); } else { full_message_ = std::runtime_error::what(); } } virtual ~Error() = default; virtual const std::string& type() const noexcept { return type_; } virtual const char* what() const noexcept override { return full_message_.c_str(); } protected: std::string type_; std::string full_message_; }; /*! * \brief Error type for errors from XGRAMMAR_CHECK, XGRAMMAR_ICHECK, and XGRAMMAR_LOG(FATAL). This * error contains a backtrace of where it occurred. */ class LoggingError : public Error { public: /*! \brief Construct an error. Not recommended to use directly. Instead use XGRAMMAR_LOG(FATAL). * * \param file The file where the error occurred. * \param lineno The line number where the error occurred. * \param message The error message to display. * \param time The time at which the error occurred. This should be in local time. */ LoggingError( std::string file, int lineno, std::string message, std::time_t time = std::time(nullptr) ) : Error("", "LoggingError"), file_(file), lineno_(lineno), time_(time) { std::ostringstream s; s << "[" << std::put_time(std::localtime(&time), "%H:%M:%S") << "] " << file << ":" << lineno << ": " << message << std::endl; full_message_ = s.str(); } /*! \return The file in which the error occurred. */ const std::string& file() const { return file_; } /*! \return The time at which this error occurred. */ const std::time_t& time() const { return time_; } /*! \return The line number at which this error occurred. */ int lineno() const { return lineno_; } private: std::string file_; int lineno_; std::time_t time_; }; // Provide support for customized logging. #if XGRAMMAR_LOG_CUSTOMIZE /*! * \brief Custom implementations of LogFatal. * * \sa XGRAMMAR_LOG_CUSTOMIZE */ [[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message); /*! * \brief Custom implementations of LogMessage. * * \sa XGRAMMAR_LOG_CUSTOMIZE */ void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message); /*! * \brief Class to accumulate an error message and throw it. Do not use * directly, instead use LOG(FATAL). */ class LogFatal { public: LogFatal(const std::string& file, int lineno) : file_(file), lineno_(lineno) {} #ifdef _MSC_VER #pragma disagnostic push #pragma warning(disable : 4722) #endif [[noreturn]] ~LogFatal() noexcept(false) { LogFatalImpl(file_, lineno_, stream_.str()); } #ifdef _MSC_VER #pragma disagnostic pop #endif std::ostringstream& stream() { return stream_; } private: std::ostringstream stream_; std::string file_; int lineno_; }; /*! * \brief Class to accumulate an log message. Do not use directly, instead use * LOG(INFO), LOG(WARNING), LOG(ERROR). */ class LogMessage { public: LogMessage(const std::string& file, int lineno, int level) : file_(file), lineno_(lineno), level_(level) {} ~LogMessage() { LogMessageImpl(file_, lineno_, level_, stream_.str()); } std::ostringstream& stream() { return stream_; } private: std::string file_; int lineno_; int level_; std::ostringstream stream_; }; #else // if XGRAMMAR_LOG_CUSTOMIZE /*! * \brief Class to accumulate an error message and throw it. Do not use * directly, instead use XGRAMMAR_LOG(FATAL). * \note The `LogFatal` class is designed to be an empty class to reduce stack size usage. * To play this trick, we use the thread-local storage to store its internal data. */ class LogFatal { public: LogFatal(const char* file, int lineno) { GetEntry().Init(file, lineno); } #ifdef _MSC_VER #pragma disagnostic push #pragma warning(disable : 4722) #endif [[noreturn]] ~LogFatal() noexcept(false) { GetEntry().Finalize(); throw; } #ifdef _MSC_VER #pragma disagnostic pop #endif std::ostringstream& stream() { return GetEntry().stream_; } private: struct Entry { void Init(const char* file, int lineno) { this->stream_.str(""); this->file_ = file; this->lineno_ = lineno; } [[noreturn]] LoggingError Finalize() noexcept(false) { LoggingError error(file_, lineno_, stream_.str()); throw error; } std::ostringstream stream_; std::string file_; int lineno_; }; static Entry& GetEntry(); }; /*! * \brief Class to accumulate an log message. Do not use directly, instead use * XGRAMMAR_LOG(INFO), XGRAMMAR_LOG(WARNING), XGRAMMAR_LOG(ERROR). */ class LogMessage { public: LogMessage(const std::string& file, int lineno, int level) { std::time_t t = std::time(nullptr); stream_ << "[" << std::put_time(std::localtime(&t), "%H:%M:%S") << "] " << file << ":" << lineno << level_strings_[level]; } ~LogMessage() { std::cerr << stream_.str() << std::endl; } std::ostringstream& stream() { return stream_; } private: std::ostringstream stream_; static const char* level_strings_[]; }; #endif // XGRAMMAR_LOG_CUSTOMIZE #define XGRAMMAR_LOG_LEVEL_INFO 0 #define XGRAMMAR_LOG_LEVEL_DEBUG 1 #define XGRAMMAR_LOG_LEVEL_WARNING 2 #define XGRAMMAR_LOG_LEVEL_FATAL 3 #define XGRAMMAR_LOG_INFO LogMessage(__FILE__, __LINE__, XGRAMMAR_LOG_LEVEL_INFO).stream() #define XGRAMMAR_LOG_DEBUG LogMessage(__FILE__, __LINE__, XGRAMMAR_LOG_LEVEL_DEBUG).stream() #define XGRAMMAR_LOG_WARNING LogMessage(__FILE__, __LINE__, XGRAMMAR_LOG_LEVEL_WARNING).stream() #define XGRAMMAR_LOG_FATAL LogFatal(__FILE__, __LINE__).stream() #define XGRAMMAR_LOG(level) XGRAMMAR_LOG_##level #define XGRAMMAR_CHECK(x) \ if (!(x)) LogFatal(__FILE__, __LINE__).stream() << "Check failed: (" #x << ") is false: " #define XGRAMMAR_ICHECK(x) \ if (!(x)) LogFatal(__FILE__, __LINE__).stream() << "Internal check failed: (" #x << ") is false: " #ifndef NDEBUG #define XGRAMMAR_DCHECK(x) XGRAMMAR_ICHECK(x) #else #define XGRAMMAR_DCHECK(x) \ while (false) XGRAMMAR_ICHECK(x) #endif // NDEBUG } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_LOGGING_H_ xgrammar-0.1.19/cpp/support/thread_pool.h000066400000000000000000000145511500705317600204010ustar00rootroot00000000000000/*! * Copyright (c) 2023 by Contributors * \file support/thread_pool.h * \brief Thread pool. */ #ifndef XGRAMMAR_SUPPORT_THREAD_POOL_H_ #define XGRAMMAR_SUPPORT_THREAD_POOL_H_ #include #include #include #include #include #include #include #include #include "logging.h" namespace xgrammar { /*! * \brief A thread pool implementation for parallel task execution. * * ThreadPool manages a pool of worker threads that can execute tasks asynchronously. * Tasks are submitted to a queue and executed by available threads from the pool. * The pool automatically handles thread synchronization and task distribution. */ class ThreadPool { public: /*! * \brief Construct a new thread pool with the specified number of threads. * \param num_threads Number of worker threads to create. Defaults to hardware concurrency. * \note The pool starts the worker threads immediately upon construction. */ ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) { // Initialize thread pool with num_threads threads for (size_t i = 0; i < num_threads; ++i) { workers_.emplace_back([this] { while (true) { std::function task; { // Lock queue while waiting for new task std::unique_lock lock(queue_mutex_); queue_condition_.wait(lock, [this] { return shutdown_ || !task_queue_.empty(); }); // Exit thread if shutdown and queue is empty if (shutdown_ && task_queue_.empty()) return; // Get task from queue task = std::move(task_queue_.front()); task_queue_.pop(); } task(); TaskComplete(); } }); } } /*! * \brief Add a new task to be executed by the thread pool. * \tparam F Type of the function to execute * \tparam Args Types of the arguments to pass to the function * \param f Function to execute * \param args Arguments to pass to the function * \return std::shared_future containing the result of the function call * \note Tasks are executed in FIFO order but may complete in any order. */ template auto Submit(F&& f, Args&&... args) -> std::shared_future> { using return_type = std::invoke_result_t; // Package the task with its arguments into a shared pointer auto task = std::make_shared>( std::bind(std::forward(f), std::forward(args)...) ); std::shared_future res = task->get_future().share(); { std::unique_lock lock(queue_mutex_); XGRAMMAR_CHECK(!shutdown_) << "Cannot submit task to stopped ThreadPool"; ++unfinished_task_count_; // Increment task count // Directly add the task without wrapping task_queue_.emplace([task]() { (*task)(); }); } queue_condition_.notify_one(); return res; } /*! * \brief Add a new task to be executed by the thread pool without returning a future. * \tparam F Type of the function to execute * \tparam Args Types of the arguments to pass to the function * \param f Function to execute * \param args Arguments to pass to the function * \note Tasks are executed asynchronously by the worker threads. */ template void Execute(F&& f, Args&&... args) { { std::unique_lock lock(queue_mutex_); XGRAMMAR_CHECK(!shutdown_) << "Cannot execute task in stopped ThreadPool"; ++unfinished_task_count_; // Increment task count // Directly add the task without wrapping task_queue_.emplace(std::bind(std::forward(f), std::forward(args)...)); } queue_condition_.notify_one(); } void Wait() { std::unique_lock lock(queue_mutex_); tasks_done_condition_.wait(lock, [this] { return unfinished_task_count_ == 0; }); } /*! * \brief Join all threads in the pool. * * Sets shutdown flag and waits for all threads to complete their current tasks * before destroying the pool. Any remaining tasks in the queue will be executed * before shutdown completes. */ void Join() { { std::unique_lock lock(queue_mutex_); if (shutdown_) return; // Already shut down shutdown_ = true; } queue_condition_.notify_all(); // Wake up all threads so they can exit for (std::thread& worker : workers_) { if (worker.joinable()) worker.join(); // Wait for thread to finish } } /*! * \brief Destructor that ensures graceful shutdown of the thread pool. */ ~ThreadPool() { Join(); } // Prevent copying or moving of the thread pool ThreadPool(const ThreadPool&) = delete; ThreadPool(ThreadPool&&) = delete; ThreadPool& operator=(const ThreadPool&) = delete; ThreadPool& operator=(ThreadPool&&) = delete; private: void TaskComplete() { std::unique_lock lock(queue_mutex_); --unfinished_task_count_; if (unfinished_task_count_ == 0) { tasks_done_condition_.notify_all(); // Notify waiting threads } } /*! \brief Thread container */ std::vector workers_; /*! \brief Task queue */ std::queue> task_queue_; /*! \brief Mutex to protect task queue */ std::mutex queue_mutex_; /*! \brief Condition variable for thread synchronization */ std::condition_variable queue_condition_; /*! \brief Condition variable for task completion */ std::condition_variable tasks_done_condition_; /*! \brief Flag to indicate thread pool shutdown */ bool shutdown_ = false; /*! \brief Number of unfinished tasks */ int unfinished_task_count_ = 0; }; inline void ParallelFor(int low, int high, int num_threads, std::function f) { if (high - low == 1) { f(low); return; } ThreadPool pool(num_threads); int total = high - low; int chunk_size = (total + num_threads - 1) / num_threads; for (int t = 0; t < num_threads; ++t) { int start = low + t * chunk_size; int end = std::min(start + chunk_size, high); if (start >= end) break; // No more iterations to process pool.Execute([f, start, end]() { for (int i = start; i < end; ++i) { f(i); } }); } pool.Join(); } } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_THREAD_POOL_H_ xgrammar-0.1.19/cpp/support/thread_safe_cache.h000066400000000000000000000321131500705317600214630ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/thread_safe_cache.h * \brief The header for thread-safe caching functionality. */ #ifndef XGRAMMAR_SUPPORT_THREAD_SAFE_CACHE_H_ #define XGRAMMAR_SUPPORT_THREAD_SAFE_CACHE_H_ #include #include // IWYU pragma: keep #include #include #include #include #include #include #include #include #include "container.h" namespace xgrammar { /*! * \brief Primary template for ThreadSafeCache * \details This class provides thread-safe caching functionality in two forms: * 1. Single value cache when only Value template parameter is provided * 2. Key-value cache when both Key and Value template parameters are provided */ template class ThreadSafeCache; /*! * \brief Thread-safe cache for a single computed value * \tparam Value The type of value being cached * \details Specialization that provides: * - Thread-safe access to a single cached value * - Lazy computation on first access * - Reader-writer locking for concurrent reads */ template class ThreadSafeCache { public: /*! * \brief Constructs a new single-value cache * \param compute The function that computes the cached value */ explicit ThreadSafeCache(std::function compute) : compute_(std::move(compute)) {} /*! * \brief Gets or computes the cached value * \return The cached or newly computed value */ Value Get() { // First try reading from cache with shared lock { std::shared_lock cache_lock(cache_mutex_); if (cache_.has_value()) { return cache_.value(); // Cache hit } } // Acquire exclusive lock to compute value std::unique_lock cache_lock(cache_mutex_); // Double-check to prevent redundant computation if (cache_.has_value()) { return cache_.value(); } Value value = compute_(); XGRAMMAR_DCHECK(!cache_.has_value()); cache_ = value; return value; } /*! * \brief Clears the cached value * This function removes the cached value, so the next call to Get() will recompute it. */ void Clear() { std::unique_lock cache_lock(cache_mutex_); cache_.reset(); } private: /*! \brief Optional container holding the cached value */ std::optional cache_; /*! \brief Function used to compute the value when not cached */ std::function compute_; /*! \brief Reader-writer lock protecting access to cache_ */ std::shared_mutex cache_mutex_; }; /*! * \brief A thread-safe key-value cache with on-demand computation * \tparam Key The type of keys used to lookup values. Should be hashable. * \tparam Value The type of values stored in the cache * \details This cache provides thread-safe access to computed values with the following features: * - Lazy computation: Values are only computed when first requested * - Thread safety: Uses reader-writer locks for concurrent reads * - Parallel computation: Different keys can be computed simultaneously * - Double-checked locking: Prevents redundant computation */ template class ThreadSafeCache { public: /*! * \brief Constructs a new thread-safe cache * \param compute The function that computes values for uncached keys */ explicit ThreadSafeCache(std::function compute) : compute_(std::move(compute)) {} /*! * \brief Gets or computes the value for a key * \param key The key to lookup * \return The cached or newly computed value of the key */ Value Get(const Key& key) { // Why we need this: // - When adding new elements to a unordered_map, the map may be rehashed, // - which means all the iterators may be invalidated. // - However, cppreference says: // - "References and pointers to either key or data stored in the container are only invalidated // - by erasing that element, even when the corresponding iterator is invalidated." // - (See https://en.cppreference.com/w/cpp/container/unordered_map) // - Therefore, we should maintain 2 locks. // - When we add something to the cache, we should hold the cache_mutex_. // - When we erase something from the cache, we should hold the clear_mutex_. auto erase_lock = std::shared_lock(erase_mutex_); // First attempt to read from cache_ { auto cache_lock = std::shared_lock(cache_mutex_); auto it = cache_.find(key); if (it != cache_.end()) { // Cache hit auto& entry = it->second; // The iterator is invalidated after releasing the lock cache_lock.unlock(); // Therefore, we should hold the entry by reference first // We should not hold lock here, since this function may be blocking. return entry.get(compute_, key); } } // Acquire exclusive lock to compute value { auto cache_lock = std::unique_lock(cache_mutex_); auto& entry = cache_[key]; // Create a new entry cache_lock.unlock(); // Release the lock before blocking // We should not hold lock here, since this function may be blocking. return entry.get(compute_, key); } } /*! * \brief Clears all cached values and associated per-key mutexes * This function removes all cached key-value pairs, so subsequent calls to Get() will recompute * them. */ void Clear() { auto erase_lock = std::unique_lock(erase_mutex_); cache_.clear(); } private: struct Entry { Value value; std::once_flag flag; const Value& get(const std::function& f, const Key& key) { // block in this lambda until the value is computed std::call_once(flag, [&] { value = f(key); }); return value; } }; /*! \brief The cache mapping keys to computed values */ std::unordered_map cache_; /*! \brief The function used to compute values for uncached keys */ std::function compute_; /*! \brief Reader-writer lock protecting access to cache_ */ std::shared_mutex cache_mutex_; /*! \brief Mutex protecting removing elements */ std::shared_mutex erase_mutex_; }; namespace details { template class LRUCacheImpl { public: struct Entry { Value value; // value of the node int index; // node index }; /*! \brief Visits the node and moves it to the back of the LRU list. Return its value. */ const Value& LRUVisit(const std::pair& pair) { const auto& entry = pair.second; lru_list_.MoveBack(entry.index); return entry.value; } /*! \brief Initializes the node with the given value and moves it to the back of the LRU list. */ void LRUInit(std::pair& pair, const Value& init) { auto& entry = pair.second; entry.value = init; entry.index = lru_list_.PushBack(&pair).Index(); } /*! * \brief Evicts the least recently used nodes until the predicate returns false. * \param predicate The function that returns true if eviction should continue. * \param evict The function takes a value and returns true if the value can be evicted. * This will be only called when the predicate returns true. * If this function returns true, it should update the size information before return. * \details This function will evict the least recently used nodes until the predicate returns * false. The evict function will be called for each node to determine if it should be evicted. */ template void LRUEvict(const Predicate& predicate, const Evict& evict) { if (!predicate()) return; auto iter = lru_list_.begin(); if (iter == lru_list_.end()) return; do { auto& [key, entry] = **iter; if (evict(entry.value)) { iter = lru_list_.Erase(iter); map_.erase(key); } else { ++iter; // simply skip those waiting for computation } } while (predicate() && iter != lru_list_.end()); } std::unordered_map& GetMap() { return map_; } private: std::unordered_map map_; List*> lru_list_; }; } // namespace details /** * \brief A thread-safe key-value cache with on-demand computation and LRU eviction * \tparam Key The type of keys used to lookup values. Should be hashable. * \tparam Value The type of values stored in the cache * \tparam Computer The functor that computes values for uncached keys * \tparam SizeEstimator The functor that estimates the size of a value in bytes * \details This cache provides thread-safe access to computed values with the following features: * - Lazy computation: Values are only computed when first requested * - LRU eviction: When the cache is full, the least recently used value is evicted * - Thread safety: Uses reader-writer locks for concurrent reads * \attention User should guarantee the following: * 1. The policy class should provide a compute method that takes a key and returns a value. * 2. The value type should have a MemorySize method that returns the size of the value in bytes. */ template class ThreadSafeLRUCache { private: struct SizedValue { Value value; std::size_t size; }; public: inline static constexpr std::size_t UNLIMITED_SIZE = static_cast(-1); explicit ThreadSafeLRUCache( std::size_t max_size = UNLIMITED_SIZE, const Computer& computer = Computer{}, const SizeEstimator& size_estimator = SizeEstimator{} ) : max_size_(max_size), computer_(computer), size_estimator_(size_estimator), cache_() {} std::size_t MaxMemorySize() const { return max_size_; } std::size_t MemorySize() const { return current_size_; } Value Get(const Key& key) { auto future = GetFuture(key); return future.get().value; } void Clear() { // Remove all the ready entries. const auto lock_map = std::lock_guard{map_mutex_}; cache_.LRUEvict( [] { return true; }, [&](const std::shared_future& value) { // always evict and block until the value is ready current_size_ -= value.get().size; return true; } ); } private: std::shared_future GetFuture(const Key& key) { if (this->max_size_ == UNLIMITED_SIZE) return GetFutureUnlimited(key); auto& map = cache_.GetMap(); { auto lock_map = std::shared_lock{map_mutex_}; auto it = map.find(key); if (it != map.end()) { // We only need to hold LRU lock when shared lock is held here. // When unique lock of map_mutex_ is held, only 1 thread can access the // LRU list at the same time, so we do not need to hold the LRU lock then. const auto lock_lru = std::lock_guard{lru_mutex_}; return cache_.LRUVisit(*it); } } auto task = std::packaged_task{[this, &key] { auto result = SizedValue(); result.value = computer_(key); result.size = size_estimator_(result.value); current_size_ += result.size; return result; }}; auto lock_map = std::unique_lock{map_mutex_}; auto [it, success] = map.try_emplace(key); if (!success) return cache_.LRUVisit(*it); // in this case, we insert the task, and we need to compute the value auto future = task.get_future().share(); // perform eviction if the cache is full cache_.LRUInit(*it, future); cache_.LRUEvict( [&] { return current_size_ > max_size_; }, [&](const std::shared_future& value) { using namespace std::chrono_literals; // if not ready, then do not wait and block here if (value.wait_for(0s) != std::future_status::ready) return false; current_size_ -= value.get().size; return true; } ); // perform the costly computation outside all locks lock_map.unlock(); task(); return future; } std::shared_future GetFutureUnlimited(const Key& key) { auto& map = cache_.GetMap(); { auto lock_map = std::shared_lock{map_mutex_}; auto it = map.find(key); if (it != map.end()) return it->second.value; } auto task = std::packaged_task{[this, &key] { auto result = SizedValue(); result.value = computer_(key); result.size = size_estimator_(result.value); current_size_ += result.size; return result; }}; auto lock_map = std::unique_lock{map_mutex_}; auto [it, success] = map.try_emplace(key); if (!success) return it->second.value; auto future = task.get_future().share(); it->second.value = future; // perform the costly computation outside all locks lock_map.unlock(); task(); return future; } private: const std::size_t max_size_; const Computer computer_; const SizeEstimator size_estimator_; details::LRUCacheImpl> cache_; std::atomic_size_t current_size_{0}; std::shared_mutex map_mutex_; std::mutex lru_mutex_; }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_THREAD_SAFE_CACHE_H_ xgrammar-0.1.19/cpp/support/union_find_set.h000066400000000000000000000060261500705317600211020ustar00rootroot00000000000000/*! * Copyright (c) 2025 by Contributors * \file xgrammar/support/union_find_set.h */ #ifndef XGRAMMAR_SUPPORT_UNION_FIND_SET_H_ #define XGRAMMAR_SUPPORT_UNION_FIND_SET_H_ #include #include #include #include namespace xgrammar { template class UnionFindSet { private: std::unordered_map parent; std::unordered_map rank; public: UnionFindSet() = default; ~UnionFindSet() = default; /*! \brief Insert a new element into the union-find set. \param value The value to be inserted. \return true if the value was successfully inserted, false if it already exists. */ bool Make(const T& value) { if (parent.find(value) != parent.end()) { return false; } parent[value] = value; rank[value] = 0; return true; } /*! \brief Union two elements in the union-find set. \param a The first element. \param b The second element. \return true if the union was successful, false if the elements are already in the same set. */ bool Union(T a, T b) { std::queue queue; while (parent[a] != a) { queue.push(a); a = parent[a]; } while (!queue.empty()) { parent[queue.front()] = a; queue.pop(); } while (parent[b] != b) { queue.push(b); b = parent[b]; } while (!queue.empty()) { parent[queue.front()] = b; queue.pop(); } if (a == b) { return false; } if (rank[a] < rank[b]) { parent[a] = b; rank[b]++; } else { parent[b] = a; rank[a]++; } return true; } /*! \brief Find the representative of the set containing the given element. \param value The element whose representative is to be found. \return The representative of the set containing the element. */ T find(T value) { std::queue queue; while (parent[value] != value) { queue.push(value); value = parent[value]; } while (!queue.empty()) { parent[queue.front()] = value; queue.pop(); } return value; } /* \brief Check if two elements are in the same set. \param a The first element. \param b The second element. \return true if the elements are in the same set, false otherwise. */ bool SameSet(T a, T b) const { return find(a) == find(b); } /*! \brief Get all the equivalence classes in the union-find set. \return A vector of unordered sets, each representing an equivalence class. */ std::vector> GetAllSets() const { std::vector> result; std::unordered_map which_set; for (const auto& [key, value] : parent) { if (which_set.find(value) == which_set.end()) { which_set[value] = result.size(); result.push_back(std::unordered_set()); } result[which_set[value]].insert(key); } return result; } void Clear() { parent.clear(); rank.clear(); } }; } // namespace xgrammar #endif // XGRAMMAR_SUPPORT_UNION_FIND_SET_H_ xgrammar-0.1.19/cpp/support/utils.h000066400000000000000000000121061500705317600172330ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/support/utils.h * \brief Utility functions. */ #ifndef XGRAMMAR_SUPPORT_UTILS_H_ #define XGRAMMAR_SUPPORT_UTILS_H_ #include #include #include #include #include #include #include #include #include "logging.h" namespace xgrammar { /*! * \brief Hash and combine value into seed. * \ref https://www.boost.org/doc/libs/1_84_0/boost/intrusive/detail/hash_combine.hpp */ inline void HashCombineBinary(uint32_t& seed, uint32_t value) { seed ^= value + 0x9e3779b9 + (seed << 6) + (seed >> 2); } /*! * \brief Find the hash sum of several uint32_t args. */ template inline uint32_t HashCombine(Args... args) { uint32_t seed = 0; (..., HashCombineBinary(seed, args)); return seed; } // Sometimes GCC fails to detect some branches will not return, such as when we use LOG(FATAL) // to raise an error. This macro manually mark them as unreachable to avoid warnings. #ifdef __GNUC__ #define XGRAMMAR_UNREACHABLE() __builtin_unreachable() #else #define XGRAMMAR_UNREACHABLE() #endif // Return the memory consumption in heap memory of a container. template inline constexpr std::size_t MemorySize(const Container& container) { using Element_t = std::decay_t; static_assert(std::is_trivially_copyable_v, "Element type must be trivial"); static_assert(!std::is_trivially_copyable_v, "Container type must not be trivial"); return sizeof(Element_t) * std::size(container); } template inline constexpr std::size_t MemorySize(const std::optional& range) { return range.has_value() ? MemorySize(*range) : 0; } /*! * \brief A Result type similar to Rust's Result, representing either success (Ok) or failure (Err). * \tparam T The type of the success value */ template class Result { public: /*! \brief Construct a success Result */ static Result Ok(T value) { return Result(std::move(value), nullptr); } /*! \brief Construct an error Result */ static Result Err(std::shared_ptr error) { return Result(std::nullopt, std::move(error)); } /*! \brief Check if Result contains success value */ bool IsOk() const { return value_.has_value(); } /*! \brief Check if Result contains error */ bool IsErr() const { return error_ != nullptr; } /*! \brief Get the success value, or terminate if this is an error */ const T& Unwrap() const& { if (!IsOk()) { XGRAMMAR_LOG(FATAL) << "Called Unwrap() on an Err value"; XGRAMMAR_UNREACHABLE(); } return *value_; } /*! \brief Get the success value, or terminate if this is an error */ T&& Unwrap() && { if (!IsOk()) { XGRAMMAR_LOG(FATAL) << "Called Unwrap() on an Err value"; XGRAMMAR_UNREACHABLE(); } return std::move(*value_); } /*! \brief Get the error value as a pointer, or terminate if this is not an error */ std::shared_ptr UnwrapErr() const& { if (!IsErr()) { XGRAMMAR_LOG(FATAL) << "Called UnwrapErr() on an Ok value"; XGRAMMAR_UNREACHABLE(); } return error_; } /*! \brief Get the error value as a pointer, or terminate if this is not an error */ std::shared_ptr UnwrapErr() && { if (!IsErr()) { XGRAMMAR_LOG(FATAL) << "Called UnwrapErr() on an Ok value"; XGRAMMAR_UNREACHABLE(); } return std::move(error_); } /*! \brief Get the success value if present, otherwise return the provided default */ T UnwrapOr(T default_value) const { return IsOk() ? *value_ : default_value; } /*! \brief Map success value to new type using provided function */ template Result Map(F&& f) const { if (IsOk()) { return Result::Ok(f(*value_)); } return Result::Err(error_); } /*! \brief Map error value to new type using provided function */ template Result MapErr(F&& f) const { if (IsErr()) { return Result::Err(f(error_)); } return Result::Ok(*value_); } private: Result(std::optional value, std::shared_ptr error) : value_(std::move(value)), error_(std::move(error)) {} std::optional value_; std::shared_ptr error_; }; } // namespace xgrammar namespace std { template struct hash> { size_t operator()(const std::pair& pair) const { return xgrammar::HashCombine(std::hash{}(pair.first), std::hash{}(pair.second)); } }; template struct hash> { size_t operator()(const std::tuple& tuple) const { return std::apply( [](const Args&... args) { return xgrammar::HashCombine(std::hash{}(args)...); }, tuple ); } }; template struct hash> { size_t operator()(const std::vector& vec) const { uint32_t seed = 0; for (const auto& item : vec) { xgrammar::HashCombineBinary(seed, std::hash{}(item)); } return seed; } }; } // namespace std #endif // XGRAMMAR_SUPPORT_UTILS_H_ xgrammar-0.1.19/cpp/testing.cc000066400000000000000000000020761500705317600161770ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/structural_tag.cc */ #include #include #include #include #include "grammar_parser.h" #include "support/encoding.h" namespace xgrammar { std::string PrintTokenByIds( const std::vector& token_ids, const TokenizerInfo& tokenizer_info, int max_print_num ) { std::stringstream ss; const auto& sorted_decoded_vocab = tokenizer_info.GetDecodedVocab(); ss << "["; int print_num = std::min(static_cast(token_ids.size()), max_print_num); for (int i = 0; i < print_num; ++i) { ss << "#" << token_ids[i] << " <" << PrintAsEscapedUTF8(sorted_decoded_vocab[token_ids[i]]) << ">"; if (i < print_num - 1) { ss << ", "; } } if (static_cast(token_ids.size()) > max_print_num) { ss << ", ..."; } ss << "]"; return ss.str(); } Grammar _EBNFToGrammarNoNormalization( const std::string& ebnf_string, const std::string& root_rule_name ) { return ParseEBNF(ebnf_string, root_rule_name); } } // namespace xgrammar xgrammar-0.1.19/cpp/testing.h000066400000000000000000000010641500705317600160350ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/testing.h * \brief The header testing utilities. */ #ifndef XGRAMMAR_TESTING_H_ #define XGRAMMAR_TESTING_H_ #include #include #include namespace xgrammar { std::string PrintTokenByIds( const std::vector& token_ids, const TokenizerInfo& tokenizer_info, int max_print_num ); Grammar _EBNFToGrammarNoNormalization( const std::string& ebnf_string, const std::string& root_rule_name ); } // namespace xgrammar #endif // XGRAMMAR_TESTING_H_ xgrammar-0.1.19/cpp/tokenizer_info.cc000066400000000000000000000453321500705317600175510ustar00rootroot00000000000000/*! * Copyright (c) 2023 by Contributors * \file xgrammar/tokenizer_info.cc */ #include #include #include #include #include #include #include #include #include #include "support/encoding.h" #include "support/logging.h" namespace xgrammar { class TokenizerInfo::Impl { public: Impl( const std::vector& encoded_vocab, VocabType vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ); VocabType GetVocabType() const { return vocab_type_; } bool GetAddPrefixSpace() const { return add_prefix_space_; } int GetVocabSize() const { return vocab_size_; } const std::vector& GetDecodedVocab() { return decoded_vocab_; } const std::vector& GetStopTokenIds() const { return stop_token_ids_; } const std::vector& GetSpecialTokenIds() const { return special_token_ids_; } const std::vector>& GetSortedDecodedVocab() const { return sorted_decoded_vocab_; } std::string DumpMetadata() const; static std::shared_ptr FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ); static std::string DetectMetadataFromHF(const std::string& backend_str); private: static bool IsSpecialToken(const std::string& decoded_token); /*! \brief The vocabulary type. */ VocabType vocab_type_; /*! \brief The size of the vocabulary. */ int vocab_size_; /*! \brief Whether to add prefix space. */ bool add_prefix_space_; /*! \brief The vocabulary. Special tokens are included. */ std::vector decoded_vocab_; /*! \brief All (id, token) pairs sorted in lexicographic order. This sorting is done to * maximize prefix reuse during matching. Special tokens and stop tokens are not included. */ std::vector> sorted_decoded_vocab_; /*! \brief The stop tokens. When the GrammarMatcher can reach the end of the grammar, * stop tokens can be accepted. */ std::vector stop_token_ids_; /*! \brief The special tokens. These tokens are ignored (masked out) during the grammar-guided * generation. */ std::vector special_token_ids_; /*! * \brief The tokens used to detect stop tokens from the vocabulary. * * LLaMA2: * LLaMA3: <|end_of_text|>, <|eot_id|> * Phi-2: <|endoftext|> * Gemma: , * DeepSeek-V2: <｜end▁of▁sentence｜> */ inline static const std::unordered_set DETECTION_STOP_TOKENS = { "", "<|end_of_text|>", "<|eot_id|>", "<|endoftext|>", "", "<|eos|>", "", "<｜end▁of▁sentence｜>" }; }; /************* Token decoders: ByteFallback and ByteLevel *************/ class TokenDecoder { public: /*! * \brief Post-process a raw token to the actual token with the given post-processing method. */ static std::string DecodeToken(const std::string& token, VocabType vocab_type) { // TODO(yixin): Avoid allocating new string in decoder calls if (vocab_type == VocabType::BYTE_FALLBACK) { return SpaceReplacerDecoder(ByteFallbackDecoder(token)); } else if (vocab_type == VocabType::BYTE_LEVEL) { return ByteLevelDecoder(token); } else { return token; } } private: /*! \brief ByteFallback decoder: transform tokens like <0x1B> to hex char byte 1B */ static std::string ByteFallbackDecoder(const std::string& token) { if (token.length() == 6 && token.substr(0, 3) == "<0x" && token.back() == '>') { int byte = 0; for (int i = 0; i < 2; ++i) { byte *= 16; byte += token[3 + i] >= '0' && token[3 + i] <= '9' ? token[3 + i] - '0' : token[3 + i] - 'A' + 10; } XGRAMMAR_CHECK(byte >= 0 && byte < 256); return std::string(/*n=*/1, static_cast(byte)); } return token; } /*! \brief SpaceReplacer decoder: transform "\u2581" back to space */ static std::string SpaceReplacerDecoder(const std::string& token) { // \u2581 is the unicode for "lower one eighth block" // UTF8 encoding for \u2581 is 0xE2 0x96 0x81 std::string result; for (int i = 0; i < static_cast(token.size()); ++i) { if (i + 2 < static_cast(token.size()) && token[i] == char(0xE2) && token[i + 1] == char(0x96) && token[i + 2] == char(0x81)) { result += ' '; i += 2; } else { result += token[i]; } } return result; } /*! * \brief ByteLevel decoder: inverses the bytes-to-unicode transformation in the encoding * process as in * https://github.com/huggingface/transformers/blob/87be06ca77166e6a6215eee5a990ab9f07238a18/src/transformers/models/gpt2/tokenization_gpt2.py#L38-L59 */ static std::string ByteLevelDecoder(const std::string& token) { // The inverse map of bytes_to_unicode. -1 means there is no mapping to this unicode. static const std::array char_to_byte_map = { // clang-format off -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, -1, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 173 // clang-format on }; auto unicode_codepoints = ParseUTF8(token.c_str(), false); if (unicode_codepoints.size() == 1 && unicode_codepoints[0] == kInvalidUTF8) { return token; } std::string decoded; decoded.reserve(unicode_codepoints.size()); for (auto unicode_codepoint : unicode_codepoints) { XGRAMMAR_CHECK(unicode_codepoint >= 0); if (unicode_codepoint >= static_cast(char_to_byte_map.size()) || char_to_byte_map[unicode_codepoint] == -1) { // If there is no mapping, return the original token return token; } decoded += static_cast(char_to_byte_map[unicode_codepoint]); } return decoded; } }; /************* Metadata detection from huggingface tokenizer.json *************/ class HFTokenizerAnalyzer { public: /*! * \brief Detect the vocabulary type from tokenizer.json. * \details Find {"type": "ByteFallback"} or {"type": "ByteLevel"} in "decoder" field of the * tokenizer. */ static VocabType DetectVocabType(const picojson::object& hf_tokenizer_obj) { #define CHECK_AND_WARNING(condition, message) \ if (!(condition)) { \ XGRAMMAR_LOG(WARNING) << "Vocab type detection failed: (" #condition \ << ") is false: " << (message) << " Using RAW VocabType by default."; \ return VocabType::RAW; \ } CHECK_AND_WARNING( hf_tokenizer_obj.count("decoder") && hf_tokenizer_obj.at("decoder").is(), "Decoder field is not found in tokenizer.json." ); auto decoder_obj = hf_tokenizer_obj.at("decoder").get(); CHECK_AND_WARNING( decoder_obj.count("type") && decoder_obj.at("type").is(), "Type field is not found in decoder field" ); auto type = decoder_obj.at("type").get(); std::vector decoders; if (type == "Sequence") { CHECK_AND_WARNING( decoder_obj.count("decoders") && decoder_obj.at("decoders").is(), "Decoders field is not found in a Sequence decoder" ); decoders = decoder_obj.at("decoders").get(); } else { decoders.emplace_back(hf_tokenizer_obj.at("decoder")); } for (const auto& decoder : decoders) { CHECK_AND_WARNING(decoder.is(), "Decoder is not an object"); auto decoder_obj = decoder.get(); CHECK_AND_WARNING( decoder_obj.count("type") && decoder_obj.at("type").is(), "Type field is not found in decoder field" ); auto type = decoder_obj.at("type").get(); if (type == "ByteLevel") { return VocabType::BYTE_LEVEL; } else if (type == "ByteFallback") { return VocabType::BYTE_FALLBACK; } } // If neither byte_level nor byte_fallback decoder is detected, return RAW. return VocabType::RAW; #undef CHECK_AND_WARNING } static bool DetectPrependNormalizer(const picojson::object& hf_tokenizer_obj) { if (!hf_tokenizer_obj.count("normalizer") || !hf_tokenizer_obj.at("normalizer").is()) { return false; } const picojson::value& normalizer_value = hf_tokenizer_obj.at("normalizer"); if (!normalizer_value.is()) { return false; } const picojson::object& normalizer_obj = normalizer_value.get(); if (!normalizer_obj.count("type") || !normalizer_obj.at("type").is()) { return false; } auto type = normalizer_obj.at("type").get(); std::vector normalizers; if (type == "Sequence") { if (!normalizer_obj.count("normalizers") || !normalizer_obj.at("normalizers").is()) { return false; } normalizers = normalizer_obj.at("normalizers").get(); } else { normalizers.emplace_back(normalizer_value); } for (const auto& normalizer : normalizers) { if (!normalizer.is()) { continue; } auto normalizer_obj = normalizer.get(); if (!normalizer_obj.count("type") || !normalizer_obj.at("type").is()) { continue; } auto type = normalizer_obj.at("type").get(); if (type == "Prepend" && normalizer_obj.count("prepend") && normalizer_obj.at("prepend").is() && normalizer_obj.at("prepend").get() == "▁") { return true; } } return false; } static bool DetectMetaspacePreTokenizer(const picojson::object& hf_tokenizer_obj) { if (!hf_tokenizer_obj.count("pre_tokenizer") || !hf_tokenizer_obj.at("pre_tokenizer").is()) { return false; } auto pre_tokenizer_obj = hf_tokenizer_obj.at("pre_tokenizer").get(); if (!pre_tokenizer_obj.count("type") || !pre_tokenizer_obj.at("type").is()) { return false; } auto type = pre_tokenizer_obj.at("type").get(); if (!pre_tokenizer_obj.count("prepend_scheme") || !pre_tokenizer_obj.at("prepend_scheme").is()) { return false; } auto prepend_scheme = pre_tokenizer_obj.at("prepend_scheme").get(); return type == "Metaspace" && (prepend_scheme == "always" || prepend_scheme == "first"); } /*! * \brief Detect whether add prefix space from tokenizer.json. * \details Find {"type": "Prepend", "prepend": "▁"} in "normalizer" field of the tokenizer, or * "pre_tokenizer": {"type": "Metaspace", "prepend_scheme": "always" | "first"} in the tokenizer. */ static bool DetectAddPrefixSpace(const picojson::object& hf_tokenizer_obj) { return DetectPrependNormalizer(hf_tokenizer_obj) || DetectMetaspacePreTokenizer(hf_tokenizer_obj); } }; /************* TokenizerInfo::Impl *************/ bool TokenizerInfo::Impl::IsSpecialToken(const std::string& token) { return token == ""; } TokenizerInfo::Impl::Impl( const std::vector& encoded_vocab, VocabType vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) : vocab_type_(vocab_type), vocab_size_(vocab_size.value_or(encoded_vocab.size())), add_prefix_space_(add_prefix_space) { decoded_vocab_.reserve(encoded_vocab.size()); sorted_decoded_vocab_.reserve(encoded_vocab.size()); for (int i = 0; i < static_cast(encoded_vocab.size()); ++i) { const std::string& token = TokenDecoder::DecodeToken(encoded_vocab[i], vocab_type_); decoded_vocab_.push_back(token); if ((!stop_token_ids && DETECTION_STOP_TOKENS.count(token)) || (stop_token_ids && std::find(stop_token_ids->begin(), stop_token_ids->end(), i) != stop_token_ids->end())) { stop_token_ids_.push_back(i); } else if (IsSpecialToken(token)) { special_token_ids_.push_back(i); } else { sorted_decoded_vocab_.push_back({i, token}); } } for (int i = encoded_vocab.size(); i < vocab_size_; ++i) { special_token_ids_.push_back(i); } auto f_compare_token = [](const std::pair& a, const std::pair& b) { return a.second < b.second; }; std::sort(sorted_decoded_vocab_.begin(), sorted_decoded_vocab_.end(), f_compare_token); } std::string TokenizerInfo::Impl::DumpMetadata() const { picojson::object obj; obj["vocab_type"] = picojson::value(static_cast(vocab_type_)); obj["vocab_size"] = picojson::value(static_cast(vocab_size_)); obj["add_prefix_space"] = picojson::value(add_prefix_space_); picojson::array stop_token_ids_array; for (auto id : stop_token_ids_) { stop_token_ids_array.push_back(picojson::value(static_cast(id))); } obj["stop_token_ids"] = picojson::value(stop_token_ids_array); return picojson::value(obj).serialize(false); } std::shared_ptr TokenizerInfo::Impl::FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ) { picojson::value v; std::string err = picojson::parse(v, metadata); XGRAMMAR_CHECK(err.empty()) << "Failed to parse metadata: " << err; const picojson::object& obj = v.get(); XGRAMMAR_CHECK(obj.count("vocab_type") && obj["vocab_type"].is()) << "Missing or invalid 'vocab_type' in metadata"; int vocab_type_int = static_cast(obj["vocab_type"].get()); XGRAMMAR_CHECK(vocab_type_int == 0 || vocab_type_int == 1 || vocab_type_int == 2) << "Invalid vocab_type in metadata: " << vocab_type_int; VocabType vocab_type = static_cast(vocab_type_int); XGRAMMAR_CHECK(obj.count("vocab_size") && obj["vocab_size"].is()) << "Missing or invalid 'vocab_size' in metadata"; int vocab_size = static_cast(obj["vocab_size"].get()); XGRAMMAR_CHECK(obj.count("add_prefix_space") && obj["add_prefix_space"].is()) << "Missing or invalid 'add_prefix_space' in metadata"; bool add_prefix_space = obj["add_prefix_space"].get(); std::vector stop_token_ids; XGRAMMAR_CHECK(obj.count("stop_token_ids") && obj["stop_token_ids"].is()) << "Missing or invalid 'stop_token_ids' in metadata"; for (const auto& id : obj["stop_token_ids"].get()) { XGRAMMAR_CHECK(id.is()) << "Stop token id is not an integer"; stop_token_ids.push_back(static_cast(id.get())); } return std::make_shared( encoded_vocab, vocab_type, vocab_size, stop_token_ids, add_prefix_space ); } std::string TokenizerInfo::Impl::DetectMetadataFromHF(const std::string& backend_str) { picojson::value v; std::string err = picojson::parse(v, backend_str); XGRAMMAR_CHECK(err.empty() && v.is()) << "Failed to parse JSON object: " << err; const picojson::object& obj = v.get(); VocabType vocab_type = HFTokenizerAnalyzer::DetectVocabType(obj); bool add_prefix_space = HFTokenizerAnalyzer::DetectAddPrefixSpace(obj); // Serialize the metadata picojson::object metadata_obj; metadata_obj["vocab_type"] = picojson::value(static_cast(vocab_type)); metadata_obj["add_prefix_space"] = picojson::value(add_prefix_space); return picojson::value(metadata_obj).serialize(false); } /************* TokenizerInfo *************/ TokenizerInfo::TokenizerInfo( const std::vector& encoded_vocab, VocabType vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) : pimpl_(std::make_shared( encoded_vocab, vocab_type, vocab_size, stop_token_ids, add_prefix_space )) {} int TokenizerInfo::GetVocabSize() const { return pimpl_->GetVocabSize(); } VocabType TokenizerInfo::GetVocabType() const { return pimpl_->GetVocabType(); } bool TokenizerInfo::GetAddPrefixSpace() const { return pimpl_->GetAddPrefixSpace(); } const std::vector& TokenizerInfo::GetDecodedVocab() const { return pimpl_->GetDecodedVocab(); } const std::vector& TokenizerInfo::GetStopTokenIds() const { return pimpl_->GetStopTokenIds(); } const std::vector& TokenizerInfo::GetSpecialTokenIds() const { return pimpl_->GetSpecialTokenIds(); } const std::vector>& TokenizerInfo::GetSortedDecodedVocab() const { return pimpl_->GetSortedDecodedVocab(); } std::string TokenizerInfo::DumpMetadata() const { return pimpl_->DumpMetadata(); } TokenizerInfo TokenizerInfo::FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ) { return TokenizerInfo(Impl::FromVocabAndMetadata(encoded_vocab, metadata)); } std::string TokenizerInfo::DetectMetadataFromHF(const std::string& backend_str) { return Impl::DetectMetadataFromHF(backend_str); } } // namespace xgrammar xgrammar-0.1.19/docs/000077500000000000000000000000001500705317600143545ustar00rootroot00000000000000xgrammar-0.1.19/docs/.gitignore000066400000000000000000000000101500705317600163330ustar00rootroot00000000000000_build/ xgrammar-0.1.19/docs/Makefile000066400000000000000000000011761500705317600160210ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= python -m sphinx SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) xgrammar-0.1.19/docs/README.md000066400000000000000000000013071500705317600156340ustar00rootroot00000000000000# XGrammar Documentation The documentation was built upon [Sphinx](https://www.sphinx-doc.org/en/master/). ## Dependencies Run the following command in this directory to install dependencies first: ```bash pip3 install -r requirements.txt ``` ## Build the Documentation Then you can build the documentation by running: ```bash make html ``` ## View the Documentation Run the following command to start a simple HTTP server: ```bash cd _build/html python3 -m http.server ``` Then you can view the documentation in your browser at `http://localhost:8000` (the port can be customized by appending ` -p PORT_NUMBER` in the python command above). You may also need `--bind 0.0.0.0` for machines like Mac. xgrammar-0.1.19/docs/_static/000077500000000000000000000000001500705317600160025ustar00rootroot00000000000000xgrammar-0.1.19/docs/_static/img/000077500000000000000000000000001500705317600165565ustar00rootroot00000000000000xgrammar-0.1.19/docs/_static/img/mlc-logo-with-text-landscape.svg000066400000000000000000000375621500705317600247100ustar00rootroot00000000000000 image/svg+xml xgrammar-0.1.19/docs/api/000077500000000000000000000000001500705317600151255ustar00rootroot00000000000000xgrammar-0.1.19/docs/api/python/000077500000000000000000000000001500705317600164465ustar00rootroot00000000000000xgrammar-0.1.19/docs/api/python/index.rst000066400000000000000000000001621500705317600203060ustar00rootroot00000000000000.. _apixgrammar: xgrammar ======== .. automodule:: xgrammar :members: :imported-members: :autosummary: xgrammar-0.1.19/docs/conf.py000066400000000000000000000044571500705317600156650ustar00rootroot00000000000000# -*- coding: utf-8 -*- import os import sys import tlcpack_sphinx_addon # -- General configuration ------------------------------------------------ os.environ["XGRAMMAR_BUILD_DOCS"] = "1" sys.path.insert(0, os.path.abspath("../python")) sys.path.insert(0, os.path.abspath("../")) autodoc_mock_imports = ["torch"] # General information about the project. project = "XGrammar" author = "XGrammar Contributors" copyright = "2024, %s" % author # Version information. version = "0.1.0" release = "0.1.0" extensions = [ "sphinx_tabs.tabs", "sphinx_toolbox.collapse", "sphinxcontrib.httpdomain", "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.napoleon", "sphinx_reredirects", "autodocsumm", ] redirects = {} source_suffix = [".rst"] language = "en" exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme is set by the make target import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] templates_path = [] html_static_path = [] footer_copyright = "© 2024 XGrammar" footer_note = " " # html_logo = "_static/img/mlc-logo-with-text-landscape.svg" html_theme_options = {"logo_only": False} header_links = [ ("Home", "https://xgrammar.mlc.ai/"), ("Github", "https://github.com/mlc-ai/xgrammar"), ] header_dropdown = {"name": "Other Resources", "items": [("MLC Blog", "https://blog.mlc.ai/")]} html_context = { "footer_copyright": footer_copyright, "footer_note": footer_note, "header_links": header_links, "header_dropdown": header_dropdown, "display_github": True, "github_user": "mlc-ai", "github_repo": "xgrammar", "github_version": "main/docs/", "theme_vcs_pageview_mode": "edit", # "header_logo": "/path/to/logo", # "header_logo_link": "", # "version_selecter": "", } import xgrammar # add additional overrides templates_path += [tlcpack_sphinx_addon.get_templates_path()] html_static_path += [tlcpack_sphinx_addon.get_static_path()] xgrammar-0.1.19/docs/how_to/000077500000000000000000000000001500705317600156535ustar00rootroot00000000000000xgrammar-0.1.19/docs/how_to/ebnf_guided_generation.rst000066400000000000000000000154071500705317600230620ustar00rootroot00000000000000.. _how-to-ebnf-generation: EBNF-Guided Generation ====================== XGrammar enables efficient structured generation. Besides JSON, you can use an EBNF grammar to guide the generation, providing more flexibility for customization. We first go over how to use XGrammar in an LLM engine to achieve this in :ref:`EBNF-Guided Generation in LLM Engines `, we then provide an end-to-end JSON generation using XGrammar with HF ``transformers`` in :ref:`Try out via HF Transformers `. Install XGrammar ~~~~~~~~~~~~~~~~ :ref:`XGrammar ` is available via pip. It is always recommended to install it in an isolated conda virtual environment. .. _how-to-ebnf-generation-engine: EBNF-Guided Generation in LLM Engines ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In this section, we see how to use XGrammar in an LLM engine to ensure that the output follows ane EBNF grammar. All code snippets below are actual runnable code as we simulate the LLM generation. First, import necessary libraries for the tutorial. .. code:: python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig Then, we extract tokenizer info from the LLM we are using with ``xgr.TokenizerInfo``. With the ``tokenizer_info``, instantiate ``xgr.GrammarCompiler`` that will compiler a grammar of your choice. .. code:: python # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) Then specify an EBNF grammar string. We currently use the GBNF format (GGML BNF), with the specification `here `__. .. code:: python ebnf_grammar_str = """root ::= (expr "=" term)+ expr ::= term ([-+*/] term)* term ::= num | "(" expr ")" num ::= [0-9]+""" compiled_grammar = compiler.compile_grammar(ebnf_grammar_str) With the compiled grammar, we can instantiate a ``xgr.GrammarMatcher``, the main construct we interact with that maintains the state of the structured generation. We also allocate a bitmask that will be used to mask logits. .. code:: python # Instantiate grammar matcher and allocate the bitmask matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) Now we simulate a single-request auto-regressive generation. See :ref:`how-to-engine-integration` for batched inference. .. code:: python # Here we simulate a valid sampled response sim_sampled_response = '(5+3)*2=16<|end_of_text|>' sim_sampled_token_ids = tokenizer.encode(sim_sampled_response, add_special_tokens=False) # Each loop iteration is a simulated auto-regressive step for i, sim_token_id in enumerate(sim_sampled_token_ids): # LLM inference to get logits, here we use randn to simulate. # logits is a tensor of shape (full_vocab_size,) on GPU # logits = LLM.inference() logits = torch.randn(full_vocab_size).cuda() # Apply bitmask to logits to mask invalid tokens matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_id = np.random.choice(list(range(full_vocab_size)), p=probs) # Accept token from matcher to update its state, so that the next bitmask # generated will enforce the next token to be generated. Assert to make # sure the token is indeed valid. Here we accept the simulated response # assert matcher.accept_token(next_token_id) assert matcher.accept_token(sim_token_id) # Since we accepted a stop token `<|end_of_text|>`, we have terminated assert matcher.is_terminated() # Reset to be ready for the next auto-regressive generation matcher.reset() .. _how-to-ebnf-generation-HF: Try out via HF Transformers ~~~~~~~~~~~~~~~~~~~~~~~~~~~ XGrammar can be easily integrate with HF transformers using a ``LogitsProcessor``. Note that this integration mainly aims for accessibility and may contain extra overhead. First, instantiate a model, a tokenizer, and inputs. .. code:: python import xgrammar as xgr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig device = "cuda" # Or "cpu", etc. model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Introduce yourself in JSON briefly."}, ] texts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) Then construct a ``GrammarCompiler`` and compile the grammar. .. code:: python tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) # Grammar string that represents a JSON schema json_grammar_ebnf_str = r""" root ::= basic_array | basic_object basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]" basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [ \n\t]* """ compiled_grammar = grammar_compiler.compile_grammar(json_grammar_ebnf_str) Finally, use ``LogitsProcessor`` to generate with grammar. .. code:: python xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) generated_ids = generated_ids[0][len(model_inputs.input_ids[0]) :] print(tokenizer.decode(generated_ids, skip_special_tokens=True)) xgrammar-0.1.19/docs/how_to/engine_integration.rst000066400000000000000000000233261500705317600222630ustar00rootroot00000000000000.. _how-to-engine-integration: Integration with LLM Engine =========================== XGrammar enables efficient structured generation. In this tutorial, we go over the key components of XGrammar and how to integrate XGrammar into an LLM engine. We first lay out the concepts in :ref:`High-Level Flow `. We then demonstrate how XGrammar enables :ref:`Structured Generation for Batched Inference `. The code snippets below are actual runnable code as we simulate the LLM generation. Install XGrammar ---------------- :ref:`XGrammar ` is available via pip. It is always recommended to install it in an isolated conda virtual environment. .. _how-to-engine-integration-flow: High-Level Flow --------------- In this section, we go over the key components of XGrammar when integrating it into an LLM engine for structured generation. First, import necessary libraries for the tutorial. .. code:: python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig xgr.TokenizerInfo ^^^^^^^^^^^^^^^^^ ``xgr.TokenizerInfo`` is a per-model construct that encapsulates tokenizer information, including all its vocabulary. There are several ways of instantiating it, and the most convenient way is using an ``AutoTokenizer``. Note that for some models, ``AutoConfig.vocab_size`` can be larger than ``AutoTokenizer.vocab_size`` due to paddings, with the former being the shape of the model's logits. To be safe, always pass in the former when instantiating ``xgr.TokenizerInfo``. .. code:: python # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) xgr.GrammarCompiler ^^^^^^^^^^^^^^^^^^^ With an ``xgr.TokenizerInfo``, we can instantiate an ``xgr.GrammarCompiler``. This is a construct that compiles a grammar according to the model's tokenizer info. Therefore, for each model, you can use the same ``xgr.GrammarCompiler`` persistently, as it can compile different grammars for the same ``xgr.TokenizerInfo``. Note that the ``compiler`` behavior can be configured with ``max_threads`` for multithreading, and ``enable_cache`` (defaults to true) for caching compiled grammars. .. code:: python compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) xgr.CompiledGrammar ^^^^^^^^^^^^^^^^^^^ Then, using the ``xgr.GrammarCompiler``, we can compile a grammar, with the result being an ``xgr.CompiledGrammar``. Here we use a built-in JSON grammar. For other grammars, see :ref:`how-to-json-generation` and :ref:`how-to-ebnf-generation`. Every thing we have seen up to now are per-model (rather than per-generation). .. code:: python compiled_grammar: xgr.CompiledGrammar = compiler.compile_builtin_json_grammar() xgr.GrammarMatcher ^^^^^^^^^^^^^^^^^^ With the compiled grammar, we can instantiate a ``xgr.GrammarMatcher``. It is the main construct an LLM engine interacts with that maintains the state of the structured generation. Note that each request should have its own ``xgr.GrammarMatcher`` since each has a different generation state, as we will see in :ref:`how-to-engine-integration-batched`. .. code:: python # Instantiate grammar matcher with the compiled grammar matcher = xgr.GrammarMatcher(compiled_grammar) Bitmasking Logits in Auto-regressive Generation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Now we simulate a single-request auto-regressive generation. See later section for :ref:`how-to-engine-integration-batched`. First, we pre-allocate a token bitmask with ``xgr.allocate_token_bitmask()``, which is essentially a ``torch.Tensor`` of shape ``(batch_size, vocab_size)``. You can also use your own implementation for allocating a bitmask. In each auto-regressive step, we fill the token bitmask according to the current state of the matcher with ``xgr.GrammarMatcher.fill_next_token_bitmask()``. Then, we apply the bitmask into the model's logits with ``xgr.apply_token_bitmask_inplace()``, which calls a CUDA kernel if ``logits`` is on CUDA (recommended), otherwise a CPU implementation. After masking, the logits for illegal tokens are set to negative infinity, so that we will never sample them. After sampling the token, update the ``xgr.GrammarMatcher``'s state with ``xgr.GrammarMatcher.accept_token()``. Finally, use ``xgr.GrammarMatcher.reset()`` to prepare for the next generation. .. code:: python # Here we simulate a valid sampled response sim_sampled_response = '{ "library": "xgrammar" }<|end_of_text|>' sim_sampled_token_ids = tokenizer.encode(sim_sampled_response, add_special_tokens=False) # Allocate a token bitmask token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # Each loop iteration is a simulated auto-regressive step for i, sim_token_id in enumerate(sim_sampled_token_ids): # LLM inference to get logits, here we use randn to simulate. # logits is a tensor of shape (full_vocab_size,) on GPU # logits = LLM.inference() logits = torch.randn(full_vocab_size).cuda() # Apply bitmask to logits to mask invalid tokens matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_id = np.random.choice(list(range(full_vocab_size)), p=probs) # Accept token from matcher to update its state, so that the next bitmask # generated will enforce the next token to be generated. Assert to make # sure the token is indeed valid. Here we accept the simulated response # assert matcher.accept_token(next_token_id) assert matcher.accept_token(sim_token_id) # Since we accepted a stop token `<|end_of_text|>`, we have terminated assert matcher.is_terminated() # Reset to be ready for the next auto-regressive generation matcher.reset() .. _how-to-engine-integration-batched: Structured Generation for Batched Inference ------------------------------------------- The code snippets above assume a single request generation. This section demonstrates how the same concept works with batched generation. First, follow the exact same steps above for the per-model constructs ``xgr.TokenizerInfo`` and ``xgr.GrammarCompiler``. Say each request needs to generate a valid JSON. .. code:: python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) # Compile a JSON grammar compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) compiled_grammar: xgr.CompiledGrammar = compiler.compile_builtin_json_grammar() Now, we need to maintain an ``xgr.GrammarMatcher`` for each request in the batch, since each has a different generation state. Note that each request in the batch can follow a different ``xgr.CompiledGrammar``, but here for simplicity, they are all just following the general JSON grammar. .. code:: python batch_size = 2 matchers = [ xgr.GrammarMatcher(compiled_grammar) for i in range(batch_size) ] token_bitmask = xgr.allocate_token_bitmask(batch_size, tokenizer_info.vocab_size) We simulate an auto-regressive generation of batched inference. Note that here we assume the generation lengths of the two requests are the same for simplicity. But it should be easy to generalize based on how your engine supports batched inference. The key difference from single-request generation is that, in batched-request generation, each request has its own ``xgr.GrammarMatcher`` to maintain. .. code:: python sim_sampled_responses = ['{"name": "a"}<|end_of_text|>', '{"name": "b"}<|end_of_text|>'] sim_sampled_token_ids = [ tokenizer.encode(response, add_special_tokens=False) for response in sim_sampled_responses ] # Each loop iteration is a simulated auto-regressive step for loop_iter in range(len(sim_sampled_token_ids[0])): # LLM batched inference to get logits, here we use randn to simulate # Now, logits is a tensor of shape (batch_size, full_vocab_size) on GPU # logits = LLM.inference() logits = torch.randn(batch_size, full_vocab_size).cuda() # This for loop is parallelizable using threading.Thread. But estimate # the overhead in your engine. for i in range(batch_size): matchers[i].fill_next_token_bitmask(token_bitmask, i) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_ids = [ np.random.choice(list(range(full_vocab_size)), p=probs[i]) for i in range(batch_size) ] # Update the matcher for each request for i in range(batch_size): # Here we accept the simulated response # assert matchers[i].accept_token(next_token_ids[i]) matchers[i].accept_token(sim_sampled_token_ids[i][loop_iter]) # In our simulated case, all requests should have terminated since we accepted # a stop token `<|end_of_text|>` for i in range(batch_size): assert matchers[i].is_terminated() # Reset to be ready for the next generation matchers[i].reset() xgrammar-0.1.19/docs/how_to/json_generation.rst000066400000000000000000000155671500705317600216070ustar00rootroot00000000000000.. _how-to-json-generation: JSON Generation ====================== XGrammar enables efficient structured generation. One example structure is JSON and JSON Schema. In this tutorial, we go over how to use XGrammar to ensure that an LLM's output is a valid JSON, or adheres to a customized JSON schema. We first go over how to use XGrammar in an LLM engine to achieve this in :ref:`JSON Generation in LLM Engines `, we then provide an end-to-end JSON generation using XGrammar with HF ``transformers`` in :ref:`Try out via HF Transformers `. Install XGrammar ~~~~~~~~~~~~~~~~ :ref:`XGrammar ` is available via pip. It is always recommended to install it in an isolated conda virtual environment. .. _how-to-json-generation-engine: JSON Generation in LLM Engines ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In this section, we see how to use XGrammar in an LLM engine to ensure that the output is always a valid JSON. All code snippets below are actual runnable code as we simulate the LLM generation. First, import necessary libraries for the tutorial. .. code:: python import xgrammar as xgr import torch import numpy as np from transformers import AutoTokenizer, AutoConfig Then, we extract tokenizer info from the LLM we are using with ``xgr.TokenizerInfo``. With the ``tokenizer_info``, instantiate ``xgr.GrammarCompiler`` that will compiler a grammar of your choice. .. code:: python # Get tokenizer info model_id = "meta-llama/Llama-3.2-1B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_id) config = AutoConfig.from_pretrained(model_id) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=8) For JSON generation, there are generally three options for compiling the grammar: using a built-in JSON grammar, specify JSON schema with a Pydantic model, or from a JSON schema string. Pick one one of the three below to run. .. code:: python # Option 1: Compile with a built-in JSON grammar compiled_grammar: xgr.CompiledGrammar = compiler.compile_builtin_json_grammar() .. code:: python # Option 2: Compile with JSON schema from a pydantic model from pydantic import BaseModel class Person(BaseModel): name: str age: int compiled_grammar = compiler.compile_json_schema(Person) .. code:: python # Option 3: Compile with JSON schema from a JSON schema string import json person_schema = { "title": "Person", "type": "object", "properties": { "name": { "type": "string" }, "age": { "type": "integer", } }, "required": ["name", "age"] } compiled_grammar = compiler.compile_json_schema(json.dumps(person_schema)) With the compiled grammar, we can instantiate a ``xgr.GrammarMatcher``, the main construct we interact with that maintains the state of the structured generation. We also allocate a bitmask that will be used to mask logits. .. code:: python # Instantiate grammar matcher and allocate the bitmask matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) Now we simulate a single-request auto-regressive generation. See :ref:`how-to-engine-integration` for batched inference. .. code:: python # Here we simulate a valid sampled response sim_sampled_response = '{"name": "xgrammar", "age": 0}<|end_of_text|>' sim_sampled_token_ids = tokenizer.encode(sim_sampled_response, add_special_tokens=False) # Each loop iteration is a simulated auto-regressive step for i, sim_token_id in enumerate(sim_sampled_token_ids): # LLM inference to get logits, here we use randn to simulate. # logits is a tensor of shape (full_vocab_size,) on GPU # logits = LLM.inference() logits = torch.randn(full_vocab_size).cuda() # Apply bitmask to logits to mask invalid tokens matcher.fill_next_token_bitmask(token_bitmask) xgr.apply_token_bitmask_inplace(logits, token_bitmask.to(logits.device)) # Sample next token probs = torch.softmax(logits, dim=-1).cpu().numpy() next_token_id = np.random.choice(list(range(full_vocab_size)), p=probs) # Accept token from matcher to update its state, so that the next bitmask # generated will enforce the next token to be generated. Assert to make # sure the token is indeed valid. Here we accept the simulated response # assert matcher.accept_token(next_token_id) assert matcher.accept_token(sim_token_id) # Since we accepted a stop token `<|end_of_text|>`, we have terminated assert matcher.is_terminated() # Reset to be ready for the next auto-regressive generation matcher.reset() .. _how-to-json-generation-HF: Try out via HF Transformers ~~~~~~~~~~~~~~~~~~~~~~~~~~~ XGrammar can be easily integrate with HF transformers using a ``LogitsProcessor``. Note that this integration mainly aims for accessibility and may contain extra overhead. First, instantiate a model, a tokenizer, and inputs. .. code:: python import xgrammar as xgr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig device = "cuda" # Or "cpu", etc. model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Introduce yourself in JSON with two fields: name and age."}, ] texts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) Then construct a ``GrammarCompiler`` and compile the grammar. .. code:: python tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) # Option 1: Compile with a built-in JSON grammar # compiled_grammar = grammar_compiler.compile_builtin_json_grammar() # Option 2: Compile with JSON schema from a pydantic model from pydantic import BaseModel class Person(BaseModel): name: str age: int compiled_grammar = grammar_compiler.compile_json_schema(Person) Finally, use ``LogitsProcessor`` to generate with grammar. .. code:: python xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) generated_ids = generated_ids[0][len(model_inputs.input_ids[0]) :] print(tokenizer.decode(generated_ids, skip_special_tokens=True)) xgrammar-0.1.19/docs/how_to/portable_api.rst000066400000000000000000000016151500705317600210510ustar00rootroot00000000000000.. _how-to-portable-api: Portable API ============ XGrammar is implemented with a lightweight C++ core that can be integrated into many platforms. Besides the C++ backend, we also provide ready-to-use Python and JavaScript/TypeScript libraries. For the Python library, simply check out the :ref:`Python API Reference `. Below we take a high-level view of the Javascript library. .. _how-to-portable-js: Javascript SDK for Web-based LLMs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The JS SDK is designed to be used for LLMs that run in the browser, including `WebLLM `__. WebLLM integrated with XGrammar's JS SDK, ``web-xgrammar``. It uses `emscripten `__ to compile the C++ code into WebAssembly. To use this SDK, simply run ``npm install @mlc-ai/web-xgrammar``. For more, see `here `__. xgrammar-0.1.19/docs/index.rst000066400000000000000000000014751500705317600162240ustar00rootroot00000000000000👋 Welcome to XGrammar ====================== `GitHub `_ XGrammar is open-source solution for flexible, portable, and fast structured generations. The mission of this project is to bring flexible zero-overhead structure generation everywhere. Quick Start ----------- Check out :ref:`quick-start` for quick start examples of using XGrammar. .. toctree:: :maxdepth: 1 :caption: Get Started :hidden: start/install.rst start/quick_start.rst .. toctree:: :maxdepth: 1 :caption: How To :hidden: how_to/json_generation.rst how_to/ebnf_guided_generation.rst how_to/engine_integration.rst how_to/portable_api.rst .. tutorials/web_sdk.rst .. TODO .. toctree:: :maxdepth: 1 :caption: API Reference :hidden: api/python/index xgrammar-0.1.19/docs/make.bat000066400000000000000000000014331500705317600157620ustar00rootroot00000000000000@ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.https://www.sphinx-doc.org/ exit /b 1 ) if "%1" == "" goto help %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd xgrammar-0.1.19/docs/requirements.txt000066400000000000000000000003771500705317600176470ustar00rootroot00000000000000autodocsumm pydantic sentencepiece sphinx == 5.2.3 sphinx-reredirects==0.1.2 sphinx-rtd-theme sphinx-tabs == 3.4.1 sphinx-toolbox == 3.4.0 sphinxcontrib-napoleon==0.7 sphinxcontrib_httpdomain==1.8.1 tiktoken tlcpack-sphinx-addon==0.2.2 torch transformers xgrammar-0.1.19/docs/start/000077500000000000000000000000001500705317600155115ustar00rootroot00000000000000xgrammar-0.1.19/docs/start/install.rst000066400000000000000000000065111500705317600177140ustar00rootroot00000000000000.. _installation: Installation ============ .. contents:: Table of Contents :local: :depth: 2 XGrammar Python Package can be installed directly from a prebuilt developer package, or built from source. .. _installation_prebuilt_package: Option 1. Prebuilt Package -------------------------- We provide nightly built pip wheels for XGrammar via pip. .. note:: ❗ Whenever using Python, it is highly recommended to use **conda** to manage an isolated Python environment to avoid missing dependencies, incompatible versions, and package conflicts. Please make sure your conda environment has Python and pip installed. .. code-block:: bash conda activate your-environment python -m pip install xgrammar Then you can verify installation in command line: .. code-block:: bash python -c "import xgrammar; print(xgrammar)" # Prints out: CUDA Dependency ~~~~~~~~~~~~~~~ When using NVIDIA GPUs, please also install these extra dependencies to enable CUDA support for applying bitmasks: .. code-block:: bash python -m pip install cuda-python nvidia-cuda-nvrtc-cu12 | .. _installation_build_from_source: Option 2. Build from Source --------------------------- We also provide options to build XGrammar from source. This step is useful when you want to make modification or obtain a specific version of XGrammar. **Step 1. Set up build environment.** To build from source, you need to ensure that the following build dependencies are satisfied: * CMake >= 3.18 * Git * C++ Compiler (e.g. apt-get install build-essential) .. code-block:: bash # Using conda # make sure to start with a fresh environment conda env remove -n xgrammar-venv # create the conda environment with build dependency conda create -n xgrammar-venv -c conda-forge \ "cmake>=3.18" \ git \ python=3.11 \ ninja # enter the build environment conda activate xgrammar-venv # Using pip (you will need to install git seperately) python -m venv .venv source .venv/bin/activate **Step 2. Configure, build and install.** A standard git-based workflow is recommended to download XGrammar. .. code-block:: bash # 1. clone from GitHub git clone --recursive https://github.com/mlc-ai/xgrammar.git && cd xgrammar # 2. Install pre-commit hooks (optional, recommended for contributing to XGrammar) pre-commit install # 3. build and install XGrammar core and Python bindings python3 -m pip install . **Step 3. Validate installation.** You may validate if XGrammar is compiled successfully in command line. You should see the path you used to build from source with: .. code:: bash python -c "import xgrammar; print(xgrammar)" **Step 4. (Optional) Run Python Tests.** You will need a HuggingFace token and access to gated models to run the tests that have gated models. .. code:: bash # Install the test dependencies python3 -m pip install ".[test]" # To run all tests including the ones that have gated models, you will need a HuggingFace token. huggingface-cli login --token YOUR_HF_TOKEN python3 -m pytest tests/python # To run a subset of tests that do not require gated models, you can skip the tests with: python3 -m pytest tests/python -m "not hf_token_required" xgrammar-0.1.19/docs/start/quick_start.rst000066400000000000000000000052361500705317600206020ustar00rootroot00000000000000.. _quick-start: Quick Start =========== Example ------- The easiest way of trying out XGrammar is to use the ``transformers`` library in Python. After :ref:`installing XGrammar `, run the following example to see how XGrammar enables structured generation -- a JSON in this case. Perparation ^^^^^^^^^^^ Instantiate a model, a tokenizer, and inputs. .. code:: python import xgrammar as xgr import torch from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig device = "cuda" # Or "cpu", etc. model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Introduce yourself in JSON briefly."}, ] texts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) Compile Grammar ^^^^^^^^^^^^^^^ Construct a ``GrammarCompiler`` and compile the grammar. The grammar can be a built-in JSON grammar, a JSON schema string, or an EBNF string. EBNF provides more flexibility for customization. See `GBNF documentation `_ for specification. .. code:: python tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=config.vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = grammar_compiler.compile_builtin_json_grammar() # Other ways: provide a json schema string # compiled_grammar = grammar_compiler.compile_json_schema(json_schema_string) # Or provide an EBNF string # compiled_grammar = grammar_compiler.compile_grammar(ebnf_string) Generate with grammar ^^^^^^^^^^^^^^^^^^^^^ Use logits_processor to generate with grammar. .. code:: python xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) generated_ids = generated_ids[0][len(model_inputs.input_ids[0]) :] print(tokenizer.decode(generated_ids, skip_special_tokens=True)) What to Do Next --------------- - Check out :ref:`how-to-json-generation` and other How-To guides for the detailed usage guide of XGrammar. - Report any problem or ask any question: open new issues in our `GitHub repo `_. xgrammar-0.1.19/examples/000077500000000000000000000000001500705317600152425ustar00rootroot00000000000000xgrammar-0.1.19/examples/benchmark/000077500000000000000000000000001500705317600171745ustar00rootroot00000000000000xgrammar-0.1.19/examples/benchmark/README.md000066400000000000000000000076161500705317600204650ustar00rootroot00000000000000 ## Run Benchmark ### Benchmark Grammar Compile and Mask Generation #### Dependencies ``` outlines 0.1.3 outlines_core 0.1.14 lm-format-enforcer 0.10.6 ``` #### Run ```bash python3 bench_grammar_compile_mask_gen.py [-h] [--backend {xgrammar,outlines,lmformatenforcer}] [--num_iters NUM_ITERS] [--num_warmup NUM_WARMUP] ``` ### Benchmark Apply Token Bitmask Inplace Kernels #### Run ```bash python3 bench_apply_token_bitmask_inplace.py [-h] [--impl {cuda,triton}] [--batch_size BATCH_SIZE] [--vocab_size VOCAB_SIZE] [--masked_cnt MASKED_CNT] [--stride STRIDE] [--logits_dtype {float32,float16,bfloat16}] [--warmup WARMUP] [--rep REP] ``` #### Results | GPU | Batch size | Vocab size | Masked cnt | Triton (μs) | CUDA (μs) | Speedup | |:--------------:|-----------:|-----------:|-----------:|-------------:|----------:|--------:| | H100 80GB HBM3 | 1 | 128k | 1k | 5.95 | 6.57 | 0.91x | | | 1 | 128k | 64k | 6.38 | 6.46 | 0.99x | | | 1 | 128k | 127k | 6.69 | 6.48 | 1.03x | | | 8 | 128k | 1k | 6.77 | 6.94 | 0.98x | | | 8 | 128k | 64k | 8.05 | 9.19 | 0.88x | | | 8 | 128k | 127k | 8.49 | 8.08 | 1.05x | | | 64 | 128k | 1k | 14.97 | 13.82 | 1.08x | | | 64 | 128k | 64k | 43.13 | 30.98 | 1.39x | | | 64 | 128k | 127k | 33.85 | 21.43 | 1.58x | | | 512 | 128k | 1k | 82.65 | 61.13 | 1.35x | | | 512 | 128k | 64k | 293.51 | 194.06 | 1.51x | | | 512 | 128k | 127k | 240.11 | 119.77 | 2.00x | | | 4096 | 128k | 1k | 566.17 | 417.33 | 1.36x | | | 4096 | 128k | 64k | 2198.59 | 1491.79 | 1.47x | | | 4096 | 128k | 127k | 1812.39 | 897.17 | 2.02x | | A100 SXM4 80GB | 1 | 128k | 1k | 8.32 | 7.97 | 1.04x | | | 1 | 128k | 64k | 9.26 | 8.24 | 1.12x | | | 1 | 128k | 127k | 8.81 | 8.71 | 1.01x | | | 8 | 128k | 1k | 9.56 | 10.31 | 0.93x | | | 8 | 128k | 64k | 12.72 | 13.22 | 0.96x | | | 8 | 128k | 127k | 13.45 | 11.27 | 1.19x | | | 64 | 128k | 1k | 22.95 | 25.57 | 0.90x | | | 64 | 128k | 64k | 58.52 | 56.47 | 1.04x | | | 64 | 128k | 127k | 44.83 | 39.29 | 1.14x | | | 512 | 128k | 1k | 132.92 | 108.60 | 1.22x | | | 512 | 128k | 64k | 362.08 | 349.54 | 1.04x | | | 512 | 128k | 127k | 306.75 | 233.20 | 1.32x | | | 4096 | 128k | 1k | 955.99 | 777.94 | 1.23x | | | 4096 | 128k | 64k | 2756.63 | 2707.57 | 1.02x | | | 4096 | 128k | 127k | 2472.82 | 1782.41 | 1.39x | xgrammar-0.1.19/examples/benchmark/bench_apply_token_bitmask_inplace.py000066400000000000000000000067461500705317600264540ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import torch from triton.testing import do_bench from xgrammar.kernels import apply_token_bitmask_inplace_kernels from xgrammar.testing import _bool_mask_to_bitmask if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--impl", type=str, choices=["cuda", "triton"], default="cuda") parser.add_argument("--batch_size", type=int, default=4096) parser.add_argument("--vocab_size", type=int, default=128000) parser.add_argument("--masked_cnt", type=int, default=1024) parser.add_argument("--stride", type=int, default=1) parser.add_argument( "--logits_dtype", type=str, choices=["float32", "float16", "bfloat16"], default="float32" ) parser.add_argument("--warmup", type=int, default=500) parser.add_argument("--rep", type=int, default=2000) args = parser.parse_args() vocab_size = args.vocab_size batch_size = args.batch_size bitmask_size = (vocab_size + 32 - 1) // 32 masked_cnt = args.masked_cnt stride = args.stride logits_dtype = getattr(torch, args.logits_dtype) logits = torch.randn(batch_size, vocab_size, dtype=logits_dtype, device="cuda") if masked_cnt >= vocab_size: bool_mask = torch.zeros(batch_size, vocab_size, dtype=torch.bool, device="cuda") else: bool_mask = torch.ones(batch_size, vocab_size, dtype=torch.bool, device="cuda") if masked_cnt > 0: masked_positions = torch.stack( [torch.randperm(vocab_size, device="cuda")[:masked_cnt] for _ in range(batch_size)] ) bool_mask.scatter_(1, masked_positions, False) assert (bool_mask.sum(dim=-1) + masked_cnt == vocab_size).all().item() bitmask = _bool_mask_to_bitmask(bool_mask) masked_batch_ids = torch.arange(0, batch_size, stride, dtype=torch.int32, device="cuda") kwargs = {} if stride == 1 else {"indices": masked_batch_ids} logits_expected = logits.clone() logits_expected[masked_batch_ids] = torch.masked_fill( logits_expected[masked_batch_ids], ~bool_mask[masked_batch_ids], float("-inf") ) if args.impl == "cuda": if "cuda" not in apply_token_bitmask_inplace_kernels: raise ImportError("CUDA is not installed") f = lambda: apply_token_bitmask_inplace_kernels["cuda"](logits, bitmask, **kwargs) elif args.impl == "triton": if "triton" not in apply_token_bitmask_inplace_kernels: raise ImportError("Triton is not installed") f = lambda: apply_token_bitmask_inplace_kernels["triton"](logits, bitmask, **kwargs) f() torch.testing.assert_close(logits, logits_expected.to("cuda")) torch.cuda.synchronize() exec_time = do_bench(f, warmup=args.warmup, rep=args.rep) exec_time *= 10**3 print(f"Implementation: {args.impl}\t| Execution time (μs): {exec_time:.4f}") xgrammar-0.1.19/examples/benchmark/bench_grammar_compile_mask_gen.py000066400000000000000000000157241500705317600257200ustar00rootroot00000000000000"""This script benchmarks the time for grammar compilation and mask generation.""" import argparse import json import time import datasets import torch from lmformatenforcer import JsonSchemaParser, TokenEnforcer from lmformatenforcer.integrations.transformers import ( TokenEnforcerTokenizerData, build_token_enforcer_tokenizer_data, ) from outlines.fsm.guide import Guide, RegexGuide from outlines.fsm.json_schema import convert_json_schema_to_str from outlines.generate.generator import bias_logits from outlines.generate.json import build_regex_from_schema from outlines.models import TransformerTokenizer from tqdm import tqdm from transformers import AutoTokenizer import xgrammar as xgr wrong_data_indices = [1] def xgrammar_build(schema: str, grammar_compiler: xgr.GrammarCompiler): grammar = grammar_compiler.compile_json_schema(schema) matcher = xgr.GrammarMatcher(grammar) return matcher def xgrammar_exec( matcher: xgr.GrammarMatcher, logits: torch.Tensor, bitmask: torch.Tensor, token_id: int ): # Logits processing matcher.fill_next_token_bitmask(bitmask) xgr.apply_token_bitmask_inplace(logits, bitmask) # Update state assert matcher.accept_token(token_id) return def outlines_build(schema: str, tokenizer: TransformerTokenizer): schema_str = convert_json_schema_to_str(json_schema=schema) regex_string = build_regex_from_schema(schema_str, whitespace_pattern=None) guide = RegexGuide.from_regex(regex_string, tokenizer) return guide def outlines_exec(guide: Guide, logits: torch.Tensor, token_id: int, state=None): if state is None: state = guide.initial_state # Logits processing allowed_tokens = guide.get_next_instruction(state).tokens biased_logits = bias_logits(logits.view(1, -1), [allowed_tokens]) # Update state next_state = guide.get_next_state(state, token_id) return next_state def lmformatenforcer_build(schema: str, tokenizer: TokenEnforcerTokenizerData): parser = JsonSchemaParser(json.loads(schema)) token_enforcer = TokenEnforcer(tokenizer, parser) return token_enforcer def lmformatenforcer_exec(token_enforcer: TokenEnforcer, logits: torch.Tensor, token_ids): # Logits processing allowed_tokens = token_enforcer.get_allowed_tokens(token_ids) logits[allowed_tokens] = float("-inf") # Update state return if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--backend", type=str, choices=["xgrammar", "outlines", "lmformatenforcer"], default="xgrammar", ) parser.add_argument("--num_iters", type=int, default=5) parser.add_argument("--num_warmup", type=int, default=-1) args = parser.parse_args() backend = args.backend num_iters = args.num_iters num_warmup = args.num_warmup if args.num_warmup != -1 else 5 if num_iters >= 40 else 1 dataset = datasets.load_dataset("NousResearch/json-mode-eval", split="train") hf_model_path = "meta-llama/Llama-3.1-8B-Instruct" hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_path) xgrammar_tokenizer_info = xgr.TokenizerInfo.from_huggingface(hf_tokenizer) xgrammar_grammar_compiler = xgr.GrammarCompiler(xgrammar_tokenizer_info) outlines_tokenizer = TransformerTokenizer(hf_tokenizer) lmformatenforcer_tokenizer = build_token_enforcer_tokenizer_data(hf_tokenizer) vocab_size = len(hf_tokenizer) build_time = 0 exec_time = 0 total_data_points = 0 total_tokens = 0 fail_cnt = 0 tqdm_iter = tqdm(range(-num_warmup, num_iters)) for iter in tqdm_iter: if iter < 0: tqdm_iter.set_description(f"Backend: {backend}, Warmup Iter: {iter + num_warmup}") else: tqdm_iter.set_description(f"Backend: {backend}, Iter: {iter}") if iter == 0: # Reset time build_time = 0 exec_time = 0 tqdm_data_point_iter = tqdm(range(len(dataset))) for data_point_idx in tqdm_data_point_iter: tqdm_data_point_iter.set_description( f"Backend: {backend}, Data Point: {data_point_idx}" ) if data_point_idx in wrong_data_indices: continue schema = dataset["schema"][data_point_idx] completion = dataset["completion"][data_point_idx] token_ids = hf_tokenizer.encode(completion, add_special_tokens=False) prompt = hf_tokenizer.apply_chat_template( dataset["prompt"][data_point_idx], tokenize=False ) prompt_token_ids = hf_tokenizer.encode(prompt) print(f"Prompt: {prompt}, Schema: {schema}") start = time.perf_counter() try: if backend == "xgrammar": worker = xgrammar_build(schema, xgrammar_grammar_compiler) bitmask = xgr.allocate_token_bitmask(worker.vocab_size) elif backend == "outlines": worker = outlines_build(schema, outlines_tokenizer) elif backend == "lmformatenforcer": worker = lmformatenforcer_build(schema, lmformatenforcer_tokenizer) except Exception as e: if iter >= 0: fail_cnt += 1 continue build_time += time.perf_counter() - start # use different logits for each mask generation process # to avoid caching effects between different tokens logits = [torch.randn(vocab_size).cuda() for _ in range(len(token_ids))] torch.cuda.synchronize() start = time.perf_counter() fail_flag = False for idx, token_id in enumerate(token_ids): # Logits processing try: if backend == "xgrammar": xgrammar_exec(worker, logits[idx], bitmask, token_id) elif backend == "outlines": if idx == 0: state = None state = outlines_exec(worker, logits[idx], token_id, state) elif backend == "lmformatenforcer": lmformatenforcer_exec( worker, logits[idx], prompt_token_ids + token_ids[:idx] ) except Exception as e: if iter >= 0: fail_cnt += 1 fail_flag = True break if fail_flag: continue torch.cuda.synchronize() exec_time += time.perf_counter() - start if iter >= 0: total_data_points += 1 total_tokens += len(token_ids) print(f"Backend: {backend}") print(f"Fail count: {fail_cnt / num_iters:.0f} / {len(dataset) - len(wrong_data_indices)}") print(f"Grammar preprocessing time (ms): {build_time / total_data_points * 1e3:.4f}") print(f"Mask generation time (us/token): {exec_time / total_tokens * 1e6:.4f}") xgrammar-0.1.19/examples/benchmark/cibench_grammar_compile_mask_gen.py000066400000000000000000000366011500705317600262310ustar00rootroot00000000000000"""This script benchmarks the time for grammar compilation and mask generation using XGrammar.""" import argparse import json import time from typing import Any, Dict, List, Tuple import datasets import requests import torch from tqdm import tqdm from transformers import AutoTokenizer import xgrammar as xgr wrong_data_indices = [1] def xgrammar_build(schema: str, grammar_compiler: xgr.GrammarCompiler): grammar = grammar_compiler.compile_json_schema(schema) matcher = xgr.GrammarMatcher(grammar) return matcher def download_gorilla_file(filename: str) -> Tuple[List, List]: base_url = "https://raw.githubusercontent.com/ShishirPatil/gorilla/main/berkeley-function-call-leaderboard/data" function_url = f"{base_url}/{filename}" answer_url = f"{base_url}/possible_answer/{filename}" print(f"Downloading {filename} from GitHub...") try: function_response = requests.get(function_url) function_response.raise_for_status() function_text = function_response.text functions_data = [] for line in function_text.strip().split("\n"): if line.strip(): try: functions_data.append(json.loads(line)) except json.JSONDecodeError as e: print(f"Error parsing function line in {filename}: {e}") answer_response = requests.get(answer_url) answer_response.raise_for_status() answer_text = answer_response.text answers_data = [] for line in answer_text.strip().split("\n"): if line.strip(): try: answers_data.append(json.loads(line)) except json.JSONDecodeError as e: print(f"Error parsing answer line in {filename}: {e}") print( f"Successfully downloaded {filename}: {len(functions_data)} functions, {len(answers_data)} answers" ) return functions_data, answers_data except requests.RequestException as e: print(f"Error downloading {filename}: {e}") return [], [] def load_gorilla_data() -> List[Dict[str, Any]]: gorilla_data = [] # excluding live test cases part of BFCL v2/v3 file_patterns = [ "BFCL_v3_java.json", "BFCL_v3_javascript.json", "BFCL_v3_multiple.json", "BFCL_v3_parallel.json", "BFCL_v3_parallel_multiple.json", "BFCL_v3_simple.json", "BFCL_v3_sql.json", ] filtered_count = 0 for filename in file_patterns: functions_data, answers_data = download_gorilla_file(filename) if not functions_data or not answers_data: print(f"Skipping {filename} - failed to download data") continue print(f"Processing {filename}...") answers_by_id = {item["id"]: item for item in answers_data} for item in functions_data: item_id = item["id"] if item_id not in answers_by_id: print(f"Warning: No answer found for item {item_id}") continue if "function" not in item or not item["function"]: print(f"Warning: No function definition for item {item_id}") filtered_count += 1 continue if len(item["function"]) > 1: # print(f"Skipping item {item_id} - contains multiple functions ({len(item['function'])})") filtered_count += 1 continue function_def = item["function"][0] # Use the first function schema = convert_function_to_schema(function_def) answer = answers_by_id[item_id] if "ground_truth" not in answer or not answer["ground_truth"]: print(f"Warning: No ground truth for item {item_id}") filtered_count += 1 continue ground_truth = answer["ground_truth"][0] # Use the first ground truth completion = convert_ground_truth_to_completion(ground_truth) gorilla_data.append( {"schema": schema, "completion": completion, "id": item_id, "source": filename} ) print( f"Loaded {len(gorilla_data)} examples from Gorilla BFCL dataset (filtered out {filtered_count} examples)" ) return gorilla_data def convert_function_to_schema(function_def: Dict) -> str: """Convert a Gorilla function definition to a JSON schema string with improved type handling.""" function_name = function_def["name"] parameters = function_def["parameters"] schema = { "type": "object", "properties": {function_name: {"type": "object", "properties": {}, "required": []}}, "required": [function_name], } for key, value in parameters.get("properties", {}).items(): param_type = value.get("type", "string").lower() if param_type == "integer": schema_def = {"type": "integer"} elif param_type in ("float", "number", "double"): schema_def = {"type": "number"} elif param_type == "boolean": schema_def = {"type": "boolean"} elif param_type in ("hashmap", "map", "dict", "dictionary"): schema_def = {"type": "object", "additionalProperties": True} elif param_type in ("array", "list"): schema_def = {"type": "array", "items": {"type": "string"}} elif param_type == "any": schema_def = {} # No type restriction else: schema_def = {"type": "string"} schema["properties"][function_name]["properties"][key] = schema_def required_fields = parameters.get("required", []) if required_fields: schema["properties"][function_name]["required"] = required_fields return json.dumps(schema) def convert_ground_truth_to_completion(ground_truth: Dict) -> str: """Convert a Gorilla ground truth to a completion string with improved handling of nested structures.""" function_name = list(ground_truth.keys())[0] params = ground_truth[function_name] transformed_params = {} for key, values in params.items(): if isinstance(values, list) and len(values) == 1 and isinstance(values[0], dict): nested_obj = {} for nested_key, nested_values in values[0].items(): if isinstance(nested_values, list) and nested_values: nested_obj[nested_key] = nested_values[0] else: nested_obj[nested_key] = nested_values transformed_params[key] = nested_obj elif isinstance(values, list) and values: transformed_params[key] = values[0] else: transformed_params[key] = None completion = {function_name: transformed_params} return json.dumps(completion) def run_benchmark( dataset_name: str, dataset_data, tokenizer_info, hf_tokenizer, num_iters, num_warmup ): vocab_size = len(hf_tokenizer) build_time = 0 exec_time = 0 total_data_points = 0 total_tokens = 0 fail_cnt = 0 schema_mismatch_cnt = 0 tqdm_iter = tqdm(range(-num_warmup, num_iters), disable=True) for iter in tqdm_iter: if iter < 0: tqdm_iter.set_description(f"{dataset_name} Warmup Iter: {iter + num_warmup}") else: tqdm_iter.set_description(f"{dataset_name} Iter: {iter}") if iter == 0: build_time = 0 exec_time = 0 tqdm_data_point_iter = tqdm(range(len(dataset_data)), disable=True) for data_point_idx in tqdm_data_point_iter: tqdm_data_point_iter.set_description(f"{dataset_name} Data Point: {data_point_idx}") if dataset_name == "json-mode-eval" and data_point_idx in wrong_data_indices: continue schema = dataset_data[data_point_idx]["schema"] completion = dataset_data[data_point_idx]["completion"] if dataset_name == "gorilla-bfcl": try: schema_obj = json.loads(schema) completion_obj = ( json.loads(completion) if isinstance(completion, str) else completion ) schema_function_name = schema_obj.get("required", [""])[0] completion_function_name = ( list(completion_obj.keys())[0] if completion_obj else "" ) if ( schema_function_name and completion_function_name and schema_function_name != completion_function_name ): if iter >= 0: schema_mismatch_cnt += 1 if iter == 0: print( f"Schema-completion function name mismatch for data point {data_point_idx}:" ) print(f" Schema expects: {schema_function_name}") print(f" Completion has: {completion_function_name}") continue except Exception as e: # If there's an issue parsing the JSON, proceed anyway pass if isinstance(completion, dict): completion = json.dumps(completion) token_ids = hf_tokenizer.encode(completion, add_special_tokens=False) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) start = time.perf_counter() try: worker = xgrammar_build(schema, grammar_compiler) bitmask = xgr.allocate_token_bitmask(1, vocab_size) except Exception as e: if iter >= 0: fail_cnt += 1 if iter == 0: print(f"Failed to build grammar for data point {data_point_idx}: {e}") continue build_time += time.perf_counter() - start # Use different logits for each mask generation process # to avoid caching effects between different tokens logits = [torch.randn(vocab_size).cuda() for _ in range(len(token_ids))] torch.cuda.synchronize() start = time.perf_counter() fail_flag = False token_rejection_count = 0 # give some leniency, can remove for idx, token_id in enumerate(token_ids): try: worker.fill_next_token_bitmask(bitmask) cuda_bitmask = bitmask.cuda() xgr.apply_token_bitmask_inplace(logits[idx], cuda_bitmask) # Update state if not worker.accept_token(token_id): token_rejection_count += 1 if token_rejection_count > 5: fail_flag = True break except Exception as e: if iter >= 0: if iter == 0: # Only print once to avoid spam print( f"Failed to process token {idx} for data point {data_point_idx}: {e}" ) fail_flag = True break if fail_flag: if iter >= 0: fail_cnt += 1 continue torch.cuda.synchronize() exec_time += time.perf_counter() - start if iter >= 0: total_data_points += 1 total_tokens += len(token_ids) results = { "dataset": dataset_name, "successful_data_points": total_data_points / num_iters if num_iters > 0 else 0, "failed_data_points": fail_cnt / num_iters if num_iters > 0 else 0, "schema_mismatch_count": schema_mismatch_cnt / num_iters if num_iters > 0 else 0, "total_possible_data_points": len(dataset_data) - (len(wrong_data_indices) if dataset_name == "json-mode-eval" else 0), "grammar_compilation_time_ms": ( build_time / total_data_points * 1e3 if total_data_points > 0 else float("inf") ), "per_token_overhead_us_per_token": ( exec_time / total_tokens * 1e6 if total_tokens > 0 else float("inf") ), } return results if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--num_iters", type=int, default=5) parser.add_argument("--num_warmup", type=int, default=-1) parser.add_argument( "--datasets", type=str, default="all", help="Datasets to benchmark: json-mode-eval, gorilla, or all", ) args = parser.parse_args() num_iters = args.num_iters num_warmup = args.num_warmup if args.num_warmup != -1 else 5 if num_iters >= 40 else 1 selected_datasets = args.datasets.lower() hf_model_path = "meta-llama/Llama-3.1-8B-Instruct" print(f"Loading tokenizer from {hf_model_path}...") hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_path) xgrammar_tokenizer_info = xgr.TokenizerInfo.from_huggingface(hf_tokenizer) # Try to get GPU info try: device_count = torch.cuda.device_count() device_name = torch.cuda.get_device_name(0) if device_count > 0 else "No GPU" print(f"Running benchmark with: {device_name} (Device count: {device_count})") except: print("Could not detect GPU information") results = [] if selected_datasets in ["json-mode-eval", "all"]: print("Loading json-mode-eval dataset...") json_mode_eval_dataset = datasets.load_dataset("NousResearch/json-mode-eval", split="train") json_mode_eval_data = [ {"schema": item["schema"], "completion": item["completion"]} for item in json_mode_eval_dataset ] print(f"Running benchmark on json-mode-eval ({len(json_mode_eval_data)} examples)...") json_mode_eval_results = run_benchmark( "json-mode-eval", json_mode_eval_data, xgrammar_tokenizer_info, hf_tokenizer, num_iters, num_warmup, ) results.append(json_mode_eval_results) if selected_datasets in ["gorilla", "all"]: print("Loading Gorilla BFCL dataset directly from GitHub...") gorilla_data = load_gorilla_data() if gorilla_data: print(f"Running benchmark on Gorilla BFCL ({len(gorilla_data)} examples)...") gorilla_results = run_benchmark( "gorilla-bfcl", gorilla_data, xgrammar_tokenizer_info, hf_tokenizer, num_iters, num_warmup, ) results.append(gorilla_results) else: print("No Gorilla data loaded, skipping benchmark") print("\n===== XGrammar Benchmark Results =====") print(f"Model: {hf_model_path}") print(f"Iterations: {num_iters}") print(f"Warmup Iterations: {num_warmup}") for result in results: print(f"\nDataset: {result['dataset']}") print( f"Successful data points: {result['successful_data_points']:.0f} / {result['total_possible_data_points']}" ) print( f"Failed data points: {result['failed_data_points']:.0f} / {result['total_possible_data_points']}" ) print(f"Grammar compilation time (ms): {result['grammar_compilation_time_ms']:.4f}") print(f"Per token overhead (us/token): {result['per_token_overhead_us_per_token']:.4f}") xgrammar-0.1.19/examples/hf_transformers/000077500000000000000000000000001500705317600204445ustar00rootroot00000000000000xgrammar-0.1.19/examples/hf_transformers/transformers_example.py000066400000000000000000000046231500705317600252630ustar00rootroot00000000000000""" This example demonstrates how to use XGrammar in Huggingface's transformers, integrated with a minimal LogitsProcessor. """ import torch from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer import xgrammar as xgr device = "cuda" # device = "cpu" # 0. Instantiate with any HF model you want model_name = "Qwen/Qwen2.5-0.5B-Instruct" # model_name = "microsoft/Phi-3.5-mini-instruct" # model_name = "meta-llama/Llama-3.2-1B-Instruct" model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype=torch.float32, device_map=device ) tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size # 1. Compile grammar (NOTE: you can substitute this with other grammars like EBNF, JSON Schema) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar: xgr.CompiledGrammar = grammar_compiler.compile_builtin_json_grammar() # 2. Prepare inputs messages_list = [] prompts = [ "Introduce yourself in JSON briefly as a student.", # Uncomment for batch generation # "Introduce yourself in JSON as a professor.", ] for prompt in prompts: messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt}, ] messages_list.append(messages) texts = [ tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) for messages in messages_list ] # For batched requests, either use a model that has a padding token, or specify your own # model_inputs = tokenizer(texts, return_tensors="pt", padding=True).to(model.device) model_inputs = tokenizer(texts, return_tensors="pt").to(model.device) # 3. Instantiate logits_processor per each generate, and call generate() xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) generated_ids = model.generate( **model_inputs, max_new_tokens=512, logits_processor=[xgr_logits_processor] ) # 4. Post-process outputs and print out response generated_ids = [ output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) ] responses = tokenizer.batch_decode(generated_ids, skip_special_tokens=True) for response in responses: print(response, end="\n\n") xgrammar-0.1.19/include/000077500000000000000000000000001500705317600150475ustar00rootroot00000000000000xgrammar-0.1.19/include/xgrammar/000077500000000000000000000000001500705317600166655ustar00rootroot00000000000000xgrammar-0.1.19/include/xgrammar/compiler.h000066400000000000000000000060601500705317600206520ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/compiler.h * \brief The header for the compiler. */ #ifndef XGRAMMAR_COMPILER_H_ #define XGRAMMAR_COMPILER_H_ #include #include #include #include #include #include namespace xgrammar { /*! * \brief The compiled grammar of a GrammarMatcher. It contains the preprocessing results of the * grammar and tokenizer. */ class CompiledGrammar { public: Grammar GetGrammar() const; TokenizerInfo GetTokenizerInfo() const; /*! \brief Return the approximate memory usage of the grammar in bytes. */ std::size_t MemorySizeBytes() const; XGRAMMAR_DEFINE_PIMPL_METHODS(CompiledGrammar); }; /*! * \brief A cache to get the compiled grammar for grammar or schema. This class avoids * redundant preprocessing of the grammar or schema when constructing a CompiledGrammar. * \note This class is associated with a vocabulary when constructed. The vocabulary is used to * create every compiled grammar. If multiple toke tables are used to create init * contexts, an instance of this class for each vocabulary should be created. */ class GrammarCompiler { public: /*! * \brief Construct a GrammarCompiler with a vocabulary. This class will always * create compiled grammars with this vocabulary. * \param tokenizer_info The tokenizer info. * \param max_threads The maximum number of threads to use for compiling grammars. * \param cache_enabled Whether to enable the cache. * \param max_memory_bytes The maximum memory usage in bytes. */ GrammarCompiler( const TokenizerInfo& tokenizer_info, int max_threads = 8, bool cache_enabled = true, long long max_memory_bytes = -1 // unlimited ); /*! \brief Get the compiled grammar for a JSON schema string. */ CompiledGrammar CompileJSONSchema( const std::string& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true ); /*! \brief Get the compiled grammar for pure JSON. */ CompiledGrammar CompileBuiltinJSONGrammar(); /*! \brief Get the compiled grammar for a grammar. */ CompiledGrammar CompileGrammar(const Grammar& grammar); /*! \brief Get the compiled grammar for a structural tag. */ CompiledGrammar CompileStructuralTag( const std::vector& tags, const std::vector& triggers ); /*! \brief Get the compiled grammar for a regex. */ CompiledGrammar CompileRegex(const std::string& regex); /*! \brief Clear the internal cache of compiled grammars. */ void ClearCache(); /*! \brief Return the approximate memory usage of the compiler in bytes. */ long long GetCacheSizeBytes() const; /*! \brief Return the approximate memory usage of the compiler in bytes. -1 means unlimited. */ long long CacheLimitBytes() const; XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarCompiler); }; } // namespace xgrammar #endif // XGRAMMAR_COMPILER_H_ xgrammar-0.1.19/include/xgrammar/grammar.h000066400000000000000000000147611500705317600204750ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/grammar.h * \brief The header for the definition and construction of BNF grammar. */ #ifndef XGRAMMAR_GRAMMAR_H_ #define XGRAMMAR_GRAMMAR_H_ #include #include #include #include namespace xgrammar { struct StructuralTagItem { std::string begin; std::string schema; std::string end; bool operator==(const StructuralTagItem& other) const { return begin == other.begin && schema == other.schema && end == other.end; } }; /*! * \brief This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar. * The BNF definition here is standard BNF, and the characters are represented using regex-style * character classes (e.g. [a-z], [^a-z]). * * \details * ### Rules * The BNF grammar AST consists of a set of rules. Each rule contains a name and a definition, and * corresponds to a production in the grammar. The definition of a rule is a RuleExpr. Each rule * has a rule_id for reference. * * ### RuleExprs * RuleExpr is the definition of a rule or part of the definition of a rule. It can contain * elements, empty string, reference to other RuleExprs, or reference to other rules. Each RuleExpr * corresponds to an rule_expr_id for reference. * * For example, in the following rule: rule ::= ("a" "b") | "c" * ("a" "b"), "c", ("a" "b") | "c" are all RuleExprs. * * #### Types of RuleExprs * Every RuleExpr is represented by a type as well as a variable-length array containing its data. * RuleExpr has several types: * - Byte string: a string of bytes (0~255). Supports UTF-8 strings. * - Character class: a range of characters (each character is a unicode codepoint), e.g. [a-z], * [ac-z]. Can be negated: [^a-z], [^ac-z]. Now only ascii chars is allowed in [], but this * expression can accept/reject unicode chars. * - Character class star: a star quantifier of a character class. e.g. [a-z]*, [^a-z]*. * - EmptyStr: an empty string, i.e. "" * - Rule reference: a reference to another rule * - Sequence: a sequence of rule_exprs, e.g. ("a" "b"). These rule_exprs are concatenated together. * - Choices: a choice of rule_exprs, e.g. ("a" "b") | "c". Each rule_expr can be matched. * * #### Storage of RuleExprs * Each type of RuleExpr has a different data format. For the format of each type of RuleExpr, see * docs in Grammar::Impl::RuleExprType. * * We store all RuleExprs in csr_matrix style. That is, they are stored consecutively in one vector * (data vector) and the starting position of each RuleExpr is recorded in the indptr vector. * * \remark The character class star RuleExpr is for the special support for elements like [a-z]* * in the grammar. We add it to make the matching more efficient, as we can avoid recursion into * rules when matching a sequence of characters. It should be used like: * rule1 ::= ((element1 element2 rule2 ...) | ...) * rule2 ::= character_class_star_rule_expr(id_of_a_character_class_rule_expr) */ class Grammar { public: /*! * \brief Get the EBNF string of the grammar. */ std::string ToString() const; /*! * \brief Construct a BNF grammar with a EBNF-formatted string. The grammar will be normalized * (simplified) by default. * \param ebnf_string The EBNF-formatted string. * \param root_rule_name The name of the root rule. */ static Grammar FromEBNF( const std::string& ebnf_string, const std::string& root_rule_name = "root" ); /*! * \brief Construct a BNF grammar from the json schema string. The schema string should be in the * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. * \param schema The schema string. * \param indent The number of spaces for indentation. If set to std::nullopt, the output will be * in one line. Default: 2. * \param separators Two separators used in the schema: comma and colon. Examples: {",", ":"}, * {", ", ": "}. If std::nullopt, the default separators will be used: {",", ": "} when the * indent is not nullopt, and {", ", ": "} otherwise. This follows the convention in python * json.dumps(). Default: std::nullopt. * \param strict_mode Whether to use strict mode. In strict mode, the generated grammar will not * allow properties and items that is not specified in the schema. This is equivalent to * setting unevaluatedProperties and unevaluatedItems to false. * * This helps LLM to generate accurate output in the grammar-guided generation with JSON * schema. Default: true. */ static Grammar FromJSONSchema( const std::string& schema, bool any_whitespace = true, std::optional indent = std::nullopt, std::optional> separators = std::nullopt, bool strict_mode = true, bool print_converted_ebnf = false ); /*! * \brief Construct a grammar from a regular expression string. * \param regex The regular expression string. * \param print_converted_ebnf This method will convert the regex to EBNF first. If this is true, * the converted EBNF string will be printed. For debugging purpose. Default: false. */ static Grammar FromRegex(const std::string& regex, bool print_converted_ebnf = false); /*! * \brief Construct a grammar from a regular expression string. * \param regex The regular expression string. */ static Grammar FromStructuralTag( const std::vector& tags, const std::vector& triggers ); /*! * \brief Get the grammar of standard JSON format. We have built-in support for JSON. */ static Grammar BuiltinJSONGrammar(); /*! * \brief Create a grammar that matches any of the grammars in the list. That is equivalent to * using the `|` operator to concatenate the grammars in the list. * \param grammars The grammars to create the union of. * \returns The union of the grammars. */ static Grammar Union(const std::vector& grammars); /*! * \brief Create a grammar that matches the concatenation of the grammars in the list. That is * equivalent to using the `+` operator to concatenate the grammars in the list. * \param grammars The grammars to create the concatenation of. * \returns The concatenation of the grammars. */ static Grammar Concat(const std::vector& grammars); /*! \brief Print a BNF grammar. */ friend std::ostream& operator<<(std::ostream& os, const Grammar& grammar); XGRAMMAR_DEFINE_PIMPL_METHODS(Grammar); }; } // namespace xgrammar #endif // XGRAMMAR_GRAMMAR_H_ xgrammar-0.1.19/include/xgrammar/matcher.h000066400000000000000000000110721500705317600204620ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/matcher.h * \brief The header for the matcher. */ #ifndef XGRAMMAR_MATCHER_H_ #define XGRAMMAR_MATCHER_H_ #include #include #include #include #include #include #include namespace xgrammar { int32_t GetBitmaskSize(int vocab_size); DLDataType GetBitmaskDLType(); void _DebugGetMaskedTokensFromBitmask( std::vector* rejected_tokens, const DLTensor& token_bitmask, int vocab_size, int index = 0 ); std::pair _IsSingleTokenBitmask(const DLTensor& bitmask, int vocab_size, int index); void ApplyTokenBitmaskInplaceCPU( DLTensor* logits, const DLTensor& bitmask, int vocab_size = -1, std::optional> indices = std::nullopt ); /*! * \brief A stateful matcher to match tokens to the specified BNF grammar. This class is the core * logic of the grammar-guided generation. * * \details This class implements the non-deterministic pushdown automaton (NPDA) matching algorithm * to match characters to a BNF grammar. It keep track of the current state of the matching process * by maintaining several stacks internally as possible paths in the NPDA. It also supports * backtracking. * * It is particularly capable of finding the set of tokens that are acceptable for the next step * and storing them in a bitmask. This aids in grammar-guided generation. * * \example * \code * Tokenizer tokenizer = ...; * auto compiled_grammar = GrammarMatcher::CreateCompiledGrammar(grammar, * tokenizer->PostProcessedVocab()); * GrammarMatcher matcher(compiled_grammar, 10); * matcher->AcceptToken(67); * * // Construct a DLTensor with shape (tokenizer.GetVocabSize() + 31) / 32, and dtype int32. * DLTensor next_token_bitmask = ...; * matcher->FillNextTokenBitmask(&next_token_bitmask); * * // Rollback is supported * matcher->Rollback(1); * \endcode */ class GrammarMatcher { public: /*! * \brief Construct a GrammarMatcher from the preprocessing result of type * CompiledGrammar. * \param compiled_grammar The compiled grammar. It is obtained through * CreateCompiledGrammar as a result of preprocessing the grammar and tokenizer. */ GrammarMatcher( const CompiledGrammar& compiled_grammar, std::optional> override_stop_tokens = std::nullopt, bool terminate_without_stop_token = false, int max_rollback_tokens = 0 ); /*! * \brief Accept one token and update the state of the matcher. * \param token_id The id of the token to accept. * \return Whether the token is accepted. * \note Termination state. * When the end of the root rule is reached, the matcher can only accept the stop token. * The matcher is terminated after accepting the stop token, i.e. no AcceptToken or * FindNextTokenMask operations can be performed. The termination state can be canceled * using Rollback(). */ bool AcceptToken(int32_t token_id, bool debug_print = false); /*! * \brief Get the set of tokens that are acceptable for the next step and store them in a * bitmask. * \param next_token_bitmask The bitmask to store the result. The bitmask must be pre-allocated * and with shape (GetBitmaskSize(),) and dtype int32. * \return Whether the bitmask need to be applied (not all-true). */ bool FillNextTokenBitmask(DLTensor* next_token_bitmask, int index = 0, bool debug_print = false); /*! * \brief Find the jump-forward string for jump-forward decoding. This is the longest string that will be valid according to the current syntax. * \note This method does not change the grammar state. */ std::string FindJumpForwardString(); /*! * \brief Rollback the matcher to a previous state. * \param num_tokens The number of tokens to rollback. It cannot exceed the current number of * steps, nor can it exceed the specified maximum number of rollback tokens. */ void Rollback(int num_tokens = 1); /*! * \brief Check if the matcher has accepted the stop token and terminated. * \sa AcceptToken */ bool IsTerminated() const; /*! \brief Reset the matcher to the initial state. */ void Reset(); /*! \brief Get the maximum number of rollback tokens allowed. */ int GetMaxRollbackTokens() const; const std::vector& GetStopTokenIds() const; bool _DebugAcceptString(const std::string& input_str, bool debug_print = false); XGRAMMAR_DEFINE_PIMPL_METHODS(GrammarMatcher); }; } // namespace xgrammar #endif // XGRAMMAR_MATCHER_H_ xgrammar-0.1.19/include/xgrammar/object.h000066400000000000000000000040501500705317600203030ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/object.h * \brief Utilities for creating objects. */ #ifndef XGRAMMAR_OBJECT_H_ #define XGRAMMAR_OBJECT_H_ #include // IWYU pragma: keep #include // IWYU pragma: keep namespace xgrammar { /*! * \brief A tag type for empty constructor. * * Since XGRAMMAR_DEFINE_PIMPL_METHODS already occupies the default constructor to * construct a null object, this tag is used to define an empty constructor for * the object. */ struct EmptyConstructorTag {}; #define XGRAMMAR_DEFINE_PIMPL_METHODS(TypeName) \ public: \ class Impl; \ /* The default constructor constructs a null object. Note operating on a */ \ /* null object will fail. */ \ explicit TypeName() : pimpl_(nullptr) {} \ /* Construct object with a shared pointer to impl. The object just stores */ \ /* a pointer. */ \ explicit TypeName(std::shared_ptr pimpl) : pimpl_(std::move(pimpl)) {} \ TypeName(const TypeName& other) = default; \ TypeName(TypeName&& other) noexcept = default; \ TypeName& operator=(const TypeName& other) = default; \ TypeName& operator=(TypeName&& other) noexcept = default; \ /* Access the impl pointer. Useful in implementation. */ \ Impl* operator->() { return pimpl_.get(); } \ const Impl* operator->() const { return pimpl_.get(); } \ \ private: \ std::shared_ptr pimpl_ } // namespace xgrammar #endif // XGRAMMAR_OBJECT_H_ xgrammar-0.1.19/include/xgrammar/tokenizer_info.h000066400000000000000000000026351500705317600220710ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/tokenizer_info.h * \brief The header for the tokenizer info. */ #ifndef XGRAMMAR_TOKENIZER_INFO_H_ #define XGRAMMAR_TOKENIZER_INFO_H_ #include #include #include #include #include namespace xgrammar { enum class VocabType : int { RAW = 0, BYTE_FALLBACK = 1, BYTE_LEVEL = 2, }; class TokenizerInfo { public: TokenizerInfo( const std::vector& encoded_vocab, VocabType vocab_type = VocabType::RAW, std::optional vocab_size = std::nullopt, std::optional> stop_token_ids = std::nullopt, bool add_prefix_space = false ); VocabType GetVocabType() const; bool GetAddPrefixSpace() const; int GetVocabSize() const; const std::vector& GetDecodedVocab() const; const std::vector& GetStopTokenIds() const; const std::vector& GetSpecialTokenIds() const; const std::vector>& GetSortedDecodedVocab() const; std::string DumpMetadata() const; static TokenizerInfo FromVocabAndMetadata( const std::vector& encoded_vocab, const std::string& metadata ); static std::string DetectMetadataFromHF(const std::string& backend_str); XGRAMMAR_DEFINE_PIMPL_METHODS(TokenizerInfo); }; } // namespace xgrammar #endif // XGRAMMAR_TOKENIZER_INFO_H_ xgrammar-0.1.19/include/xgrammar/xgrammar.h000066400000000000000000000005541500705317600206600ustar00rootroot00000000000000/*! * Copyright (c) 2024 by Contributors * \file xgrammar/xgrammar.h * \brief The header for the support of grammar-guided generation. */ #ifndef XGRAMMAR_XGRAMMAR_H_ #define XGRAMMAR_XGRAMMAR_H_ #include #include #include #include #endif // XGRAMMAR_XGRAMMAR_H_ xgrammar-0.1.19/pyproject.toml000066400000000000000000000075361500705317600163530ustar00rootroot00000000000000[project] name = "xgrammar" description = "Efficient, Flexible and Portable Structured Generation" authors = [{ name = "MLC Team" }] readme = "README.md" license = { text = "Apache 2.0" } classifiers = [ "License :: OSI Approved :: Apache Software License", "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Education", "Intended Audience :: Science/Research", ] keywords = ["machine learning", "inference"] requires-python = ">=3.8, <4" dependencies = [ "pydantic", "sentencepiece", "tiktoken", "torch>=1.10.0", "transformers>=4.38.0", "triton; platform_system == 'Linux' and platform_machine == 'x86_64'", "mlx-lm; platform_system == 'Darwin' and platform_machine == 'arm64'", "ninja", ] dynamic = ["version"] [project.urls] Homepage = "https://xgrammar.mlc.ai/" GitHub = "https://github.com/mlc-ai/xgrammar" [project.optional-dependencies] test = [ "pytest", "protobuf", "huggingface-hub[cli]", # transformers==4.50.0 has error on MacOS. # https://github.com/huggingface/transformers/issues/36906 "transformers<4.50.0; platform_system == 'Darwin'", ] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" input = "python/xgrammar/version.py" [build-system] requires = ["scikit-build-core>=0.10.0", "nanobind==2.5.0"] build-backend = "scikit_build_core.build" [tool.scikit-build] minimum-version = "build-system.requires" # Build configuration build-dir = "build" build.verbose = true # CMake configuration cmake.version = "CMakeLists.txt" cmake.args = [] cmake.build-type = "RelWithDebInfo" # Logging logging.level = "INFO" # Wheel configuration wheel.packages = ["python/xgrammar"] wheel.install-dir = "xgrammar" # Source distribution configuration sdist.include = [ # Build files "/CMakeLists.txt", "/pyproject.toml", "/cmake/**/*", "/cpp/**/CMakeLists.txt", # Source code "/cpp/**/*.cc", "/cpp/**/*.cpp", "/cpp/**/*.h", "/include/**/*", "/python/xgrammar/**/*.py", # Third party files "/3rdparty/**/*", # Documentation and metadata "/docs/**/*", "/LICENSE", "/README.md", "/NOTICE", # Tests "/tests/**/*", ] sdist.exclude = ["**/.git", "**/.github", "**/__pycache__", "**/*.pyc", "build", "dist"] # Editable install settings editable.rebuild = true editable.verbose = true [tool.pytest.ini_options] addopts = "-rA --durations=0 --ignore=3rdparty" markers = ["hf_token_required: mark test as requiring a huggingface token"] [tool.mypy] strict = true [tool.ruff] include = ["python/**/*.py", "tests/**/*.py"] [tool.ruff.lint] # Never enforce `E501` (line length violations). ignore = ["C901", "E501", "E741", "F402", "F823", "E731"] select = ["C", "E", "F", "W"] [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] "tests/*" = ["E741"] [tool.ruff.lint.pylint] max-args = 10 [tool.black] exclude = "3rdparty/*" line-length = 100 skip-magic-trailing-comma = true [tool.isort] profile = "black" src_paths = ["python", "tests"] extend_skip = ["3rdparty"] line_length = 100 skip_gitignore = true [tool.cibuildwheel] build-verbosity = 1 # pypy doesn't play nice with pybind11 so skip pp* builds # pytorch stopped supporting Mac x64 back in 2.2 so there will be no Mac x64 wheels for python 3.13 so skip cp313-macosx_x86_64 # python 3.13 support is still early and wheels are missing for Linux aarch64 for pytorch so temporarily skip cp313-manylinux_aarch64 skip = [ "cp36-*", "cp37-*", "cp38-*", "pp*", "*musllinux*", "cp313-manylinux_aarch64", "cp313-macosx_x86_64", ] # pypy doesn't play nice with pybind11 build-frontend = "build[uv]" test-command = "pytest {project}/tests -m \"not hf_token_required\"" test-extras = ["test"] [tool.cibuildwheel.linux] archs = ["x86_64", "aarch64"] [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] environment = { MACOSX_DEPLOYMENT_TARGET = "10.14" } [tool.cibuildwheel.windows] archs = ["AMD64"] xgrammar-0.1.19/python/000077500000000000000000000000001500705317600147455ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/000077500000000000000000000000001500705317600165635ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/__init__.py000066400000000000000000000005671500705317600207040ustar00rootroot00000000000000from . import testing from .compiler import CompiledGrammar, GrammarCompiler from .contrib import hf from .grammar import Grammar, StructuralTagItem from .matcher import ( GrammarMatcher, allocate_token_bitmask, apply_token_bitmask_inplace, bitmask_dtype, get_bitmask_shape, reset_token_bitmask, ) from .tokenizer_info import TokenizerInfo, VocabType xgrammar-0.1.19/python/xgrammar/base.py000066400000000000000000000044701500705317600200540ustar00rootroot00000000000000"""This module provides classes to handle C++ objects from nanobind.""" import os if os.environ.get("XGRAMMAR_BUILD_DOCS") != "1": from . import xgrammar_bindings as _core else: _core = "dummy namespace" class XGRObject: """The base class for all objects in XGrammar. This class provides methods to handle the C++ handle from nanobind. In subclasses, the handle should be initialized via the the _create_from_handle, or via the _init_handle method called within the __init__ method, and should not be modified afterwards. Subclasses should use the _handle property to access the handle. When comparing two objects, the equality is checked by comparing the C++ handles. For performance considerations, objects in XGrammar should be lightweight and only maintain a handle to the C++ objects. Heavy operations should be performed on the C++ side. """ @classmethod def _create_from_handle(cls, handle) -> "XGRObject": """Construct an object of the class from a C++ handle. Parameters ---------- cls The class of the object. handle The C++ handle. Returns ------- obj : XGRObject An object of type cls. """ obj = cls.__new__(cls) obj.__handle = handle return obj def _init_handle(self, handle): """Initialize an object with a handle. This method should be called in the __init__ method of the subclasses of XGRObject to initialize the C++ handle. Parameters ---------- handle The C++ handle. """ self.__handle = handle @property def _handle(self): """Get the C++ handle of the object. Returns ------- handle The C++ handle. """ return self.__handle def __eq__(self, other: object) -> bool: """Compare two XGrammar objects by comparing their C++ handles. Parameters ---------- other : object The other object to compare with. Returns ------- equal : bool Whether the two objects have the same C++ handle. """ if not isinstance(other, XGRObject): return NotImplemented return self._handle == other._handle xgrammar-0.1.19/python/xgrammar/compiler.py000066400000000000000000000170601500705317600207530ustar00rootroot00000000000000"""Compiling grammar for efficient token mask generation.""" from typing import Any, Dict, List, Optional, Tuple, Type, Union, overload from pydantic import BaseModel from .base import XGRObject, _core from .grammar import Grammar, StructuralTagItem, _convert_schema_to_str from .tokenizer_info import TokenizerInfo class CompiledGrammar(XGRObject): """This is the primary object to store compiled grammar. A CompiledGrammar can be used to construct GrammarMatcher to generate token masks efficiently. Note ---- Do not construct this class directly, instead use :class:`GrammarCompiler` to construct the object. """ @property def grammar(self) -> Grammar: """The original grammar.""" return Grammar._create_from_handle(self._handle.grammar) @property def tokenizer_info(self) -> TokenizerInfo: """The tokenizer info associated with the compiled grammar.""" return TokenizerInfo._create_from_handle(self._handle.tokenizer_info) @property def memory_size_bytes(self) -> int: """The approximate memory usage of the compiled grammar in bytes.""" return self._handle.memory_size_bytes class GrammarCompiler(XGRObject): """The compiler for grammars. It is associated with a certain tokenizer info, and compiles grammars into CompiledGrammar with the tokenizer info. It allows parallel compilation with multiple threads, and has a cache to store the compilation result, avoiding compiling the same grammar multiple times. Parameters ---------- tokenizer_info : TokenizerInfo The tokenizer info. max_threads : int, default: 8 The maximum number of threads used to compile the grammar. cache_enabled : bool, default: True Whether to enable the cache. cache_limit_bytes : int, default: -1 The maximum memory usage for the cache in the specified unit. Note that the actual memory usage may slightly exceed this value. """ def __init__( self, tokenizer_info: TokenizerInfo, *, max_threads: int = 8, cache_enabled: bool = True, cache_limit_bytes: int = -1, ): if not isinstance(tokenizer_info, TokenizerInfo): raise ValueError( "Please convert the tokenizer to TokenizerInfo before passing it " "to GrammarCompiler." ) self._init_handle( _core.GrammarCompiler( tokenizer_info._handle, max_threads, cache_enabled, cache_limit_bytes ) ) def compile_json_schema( self, schema: Union[str, Type[BaseModel], Dict[str, Any]], *, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, ) -> CompiledGrammar: """Get CompiledGrammar from the specified JSON schema and format. The indent and separators parameters follow the same convention as in json.dumps(). Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema string or Pydantic model or JSON schema dict. indent : Optional[int], default: None The number of spaces for indentation. If None, the output will be in one line. separators : Optional[Tuple[str, str]], default: None Two separators used in the schema: comma and colon. Examples: (",", ":"), (", ", ": "). If None, the default separators will be used: (",", ": ") when the indent is not None, and (", ", ": ") otherwise. strict_mode : bool, default: True Whether to use strict mode. In strict mode, the generated grammar will not allow properties and items that is not specified in the schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. This helps LLM to generate accurate output in the grammar-guided generation with JSON schema. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ schema_str = _convert_schema_to_str(schema) return CompiledGrammar._create_from_handle( self._handle.compile_json_schema( schema_str, any_whitespace, indent, separators, strict_mode ) ) def compile_builtin_json_grammar(self) -> CompiledGrammar: """Get CompiledGrammar from the standard JSON. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ return CompiledGrammar._create_from_handle(self._handle.compile_builtin_json_grammar()) def compile_regex(self, regex: str) -> CompiledGrammar: """Get CompiledGrammar from the specified regex. Parameters ---------- regex : str The regex string. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ return CompiledGrammar._create_from_handle(self._handle.compile_regex(regex)) def compile_structural_tag( self, tags: List[StructuralTagItem], triggers: List[str] ) -> CompiledGrammar: """Compile a grammar from structural tags. See Grammar.from_structural_tag() for more details. Parameters ---------- tags : List[StructuralTagItem] The structural tags. triggers : List[str] The triggers. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ tags_tuple = [(tag.begin, _convert_schema_to_str(tag.schema_), tag.end) for tag in tags] return CompiledGrammar._create_from_handle( self._handle.compile_structural_tag(tags_tuple, triggers) ) @overload def compile_grammar(self, ebnf_string: str, *, root_rule_name: str = "root") -> CompiledGrammar: """Compile a grammar from EBNF string. The EBNF string should follow the format in https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md. Parameters ---------- ebnf_string : str The grammar string in EBNF format. root_rule_name : str, default: "root" The name of the root rule in the grammar. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ ... @overload def compile_grammar(self, grammar: Grammar) -> CompiledGrammar: """Compile a grammar object. Returns ------- compiled_grammar : CompiledGrammar The compiled grammar. """ ... def compile_grammar( self, grammar: Union[str, Grammar], *, root_rule_name: str = "root" ) -> CompiledGrammar: if isinstance(grammar, str): grammar = Grammar.from_ebnf(grammar, root_rule_name=root_rule_name) return CompiledGrammar._create_from_handle(self._handle.compile_grammar(grammar._handle)) def clear_cache(self) -> None: """Clear all cached compiled grammars.""" self._handle.clear_cache() def get_cache_size_bytes(self) -> int: """The approximate memory usage of the cache in bytes.""" return self._handle.get_cache_size_bytes() @property def cache_limit_bytes(self) -> int: """ The maximum memory usage for the cache in bytes. Returns -1 if the cache has no memory limit. """ return self._handle.cache_limit_bytes xgrammar-0.1.19/python/xgrammar/contrib/000077500000000000000000000000001500705317600202235ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/contrib/__init__.py000066400000000000000000000000001500705317600223220ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/contrib/hf.py000066400000000000000000000110061500705317600211700ustar00rootroot00000000000000""" This file helps integrate xgrammar in HF transformers package by extending transformers.LogitsProcessor, which is to be fed to `model.generate()`. """ from typing import List, Union import torch import transformers import xgrammar as xgr class LogitsProcessor(transformers.LogitsProcessor): """ LogitsProcessor for processing logits in transformers' generate() method. Example usage ------------- .. code:: python model_name = "Qwen/Qwen2.5-0.5B-Instruct" tokenizer = AutoTokenizer.from_pretrained(model_name) config = AutoConfig.from_pretrained(model_name) # This can be larger than tokenizer.vocab_size due to paddings full_vocab_size = config.vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=full_vocab_size) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = grammar_compiler.compile_builtin_json_grammar() xgr_logits_processor = xgr.contrib.hf.LogitsProcessor(compiled_grammar) model.generate(prompt, logits_processor=[xgr_logits_processor]) For an end-to-end example, see folder `examples/hf_transformers/`. Notes ----- - Note that this LogitsProcessor can only be used once. For each `generate()` call, instantiate a new one. - Note that this implementation may contain extra overhead. """ def __init__(self, compiled_grammar: Union[xgr.CompiledGrammar, List[xgr.CompiledGrammar]]): """Initialize the LogitsProcessor. Parameters ---------- compiled_grammar : xgr.CompiledGrammar | List[xgr.CompiledGrammar] One or more grammars compiled according to the given grammar and the model's tokenizer_info. """ self.matchers: List[xgr.GrammarMatcher] = [] self.compiled_grammars: List[xgr.CompiledGrammar] = ( compiled_grammar if isinstance(compiled_grammar, list) else [compiled_grammar] ) self.full_vocab_size = self.compiled_grammars[0].tokenizer_info.vocab_size self.token_bitmask = None self.prefilled = False self.batch_size = 0 def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor: """ Accept token sampled in the last iteration, fill in bitmask, and apply bitmask to logits. Returns: scores: Logits modified with bitmask. """ # Lazily initialize GrammarMatchers and bitmask if len(self.matchers) == 0: self.batch_size = input_ids.shape[0] self.compiled_grammars = ( self.compiled_grammars if len(self.compiled_grammars) > 1 else self.compiled_grammars * self.batch_size ) assert ( len(self.compiled_grammars) == self.batch_size ), "The number of compiled grammars must be equal to the batch size." self.matchers = [ xgr.GrammarMatcher(self.compiled_grammars[i]) for i in range(self.batch_size) ] self.token_bitmask = xgr.allocate_token_bitmask(self.batch_size, self.full_vocab_size) if input_ids.shape[0] != self.batch_size: raise RuntimeError( "Expect input_ids.shape[0] to be LogitsProcessor.batch_size." + f"Got {input_ids.shape[0]} for the former, and {self.batch_size} for the latter." ) if not self.prefilled: # Have not sampled a token yet self.prefilled = True else: for i in range(self.batch_size): if not self.matchers[i].is_terminated(): sampled_token = input_ids[i][-1] assert self.matchers[i].accept_token(sampled_token) for i in range(self.batch_size): if not self.matchers[i].is_terminated(): self.matchers[i].fill_next_token_bitmask(self.token_bitmask, i) # We only support masking logits on CUDA or CPU device_type = scores.device.type if device_type != "cuda": scores = scores.to("cpu") xgr.apply_token_bitmask_inplace(scores, self.token_bitmask.to(scores.device)) if device_type != "cuda": scores = scores.to(device_type) # NOTE: Cannot reset here because __call__ is not invoked when stop token # is sampled. This is why each `generate()` call needs to instantiate an # LogitsProcessor return scores xgrammar-0.1.19/python/xgrammar/contrib/mlxlm.py000066400000000000000000000053741500705317600217370ustar00rootroot00000000000000""" Usage: python mlxlm.py --model mlx-community/Qwen2.5-Coder-32B-Instruct-3bit """ import argparse import mlx.core as mx from mlx_lm.generate import generate as mlx_generate from mlx_lm.utils import load as mlx_load from transformers import AutoTokenizer import xgrammar from xgrammar.kernels import apply_token_bitmask_inplace_kernels class XGrammarLogitsProcessor: def __init__(self, grammar: xgrammar.CompiledGrammar, max_rollback_tokens: int = 16): self.matcher = xgrammar.GrammarMatcher(grammar, max_rollback_tokens=max_rollback_tokens) self.vocab_size = grammar.tokenizer_info.vocab_size self.bitmask = xgrammar.allocate_token_bitmask(1, self.vocab_size) def __call__(self, tokens: mx.array, logits: mx.array) -> mx.array: assert tokens.size > 0 # In the first call, tokens.size == #tokens in prompt last_token = tokens[-1].item() acc = self.matcher.accept_token(last_token) if not self.matcher.is_terminated() else False if not acc: self.matcher.reset() self.matcher.accept_token(last_token) if not self.matcher.is_terminated(): self.matcher.fill_next_token_bitmask(self.bitmask) return apply_token_bitmask_inplace_kernels["metal"]( mx.array(self.bitmask.numpy()), logits, self.vocab_size ) return logits def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--model", type=str, required=True) parser.add_argument( "--prompt", type=str, default="Generate a simple example JSON. No text. Only the JSON" ) parser.add_argument("--seed", type=int, default=42) return parser.parse_args() def main(): args = parse_args() model, _ = mlx_load(args.model) tokenizer = AutoTokenizer.from_pretrained(args.model) mx.random.seed(args.seed) with_logits_processor = mlx_generate( model=model, tokenizer=tokenizer, prompt=tokenizer.apply_chat_template( [{"role": "user", "content": args.prompt}], add_generation_prompt=True ), verbose=False, logits_processors=[ XGrammarLogitsProcessor( grammar=xgrammar.GrammarCompiler( tokenizer_info=xgrammar.TokenizerInfo.from_huggingface(tokenizer) ).compile_builtin_json_grammar() ) ], ) without_logits_processor = mlx_generate( model=model, tokenizer=tokenizer, prompt=tokenizer.apply_chat_template( [{"role": "user", "content": args.prompt}], add_generation_prompt=True ), verbose=False, ) assert without_logits_processor == with_logits_processor print(without_logits_processor) if __name__ == "__main__": main() xgrammar-0.1.19/python/xgrammar/grammar.py000066400000000000000000000314201500705317600205630ustar00rootroot00000000000000"""This module provides classes representing grammars.""" import json from typing import Any, Dict, List, Optional, Tuple, Type, Union from pydantic import BaseModel, Field from .base import XGRObject, _core class StructuralTagItem(BaseModel): """A structural tag item. See Grammar.from_structural_tag() for more details. Attributes ---------- begin : str The begin tag. schema_ : Union[str, Type[BaseModel]] The schema. end : str The end tag. """ begin: str schema_: Union[str, Type[BaseModel], Dict[str, Any]] = Field(alias="schema") end: str def _convert_schema_to_str(schema: Union[str, Type[BaseModel], Dict[str, Any]]) -> str: """Convert a schema to a string representation. This function handles different schema input types and converts them to a JSON string: - Pydantic models are converted using their schema methods - String inputs are returned as-is (assumed to be valid JSON) - Dictionary inputs are converted to JSON strings Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema to convert, which can be a Pydantic model class, a JSON schema string, or a dictionary representing a JSON schema. Returns ------- str The JSON schema as a string. Raises ------ ValueError, TypeError If the schema type is not supported, or the dictionary is not serializable. """ if isinstance(schema, type) and issubclass(schema, BaseModel): if hasattr(schema, "model_json_schema"): return json.dumps(schema.model_json_schema()) if hasattr(schema, "schema_json"): return json.dumps(schema.schema_json()) else: raise ValueError("The schema should have a model_json_schema or json_schema method.") elif isinstance(schema, str): return schema elif isinstance(schema, dict): return json.dumps(schema) else: raise ValueError("The schema should be a string or a Pydantic model.") class Grammar(XGRObject): """This class represents a grammar object in XGrammar, and can be used later in the grammar-guided generation. The Grammar object supports context-free grammar (CFG). EBNF (extended Backus-Naur Form) is used as the format of the grammar. There are many specifications for EBNF in the literature, and we follow the specification of GBNF (GGML BNF) in https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md. When printed, the grammar will be converted to GBNF format. """ def __str__(self) -> str: """Print the BNF grammar to a string, in EBNF format. Returns ------- grammar_string : str The BNF grammar string. """ return self._handle.to_string() @staticmethod def from_ebnf(ebnf_string: str, *, root_rule_name: str = "root") -> "Grammar": """Construct a grammar from EBNF string. The EBNF string should follow the format in https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md. Parameters ---------- ebnf_string : str The grammar string in EBNF format. root_rule_name : str, default: "root" The name of the root rule in the grammar. Raises ------ RuntimeError When converting the regex pattern fails, with details about the parsing error. """ return Grammar._create_from_handle(_core.Grammar.from_ebnf(ebnf_string, root_rule_name)) @staticmethod def from_json_schema( schema: Union[str, Type[BaseModel], Dict[str, Any]], *, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, print_converted_ebnf: bool = False, ) -> "Grammar": """Construct a grammar from JSON schema. Pydantic model or JSON schema string can be used to specify the schema. It allows any whitespace by default. If user want to specify the format of the JSON, set `any_whitespace` to False and use the `indent` and `separators` parameters. The meaning and the default values of the parameters follows the convention in json.dumps(). It internally converts the JSON schema to a EBNF grammar. Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema string or Pydantic model or JSON schema dict. any_whitespace : bool, default: True Whether to use any whitespace. If True, the generated grammar will ignore the indent and separators parameters, and allow any whitespace. indent : Optional[int], default: None The number of spaces for indentation. If None, the output will be in one line. Note that specifying the indentation means forcing the LLM to generate JSON strings strictly formatted. However, some models may tend to generate JSON strings that are not strictly formatted. In this case, forcing the LLM to generate strictly formatted JSON strings may degrade the generation quality. See for more details. separators : Optional[Tuple[str, str]], default: None Two separators used in the schema: comma and colon. Examples: (",", ":"), (", ", ": "). If None, the default separators will be used: (",", ": ") when the indent is not None, and (", ", ": ") otherwise. strict_mode : bool, default: True Whether to use strict mode. In strict mode, the generated grammar will not allow properties and items that is not specified in the schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. This helps LLM to generate accurate output in the grammar-guided generation with JSON schema. print_converted_ebnf : bool, default: False If True, the converted EBNF string will be printed. For debugging purposes. Returns ------- grammar : Grammar The constructed grammar. Raises ------ RuntimeError When converting the json schema fails, with details about the parsing error. """ schema_str = _convert_schema_to_str(schema) return Grammar._create_from_handle( _core.Grammar.from_json_schema( schema_str, any_whitespace, indent, separators, strict_mode, print_converted_ebnf ) ) @staticmethod def from_regex(regex_string: str, *, print_converted_ebnf: bool = False) -> "Grammar": """Create a grammar from a regular expression string. Parameters ---------- regex_string : str The regular expression pattern to create the grammar from. print_converted_ebnf : bool, default: False This method will convert the regex pattern to EBNF first. If this is true, the converted EBNF string will be printed. For debugging purposes. Default: False. Returns ------- grammar : Grammar The constructed grammar from the regex pattern. Raises ------ RuntimeError When parsing the regex pattern fails, with details about the parsing error. """ return Grammar._create_from_handle( _core.Grammar.from_regex(regex_string, print_converted_ebnf) ) @staticmethod def from_structural_tag(tags: List[StructuralTagItem], triggers: List[str]) -> "Grammar": """Create a grammar from structural tags. The structural tag handles the dispatching of different grammars based on the tags and triggers: it initially allows any output, until a trigger is encountered, then dispatch to the corresponding tag; when the end tag is encountered, the grammar will allow any following output, until the next trigger is encountered. The tags parameter is used to specify the output pattern. It is especially useful for LLM function calling, where the pattern is: {"arg1": ..., "arg2": ...}. This pattern consists of three parts: a begin tag (), a parameter list according to some schema ({"arg1": ..., "arg2": ...}), and an end tag (). This pattern can be described in a StructuralTagItem with a begin tag, a schema, and an end tag. The structural tag is able to handle multiple such patterns by passing them into multiple tags. The triggers parameter is used to trigger the dispatching of different grammars. The trigger should be a prefix of a provided begin tag. When the trigger is encountered, the corresponding tag should be used to constrain the following output. There can be multiple tags matching the same trigger. Then if the trigger is encountered, the following output should match one of the tags. For example, in function calling, the triggers can be ["). The corrrespondence of tags and triggers is automatically determined: all tags with the same trigger will be grouped together. User should make sure any trigger is not a prefix of another trigger: then the corrrespondence of tags and triggers will be ambiguous. To use this grammar in grammar-guided generation, the GrammarMatcher constructed from structural tag will generate a mask for each token. When the trigger is not encountered, the mask will likely be all-1 and not have to be used (fill_next_token_bitmask returns False, meaning no token is masked). When a trigger is encountered, the mask should be enforced (fill_next_token_bitmask will return True, meaning some token is masked) to the output logits. The benefit of this method is the token boundary between tags and triggers is automatically handled. The user does not need to worry about the token boundary. Parameters ---------- tags : List[StructuralTagItem] The structural tags. triggers : List[str] The triggers. Examples -------- >>> class Schema1(BaseModel): >>> arg1: str >>> arg2: int >>> class Schema2(BaseModel): >>> arg3: float >>> arg4: List[str] >>> tags = [ >>> StructuralTagItem(begin="", schema=Schema1, end=""), >>> StructuralTagItem(begin="", schema=Schema2, end=""), >>> ] >>> triggers = [">> grammar = Grammar.from_structural_tag(tags, triggers) """ tags_tuple = [(tag.begin, _convert_schema_to_str(tag.schema_), tag.end) for tag in tags] return Grammar._create_from_handle(_core.Grammar.from_structural_tag(tags_tuple, triggers)) @staticmethod def builtin_json_grammar() -> "Grammar": """Get the grammar of standard JSON. This is compatible with the official JSON grammar specification in https://www.json.org/json-en.html. Returns ------- grammar : Grammar The JSON grammar. """ return Grammar._create_from_handle(_core.Grammar.builtin_json_grammar()) @staticmethod def concat(*grammars: "Grammar") -> "Grammar": """Create a grammar that matches the concatenation of the grammars in the list. That is equivalent to using the `+` operator to concatenate the grammars in the list. Parameters ---------- grammars : List[Grammar] The grammars to create the concatenation of. Returns ------- grammar : Grammar The concatenation of the grammars. """ grammar_handles = [grammar._handle for grammar in grammars] return Grammar._create_from_handle(_core.Grammar.concat(grammar_handles)) @staticmethod def union(*grammars: "Grammar") -> "Grammar": """Create a grammar that matches any of the grammars in the list. That is equivalent to using the `|` operator to concatenate the grammars in the list. Parameters ---------- grammars : List[Grammar] The grammars to create the union of. Returns ------- grammar : Grammar The union of the grammars. """ grammar_handles = [grammar._handle for grammar in grammars] return Grammar._create_from_handle(_core.Grammar.union(grammar_handles)) xgrammar-0.1.19/python/xgrammar/kernels/000077500000000000000000000000001500705317600202265ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/kernels/__init__.py000066400000000000000000000004031500705317600223340ustar00rootroot00000000000000"""The kernels for XGrammar. There are 5 implementations: - CPU: used for CPU tensors - CUDA: not used in the current implementation - Triton: used for CUDA GPU tensors - MLX: used for MLX tensors - Torch Compile: used for torch tensors on other devices """ xgrammar-0.1.19/python/xgrammar/kernels/apply_token_bitmask_inplace_cpu.py000066400000000000000000000027501500705317600272050ustar00rootroot00000000000000"""CPU implementation for in-place applying token mask.""" from typing import List, Optional, Union import torch from ..base import _core def apply_token_bitmask_inplace_cpu( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[Union[List[int], torch.Tensor]] = None, ) -> None: """Apply token bitmask in-place on CPU.""" if logits.device.type != "cpu": raise ValueError("logits must be on CPU") if bitmask.device.type != "cpu": raise ValueError("bitmask must be on CPU") if logits.dtype != torch.float32: raise ValueError("logits must be of type float32") if bitmask.dtype != torch.int32: raise ValueError("bitmask must be of type int32") if logits.dim() != 1 and logits.dim() != 2: raise ValueError("logits should be 1D or 2D, but got {}D".format(logits.dim())) if bitmask.dim() != 1 and bitmask.dim() != 2: raise ValueError("bitmask should be 1D or 2D, but got {}D".format(bitmask.dim())) logits_shape = (1, logits.shape[0]) if logits.dim() == 1 else (logits.shape[0], logits.shape[1]) bitmask_shape = ( (1, bitmask.shape[0]) if bitmask.dim() == 1 else (bitmask.shape[0], bitmask.shape[1]) ) vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None else vocab_size _core.kernels.apply_token_bitmask_inplace_cpu( logits.data_ptr(), logits_shape, bitmask.data_ptr(), bitmask_shape, vocab_size, indices ) xgrammar-0.1.19/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.cu000066400000000000000000000234371500705317600273160ustar00rootroot00000000000000/* * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // clang-format off #include #include #include #include #include // clang-format on #ifndef CUDART_INF_FP16 #define CUDART_INF_FP16 __ushort_as_half((unsigned short)0x7C00U) #endif #ifndef CUDART_INF_BF16 #define CUDART_INF_BF16 __ushort_as_bfloat16((unsigned short)0x7F80U) #endif constexpr int32_t BITS_PER_BLOCK = 32; constexpr int32_t THREADS_PER_THREAD_BLOCK = 256; template __device__ T NegativeInfinity() { return -INFINITY; } template <> __device__ __half NegativeInfinity<__half>() { return -CUDART_INF_FP16; } template <> __device__ __nv_bfloat16 NegativeInfinity<__nv_bfloat16>() { return -CUDART_INF_BF16; } template __device__ PackedT PackedNegativeInfinity() { constexpr int kAlignment = sizeof(PackedT) / sizeof(T); T packed[kAlignment]; #pragma unroll for (int i = 0; i < kAlignment; i++) { packed[i] = NegativeInfinity(); } return *reinterpret_cast(packed); } template __global__ void __launch_bounds__(THREADS_PER_THREAD_BLOCK) LogitsBitmaskKernel( T* __restrict__ logits, const int32_t* __restrict__ bitmask, const int32_t* __restrict__ indices, int32_t vocab_size, int32_t logits_stride, int32_t bitmask_stride ) { constexpr int kAlignment = sizeof(PackedT) / sizeof(T); constexpr uint32_t kPackedMask = (1 << kAlignment) - 1; const int batch_idx = (indices == nullptr) ? blockIdx.y : indices[blockIdx.y]; const int block_offset = blockIdx.x * THREADS_PER_THREAD_BLOCK * kBitsPerThread; T* logits_gmem_ptr = logits + batch_idx * logits_stride + block_offset; const int32_t* bitmask_gmem_ptr = bitmask + batch_idx * bitmask_stride + block_offset / BITS_PER_BLOCK; const int bitmask_inner_idx = threadIdx.x % (BITS_PER_BLOCK / kAlignment); T logits_reg[kAlignment]; #pragma unroll for (int offset = threadIdx.x * kAlignment; offset < THREADS_PER_THREAD_BLOCK * kBitsPerThread; offset += THREADS_PER_THREAD_BLOCK * kAlignment) { if (block_offset + offset >= vocab_size) { break; } const uint32_t bitmask_val = (~bitmask_gmem_ptr[offset / BITS_PER_BLOCK] >> (bitmask_inner_idx * kAlignment)) & kPackedMask; if (bitmask_val == 0) { continue; } if (bitmask_val == kPackedMask) { *reinterpret_cast(logits_gmem_ptr + offset) = PackedNegativeInfinity(); continue; } *reinterpret_cast(logits_reg) = *reinterpret_cast(logits_gmem_ptr + offset); #pragma unroll for (int i = 0; i < kAlignment; i++) { if (((bitmask_val >> i) & 1)) { logits_reg[i] = NegativeInfinity(); } } *reinterpret_cast(logits_gmem_ptr + offset) = *reinterpret_cast(logits_reg); } } template ::value>> constexpr auto CeilDiv(T numerator, T denominator) { return (numerator + denominator - 1) / denominator; } template void ApplyTokenBitmaskInplaceDispatchToBitsPerThread( T* __restrict__ logits, const int32_t* __restrict__ bitmask, const int32_t* __restrict__ indices, int32_t vocab_size, int32_t logits_stride, int32_t bitmask_stride, int32_t num_rows ) { constexpr int kAlignment = sizeof(PackedT) / sizeof(T); const int32_t num_blocks_per_row = CeilDiv(2048 / THREADS_PER_THREAD_BLOCK * 128, num_rows); const int32_t num_bits_per_thread = CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * num_blocks_per_row); const dim3 block(THREADS_PER_THREAD_BLOCK); cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); if (num_bits_per_thread <= 4 && kAlignment <= 4) { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 4), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } else if (num_bits_per_thread <= 8 && kAlignment <= 8) { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 8), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } else if (num_bits_per_thread <= 16 && kAlignment <= 16) { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 16), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } else { const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 32), num_rows); LogitsBitmaskKernel<<>>( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride ); } } template void ApplyTokenBitmaskInplaceDispatchToPackedT( T* __restrict__ logits, const int32_t* __restrict__ bitmask, const int32_t* __restrict__ indices, int32_t vocab_size, int32_t logits_stride, int32_t bitmask_stride, int32_t num_rows ) { if (logits_stride % (sizeof(float4) / sizeof(T)) == 0) { ApplyTokenBitmaskInplaceDispatchToBitsPerThread( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows ); } else { ApplyTokenBitmaskInplaceDispatchToBitsPerThread( logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows ); } } void ApplyTokenBitmaskInplace( at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt ) { TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor."); TORCH_CHECK(logits.is_contiguous(), "logits must be contiguous."); TORCH_CHECK(logits.dim() == 1 || logits.dim() == 2, "logits must be a 1D or 2D tensor."); std::pair logits_shape = logits.dim() == 2 ? std::make_pair( static_cast(logits.size(0)), static_cast(logits.size(1)) ) : std::make_pair(1, static_cast(logits.size(0))); TORCH_CHECK(bitmask.is_cuda(), "bitmask must be a CUDA tensor."); TORCH_CHECK(bitmask.is_contiguous(), "bitmask must be contiguous."); TORCH_CHECK(bitmask.dim() == 1 || bitmask.dim() == 2, "bitmask must be a 1D or 2D tensor."); std::pair bitmask_shape = bitmask.dim() == 2 ? std::make_pair( static_cast(bitmask.size(0)), static_cast(bitmask.size(1)) ) : std::make_pair(1, static_cast(bitmask.size(0))); TORCH_CHECK(bitmask.dtype() == torch::kInt32, "bitmask must be of type int32."); TORCH_CHECK( (logits_shape.second + BITS_PER_BLOCK - 1) / BITS_PER_BLOCK >= bitmask_shape.second, "The provided logits's vocab size should be no less than the bitmask's vocab size " "(converted from bitmask size). But got vocab size ", logits_shape.second, " vs bitmask size ", bitmask_shape.second ); int vocab_size = std::min(logits_shape.second, bitmask_shape.second * BITS_PER_BLOCK); int32_t num_rows = logits_shape.first; int32_t* indices_ptr = nullptr; if (indices) { TORCH_CHECK(indices->is_cuda(), "indices must be a CUDA tensor."); TORCH_CHECK(indices->is_contiguous(), "indices must be contiguous."); TORCH_CHECK(indices->dim() == 1, "indices must be a 1D tensor."); TORCH_CHECK(indices->dtype() == torch::kInt32, "indices must be of type int32."); num_rows = indices->size(0); indices_ptr = indices->data_ptr(); } else { TORCH_CHECK( logits_shape.first == bitmask_shape.first, "logits and bitmask must have the same batch size." ); } switch (logits.scalar_type()) { case torch::kFloat32: { ApplyTokenBitmaskInplaceDispatchToPackedT( logits.data_ptr(), bitmask.data_ptr(), indices_ptr, vocab_size, logits_shape.second, bitmask_shape.second, num_rows ); break; } case torch::kFloat16: { ApplyTokenBitmaskInplaceDispatchToPackedT( reinterpret_cast<__half*>(logits.data_ptr()), bitmask.data_ptr(), indices_ptr, vocab_size, logits_shape.second, bitmask_shape.second, num_rows ); break; } case torch::kBFloat16: { ApplyTokenBitmaskInplaceDispatchToPackedT( reinterpret_cast<__nv_bfloat16*>(logits.data_ptr()), bitmask.data_ptr(), indices_ptr, vocab_size, logits_shape.second, bitmask_shape.second, num_rows ); break; } default: TORCH_CHECK(false, "logits dtype must be float, half or bfloat16."); break; } } TORCH_LIBRARY_FRAGMENT(TORCH_EXTENSION_NAME, m) { m.def( "apply_token_bitmask_inplace_cuda(Tensor logits, Tensor bitmask, Tensor? indices=None) -> ()" ); } TORCH_LIBRARY_IMPL(TORCH_EXTENSION_NAME, CUDA, m) { m.impl("apply_token_bitmask_inplace_cuda", &ApplyTokenBitmaskInplace); } xgrammar-0.1.19/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.py000066400000000000000000000077211500705317600273350ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from contextlib import suppress from typing import List, Optional, Union import torch import torch.utils.cpp_extension def _check_cuda_toolchain() -> None: """check if nvcc is available and if pytorch will likely find it""" import glob import os import shutil from pathlib import Path # First check if CUDA is available in PyTorch if not torch.cuda.is_available(): raise ImportError("CUDA is not available in PyTorch") # This is similar logic to what pytorch does to find the nvcc compiler nvcc_path = shutil.which("nvcc") if nvcc_path is None: cuda_home = os.environ.get("CUDA_HOME", os.environ.get("CUDA_PATH", None)) if cuda_home is None: if os.name == "nt": # This is a very hardcoded asumption about install directories but pytorch does this. cuda_homes = glob.glob("C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*") if len(cuda_homes) == 0: cuda_home = "" else: cuda_home = cuda_homes[0] else: cuda_home = "/usr/local/cuda" if cuda_home is None: raise ImportError("No CUDA toolchain found") nvcc_path = str(Path(cuda_home) / "bin" / "nvcc") if not os.path.exists(nvcc_path): raise ImportError(f"nvcc compiler not found at {nvcc_path}") def _remove_torch_nvcc_flags() -> None: REMOVE_NVCC_FLAGS = [ "-D__CUDA_NO_HALF_OPERATORS__", "-D__CUDA_NO_HALF_CONVERSIONS__", "-D__CUDA_NO_BFLOAT16_CONVERSIONS__", "-D__CUDA_NO_HALF2_OPERATORS__", ] for flag in REMOVE_NVCC_FLAGS: with suppress(ValueError): torch.utils.cpp_extension.COMMON_NVCC_FLAGS.remove(flag) def _load_torch_ops() -> None: from pathlib import Path torch_op_file_path = Path(__file__).with_suffix(".cu") with open(torch_op_file_path) as f: source = f.read() cflags = ["-O3", "-Wno-switch-bool"] cuda_cflags = ["-O3", "-std=c++17", "--threads", "4", "-use_fast_math"] # Use the safer cpp_extension.load_inline instead of cpp_extension.load torch.utils.cpp_extension.load_inline( name="xgrammar", cpp_sources=[], # No C++ sources cuda_sources=[source], extra_cflags=cflags, extra_cuda_cflags=cuda_cflags, with_cuda=True, is_python_module=False, ) _check_cuda_toolchain() _remove_torch_nvcc_flags() _load_torch_ops() _is_register_fake_available = hasattr(torch, "library") and hasattr(torch.library, "register_fake") if _is_register_fake_available: # To support torch.compile with fullgraph=True, a fake kernel is needed. @torch.library.register_fake("xgrammar::apply_token_bitmask_inplace_cuda") def _( logits: torch.Tensor, bitmask: torch.Tensor, indices: Optional[torch.Tensor] = None ) -> None: pass def apply_token_bitmask_inplace_cuda( logits: torch.Tensor, bitmask: torch.Tensor, indices: Optional[Union[List[int], torch.Tensor]] = None, ) -> None: if isinstance(indices, list): indices = torch.tensor(indices, dtype=torch.int32, device=logits.device) if indices is not None: indices = indices.to(logits.device) torch.ops.xgrammar.apply_token_bitmask_inplace_cuda(logits, bitmask, indices) xgrammar-0.1.19/python/xgrammar/kernels/apply_token_bitmask_inplace_torch_compile.py000066400000000000000000000041321500705317600312410ustar00rootroot00000000000000from typing import List, Optional import torch @torch.compile(dynamic=True) def apply_token_bitmask_inplace_kernel_no_indices_torch_compile( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: int ) -> None: # logits: (batch_size, vocab_size) # bitmask: (batch_size, bitmask_size) # mask_expanded: (batch_size, 32 * bitmask_size) mask_expanded = torch.repeat_interleave(bitmask, 32, dim=-1) # bit_indices: (32 * bitmask_size,) bit_indices = torch.arange(32, device=logits.device, dtype=torch.int32).repeat( bitmask.shape[-1] ) # bit_masks: (batch_size, 32 * bitmask_size) bit_masks = (mask_expanded >> bit_indices) & 1 bit_masks = bit_masks[..., :vocab_size] logits[..., :vocab_size] = logits[..., :vocab_size].masked_fill_(bit_masks == 0, float("-inf")) @torch.compile(dynamic=True) def apply_token_bitmask_inplace_kernel_indices_torch_compile( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: int, indices: List[int] ) -> None: # logits: (batch_size, vocab_size) # bitmask: (batch_size, bitmask_size) # mask_expanded: (batch_size, 32 * bitmask_size) mask_expanded = torch.repeat_interleave(bitmask[indices], 32, dim=-1) # bit_indices: (32 * bitmask_size,) bit_indices = torch.arange(32, device=logits.device, dtype=torch.int32).repeat( bitmask.shape[-1] ) bit_masks = (mask_expanded >> bit_indices) & 1 bit_masks = bit_masks[..., :vocab_size] logits[indices, :vocab_size] = logits[indices, :vocab_size].masked_fill_( bit_masks == 0, float("-inf") ) def apply_token_bitmask_inplace_torch_compile( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[List[int]] = None, ) -> None: vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None else vocab_size if indices is None: apply_token_bitmask_inplace_kernel_no_indices_torch_compile(logits, bitmask, vocab_size) else: apply_token_bitmask_inplace_kernel_indices_torch_compile( logits, bitmask, vocab_size, indices ) xgrammar-0.1.19/python/xgrammar/kernels/apply_token_bitmask_inplace_triton.py000066400000000000000000000073421500705317600277370ustar00rootroot00000000000000from typing import List, Optional import torch try: import triton import triton.language as tl except ImportError as err: raise ImportError("Triton is not installed") from err @triton.jit def apply_token_bitmask_inplace_kernel( logits_ptr, bitmask_ptr, indices_ptr, num_rows, vocab_size, logits_strides, bitmask_strides, NUM_SMS: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Apply a bitmask to logits in-place using Triton. The bitmask is a 01 bitwise compressed tensor, where 0 means the token is masked and 1 means the token is not masked. After applying the bitmask, the masked logits will be set to -inf. Parameters ---------- logits_ptr : tl.tensor Pointer to the logits tensor to apply the bitmask to. bitmask_ptr : tl.tensor Pointer to the bitmask tensor to apply. indices_ptr : Optional[tl.tensor] Optional pointer to indices tensor specifying which rows to apply the mask to. num_rows : int Number of rows to process. If indices_ptr is provided, this is the number of unique indices. vocab_size : int Size of the vocabulary dimension. If the logits does not have a vocab padding, this is the same as the logits's second dimension. Otherwise, this is the actual size of the vocabulary. logits_strides : int Stride between rows in the logits tensor. bitmask_strides : int Stride between rows in the bitmask tensor. NUM_SMS : int Number of streaming multiprocessors to use. BLOCK_SIZE : int Size of processing blocks. """ pid = tl.program_id(0) num_blocks = tl.cdiv(vocab_size, BLOCK_SIZE) for work_id in tl.range(pid, num_rows * num_blocks, NUM_SMS): row_id = work_id // num_blocks block_offset = (work_id % num_blocks) * BLOCK_SIZE batch_id = row_id if indices_ptr is None else tl.load(indices_ptr + row_id) offsets = block_offset + tl.arange(0, BLOCK_SIZE) bitmask_offsets = block_offset // 32 + tl.arange(0, BLOCK_SIZE // 32) vocab_mask = offsets < vocab_size packed_bitmask_mask = bitmask_offsets < bitmask_strides packed_bitmask = tl.load( bitmask_ptr + batch_id * bitmask_strides + bitmask_offsets, packed_bitmask_mask ) bitmask = ((packed_bitmask[:, None] >> (tl.arange(0, 32)[None, :])) & 1) == 0 bitmask = bitmask.reshape(BLOCK_SIZE) tl.store( logits_ptr + batch_id * logits_strides + offsets, -float("inf"), vocab_mask & bitmask ) def apply_token_bitmask_inplace_triton( logits: torch.Tensor, bitmask: torch.Tensor, vocab_size: Optional[int] = None, indices: Optional[List[int]] = None, ): NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count BLOCK_SIZE = 4096 assert bitmask.dtype == torch.int32, "bitmask must be of type int32" detected_vocab_size = min(logits.shape[-1], bitmask.shape[-1] * 32) if vocab_size is None: vocab_size = detected_vocab_size else: assert ( vocab_size <= detected_vocab_size ), f"vocab_size {vocab_size} is larger than the detected vocab_size {detected_vocab_size}" num_rows = len(indices) if indices is not None else logits.shape[0] if logits.ndim == 2 else 1 if indices is not None: indices = torch.tensor(indices, dtype=torch.int32, device=logits.device) grid = (NUM_SMS,) apply_token_bitmask_inplace_kernel[grid]( logits, bitmask, indices, num_rows, vocab_size, logits.shape[-1], bitmask.shape[-1], NUM_SMS, BLOCK_SIZE, num_warps=BLOCK_SIZE // 32 // (16 // logits.element_size()), num_stages=3, ) xgrammar-0.1.19/python/xgrammar/kernels/apply_token_bitmask_mlx.py000066400000000000000000000015631500705317600255240ustar00rootroot00000000000000"""MLX kernel for applying token bitmasks.""" import itertools import mlx.core as mx @mx.compile def apply_token_bitmask_mlx(bitmask: mx.array, logits: mx.array, vocab_size: int): """Apply a token bitmask to logits using MLX for Metal GPUs. Args: bitmask: A tensor of shape (batch_size, (vocab_size + 31) // 32) containing the bitmask. Each bit in the bitmask determines whether the corresponding token is allowed (1) or not (0). logits: A tensor of shape (batch_size, vocab_size) containing the logits. Returns: The logits with -inf for tokens that are not allowed. """ bitmap = mx.array( [l[::-1] for l in itertools.product(*[[float("-inf"), 0]] * 8)], dtype=logits.dtype ) bitmask = bitmask.view(mx.uint8) return logits[..., :vocab_size] + bitmap[bitmask].flatten(-2)[..., :vocab_size] xgrammar-0.1.19/python/xgrammar/matcher.py000066400000000000000000000313311500705317600205610ustar00rootroot00000000000000"""Match the output of the LLM to the specified grammar, then generate the mask for the next token. """ import math from typing import List, Optional, Tuple, Union import torch from .base import XGRObject, _core from .compiler import CompiledGrammar """The dtype of the bitmask: int32.""" bitmask_dtype = torch.int32 def get_bitmask_shape(batch_size: int, vocab_size: int) -> Tuple[int, int]: """Return the shape of the bitmask: (batch_size, ceil(vocab_size / 32)).""" return (batch_size, math.ceil(vocab_size / 32)) _FULL_MASK = torch.tensor(-1, dtype=bitmask_dtype) def allocate_token_bitmask(batch_size: int, vocab_size: int) -> torch.Tensor: """Allocate the bitmask for the next token prediction. The bitmask is an int32 tensor on CPU with shape (batch_size, ceil(vocab_size / 32)). Users who have their own needs to manage CUDA memory can construct the tensor with get_bitmask_shape and bitmask_dtype themselves. The reason why we use int32 instead of uint32 is that old versions of PyTorch do not support uint32. Parameters ---------- batch_size : int The batch size of the bitmask. vocab_size : int The size of the vocabulary. Returns ------- bitmask : torch.Tensor The shape of the bitmask. """ # In CUDA, use pinned memory to speed up data transfer from CPU to GPU return torch.full(get_bitmask_shape(batch_size, vocab_size), _FULL_MASK, dtype=bitmask_dtype) def reset_token_bitmask(bitmask: torch.Tensor) -> None: """Reset the bitmask to the full mask.""" bitmask.fill_(_FULL_MASK) def apply_token_bitmask_inplace( logits: torch.Tensor, bitmask: torch.Tensor, *, vocab_size: Optional[int] = None, indices: Optional[List[int]] = None, ) -> None: """Apply the bitmask to the logits in-place. The bitmask is a 01 bitwise compressed tensor, where 0 means the token is masked and 1 means the token is not masked. It can be generated by allocate_token_bitmask and filled by fill_next_token_bitmask. After applying the bitmask, the masked logits will be set to -inf. The shape of logits and bitmask should be (batch_size, vocab_size) and (batch_size, bitmask_size) respectively. bitmask_size = ceil(vocab_size / 32). The operation is: .. code:: python for i in range(batch_size): for j in range(vocab_size): if get_bitmask_value(bitmask, i, j) == 0: logits[i, j] = -inf get_bitmask_value(bitmask, i, j) gets the j-th bit of the i-th row of the bitmask. ## Padding This method allows additional padding on the vocabulary dimension of logits or bitmask. If padding exists, provide the real vocab size to the vocab_size parameter, and the operation will be applied to logits[..., :vocab_size] and bitmask[..., :ceil(vocab_size / 32)]. If vocab_size is not provided, the vocab size will be detected as min(logits.shape[-1], bitmask.shape[-1] * 32). ## Indices Indices can be used to specify which logits in the batch to apply the bitmask to. It is especially useful when there are structured requests and unstructured requests mixed in the same batch by skipping masking the logits in the unstructured requests. When specified, the operation will be .. code:: python for batch_id in indices: for j in range(vocab_size): if get_bitmask_value(bitmask, batch_id, j) == 0: logits[batch_id, j] = -inf When indices is specified, the batch sizes of logits and bitmask do not need to be the same. As long as the indices are valid, the operation will be performed. ## Device The logits and bitmask should be on the same device. If both them are on GPU, we launch a GPU kernel to apply bitmask. If both them are on CPU, we use a CPU implementation. The GPU kernel is optimized and should be preferred. In practice, the bitmask is allocated on CPU, and the logits is usually on GPU, so users should manually copy the bitmask to GPU before calling this function. Parameters ---------- logits : torch.Tensor The tensor to apply the bitmask to. bitmask : torch.Tensor The bitmask to apply. vocab_size : Optional[int], default: None The size of the vocabulary. If not provided, the vocab size will be detected as min(logits.shape[-1], bitmask.shape[-1] * 32). indices : Optional[List[int]], default: None A list of indices to specify which logits in the batch to apply the bitmask to. Should be unique. If None, apply the bitmask to all logits in the batch. """ if bitmask.device != logits.device: raise ValueError( "logits and bitmask should be on the same device. " + f"But got logits.device: {logits.device}, bitmask.device: {bitmask.device}" ) # dispatch to different implementations based on the device if logits.device.type == "cpu": from .kernels.apply_token_bitmask_inplace_cpu import apply_token_bitmask_inplace_cpu apply_token_bitmask_inplace_cpu(logits, bitmask, vocab_size, indices) elif logits.device.type == "cuda": from .kernels.apply_token_bitmask_inplace_triton import apply_token_bitmask_inplace_triton apply_token_bitmask_inplace_triton(logits, bitmask, vocab_size, indices) else: from .kernels.apply_token_bitmask_inplace_torch_compile import ( apply_token_bitmask_inplace_torch_compile, ) apply_token_bitmask_inplace_torch_compile(logits, bitmask, vocab_size, indices) class GrammarMatcher(XGRObject): """Match the output of the LLM to the specified grammar, then generate the mask for the next token. This is the core class in the grammar-guided generation. This class maintains a stateful matcher that can accept tokens and strings, then match them to the specified grammar. The matcher can provide a bitmask for the next token prediction, so that the output of the LLM follows the specified grammar. Its state can be reset and rolled back by tokens. It also provides utilities for jump-forward decoding. After matching the whole grammar, the matcher will accept a stop token. The token mask at this time will only allow stop tokens. After accepting the stop token, the matcher will terminate, then it cannot accept any new token or generate a new token mask, meaning the generation is finished. Under the hood, it utilizes a pushdown automaton with backtracking to match the grammar, with optimizations specific to LLM token mask generation. Parameters ---------- compiled_grammar : CompiledGrammar The initialization context for the grammar matcher. override_stop_tokens : Optional[Union[int, List[int]]], default: None If not None, the stop tokens to override the ones in the grammar. terminate_without_stop_token : bool, default: False Whether to terminate the matcher without accepting a stop token. max_rollback_tokens : int, default: 0 The maximum number of rollback tokens allowed. The rollback operation is useful for jump-forward decoding and speculative decoding. """ def __init__( self, compiled_grammar: CompiledGrammar, *, override_stop_tokens: Optional[Union[int, List[int]]] = None, terminate_without_stop_token: bool = False, max_rollback_tokens: int = 0, ) -> None: if not isinstance(compiled_grammar, CompiledGrammar): raise ValueError("The grammar should be compiled before passing it to GrammarMatcher.") if isinstance(override_stop_tokens, int): override_stop_tokens = [override_stop_tokens] self._init_handle( _core.GrammarMatcher( compiled_grammar._handle, override_stop_tokens, terminate_without_stop_token, max_rollback_tokens, ) ) def accept_token(self, token_id: int, *, debug_print: bool = False) -> bool: """Accept one token and update the state of the matcher. Parameters ---------- token_id : int The id of the token to accept. debug_print : bool, default: False Whether to print information about the internal state of the matcher. Helpful for debugging. Returns ------- accepted : bool Whether the token is accepted. """ return self._handle.accept_token(token_id, debug_print) def fill_next_token_bitmask( self, bitmask: torch.Tensor, index: int = 0, *, debug_print: bool = False ) -> bool: """Fill the bitmask for the next token prediction. The input bitmask can be generated by allocate_token_bitmask, and must be on CPU. bitmask[index] will be filled with the next token bitmask. This method does not change the matcher state. Parameters ---------- bitmask : torch.Tensor The bitmask for the next token prediction. index : int, default: 0 The batch id of the bitmask. debug_print : bool, default: False Whether to print information about generated bitmask. Helpful for debugging. Returns ------- need_apply : bool Whether the bitmask need to be applied (not all-true). An optimization: if False, this means the bitmask is already all-true, so no need to apply it. """ if bitmask.device.type != "cpu": raise ValueError("bitmask should be on CPU.") if bitmask.dtype != bitmask_dtype: raise ValueError(f"bitmask should be of type {bitmask_dtype}.") return self._handle.fill_next_token_bitmask( bitmask.data_ptr(), list(bitmask.shape), index, debug_print ) def find_jump_forward_string(self) -> str: """Find the jump-forward string for jump-forward decoding. This is the longest string that certainly conforms with the current grammar from the current matcher state. This string can become the output of the LLM without requiring LLM decoding. This method does not change the matcher state. Returns ------- jump_forward_string : str The jump-forward string. """ return self._handle.find_jump_forward_string() def rollback(self, num_tokens: int = 1) -> None: """Rollback the matcher to a previous state by several tokens. Parameters ---------- num_tokens : int, default: 1 The number of tokens to rollback. It cannot exceed the current number of steps, nor can it exceed the specified maximum number of rollback tokens. """ self._handle.rollback(num_tokens) def is_terminated(self) -> bool: """Check if the matcher has terminated. If terminate_without_stop_token is False, the matcher will terminate if it has accepted the stop token. Otherwise, the matcher will terminate after matching the whole grammar. Returns ------- terminated : bool Whether the matcher has terminated. """ return self._handle.is_terminated() def reset(self) -> None: """Reset the matcher to the initial state.""" return self._handle.reset() @property def max_rollback_tokens(self) -> int: """Get the maximum number of rollback tokens allowed. Returns ------- max_rollback_tokens : int The maximum number of rollback tokens. """ return self._handle.max_rollback_tokens @property def stop_token_ids(self) -> List[int]: """The ids of the stop tokens used in the matcher. If specified, the provided stop tokens will be used. Otherwise, the stop tokens will be detected from the vocabulary. Returns ------- stop_token_ids : List[int] The ids of the stop tokens. """ return self._handle.stop_token_ids def _debug_accept_string( self, input_str: Union[str, bytes], *, debug_print: bool = False ) -> bool: """Accept a string and update the state of the matcher. The whole string is considered as one step in rollback. It is only used to complement the functionality of accept_token. Parameters ---------- input_str : Union[str, bytes] The string to be accepted. debug_print : bool, default: False Whether to print information about the internal state of the matcher. Helpful for debugging. Returns ------- accepted : bool Whether the string is accepted. """ return self._handle._debug_accept_string(input_str, debug_print) xgrammar-0.1.19/python/xgrammar/support/000077500000000000000000000000001500705317600202775ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/support/__init__.py000066400000000000000000000000001500705317600223760ustar00rootroot00000000000000xgrammar-0.1.19/python/xgrammar/support/logging.py000066400000000000000000000011221500705317600222730ustar00rootroot00000000000000""" Logging support for XGrammar. It derives from Python's logging module, and in the future, it can be easily replaced by other logging modules such as structlog. """ import logging def enable_logging(): """Enable XGrammar's default logging formpat""" logging.basicConfig( level=logging.INFO, style="{", datefmt="%Y-%m-%d %H:%M:%S", format="[{asctime}] {levelname} {filename}:{lineno}: {message}", ) def getLogger(name: str): # pylint: disable=invalid-name """Get a logger according to the given name""" return logging.getLogger(name) xgrammar-0.1.19/python/xgrammar/testing.py000066400000000000000000000251631500705317600206210ustar00rootroot00000000000000"""Testing utilities.""" import time from typing import Any, Dict, List, Optional, Tuple, Type, Union import torch from pydantic import BaseModel from .base import _core from .compiler import CompiledGrammar, GrammarCompiler from .grammar import Grammar, _convert_schema_to_str from .matcher import GrammarMatcher, bitmask_dtype from .tokenizer_info import TokenizerInfo def _json_schema_to_ebnf( schema: Union[str, Type[BaseModel], Dict[str, Any]], *, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, ) -> str: """Convert JSON schema string to BNF grammar string. For test purposes. Parameters ---------- schema : Union[str, Type[BaseModel], Dict[str, Any]] The schema string or Pydantic model or JSON schema dict. indent : Optional[int], default: None The number of spaces for indentation. If None, the output will be in one line. separators : Optional[Tuple[str, str]], default: None Two separators used in the schema: comma and colon. Examples: (",", ":"), (", ", ": "). If None, the default separators will be used: (",", ": ") when the indent is not None, and (", ", ": ") otherwise. strict_mode : bool, default: True Whether to use strict mode. In strict mode, the generated grammar will not allow properties and items that is not specified in the schema. This is equivalent to setting unevaluatedProperties and unevaluatedItems to false. This helps LLM to generate accurate output in the grammar-guided generation with JSON schema. Returns ------- bnf_string : str The BNF grammar string. """ schema_str = _convert_schema_to_str(schema) return _core.testing._json_schema_to_ebnf( schema_str, any_whitespace, indent, separators, strict_mode ) def _regex_to_ebnf(regex: str, with_rule_name: bool = True) -> str: r"""Convert a regex string to BNF grammar string. For test purposes. The regex grammar follows the syntax in JavaScript (ECMA 262). Check https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions for a tutorial. Currently the following features are not supported: 1. Backreference (\1) 2. non-capturing group, naming capture groups and assertions ((?...)) 3. Unicode character class escape (\p{...}) 4. Word boundary (\b) 5. Unicode property escapes (\p{...}) 6. Quantifier with range {x,y}. Now user can just repeat the element as a workaround. This method is primarily intended for testing and debugging purposes. Parameters ---------- regex : str The regex string to be converted. Returns ------- bnf_string : str The BNF grammar string converted from the input regex. """ return _core.testing._regex_to_ebnf(regex, with_rule_name) def _ebnf_to_grammar_no_normalization(ebnf_string: str, root_rule_name: str = "root") -> Grammar: """Convert a BNF grammar string to a Grammar object without normalization. For test purposes. The result grammar cannot be compiled / used in GrammarMatcher. Parameters ---------- ebnf_string : str The BNF grammar string to be converted. Returns ------- grammar : Grammar The unnormalized Grammar object converted from the input BNF grammar string. """ return Grammar._create_from_handle( _core.testing._ebnf_to_grammar_no_normalization(ebnf_string, root_rule_name) ) def _is_grammar_accept_string( grammar: Union[Grammar, str], input_str: str, *, debug_print: bool = False, print_time: bool = False, ) -> bool: """Check if a grammar accepts a string. For test purposes. Parameters ---------- grammar : Union[Grammar, str] The grammar to check. Can be either a Grammar object or a BNF grammar string. input_str : str The input string to check. debug_print : bool, default: False Whether to print debug information during matching. print_time : bool, default: False Whether to print timing information. Returns ------- bool True if the grammar accepts the string, False otherwise. """ if isinstance(grammar, str): grammar = Grammar.from_ebnf(grammar) grammar_compiler = GrammarCompiler(TokenizerInfo([]), cache_enabled=False) compiled_grammar = grammar_compiler.compile_grammar(grammar) matcher = GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) if print_time: start = time.monotonic_ns() accepted = matcher._debug_accept_string(input_str, debug_print=debug_print) if print_time: end = time.monotonic_ns() print(f"Accepting {input_str}, result: {accepted}, time: {(end - start) / 1e3} us") if not accepted: return False return matcher.is_terminated() def _get_masked_tokens_from_bitmask( bitmask: torch.Tensor, vocab_size: int, index: int = 0 ) -> List[int]: """Get the ids of the rejected tokens from the bitmask. Mainly for debug purposes. Parameters ---------- bitmask : torch.Tensor The rejected token bitmask. Should be generated by allocate_token_bitmask and filled by fill_next_token_bitmask. Should be on CPU. index : int, default: 0 The batch index of the bitmask. For batch inference, bitmask[index] will be used. Otherwise is ignored. Returns ------- rejected_token_ids : List[int] A list of rejected token ids. """ if bitmask.device.type != "cpu": raise ValueError("bitmask should be on CPU.") if bitmask.dtype != bitmask_dtype: raise ValueError(f"bitmask should be of type {bitmask_dtype}.") return _core.testing._get_masked_tokens_from_bitmask( bitmask.data_ptr(), list(bitmask.shape), vocab_size, index ) def _is_single_token_bitmask( bitmask: torch.Tensor, vocab_size: int, index: int = 0 ) -> Tuple[bool, int]: """Check if the bitmask is a single token bitmask. Parameters ---------- bitmask : torch.Tensor The bitmask to check. Should be on CPU. vocab_size : int The size of the vocabulary. index : int, default: 0 The index of the bitmask. Returns ------- is_single_token : bool True if the bitmask is a single token bitmask, False otherwise. token_id : int The id of the token if the bitmask is a single token bitmask, -1 otherwise. """ return _core.testing._is_single_token_bitmask( bitmask.data_ptr(), list(bitmask.shape), vocab_size, index ) def _bool_mask_to_bitmask(bool_mask: torch.Tensor) -> torch.Tensor: """Get the bitmask from bool mask. If the bool mask does not align with the 32-bit block size, it will add extra 1 paddings. Parameters ---------- bool_mask : torch.Tensor The rejected token bool mask. For each element value, True means the token is allowed, while False means the token is rejected. Returns ------- bitmask : torch.Tensor The rejected token bitmask. """ bool_mask_int32 = bool_mask.to(torch.int32) # Pad to multiple of 32 pad_size = (32 - bool_mask.shape[1] % 32) % 32 if pad_size > 0: bool_mask_int32 = torch.nn.functional.pad(bool_mask_int32, (0, pad_size), value=1) bool_mask_view = bool_mask_int32.view(bool_mask.shape[0], -1, 32) # To avoid error for overflow, we construct int64 weights and convert to int32 weights = torch.tensor( [1 << i for i in range(32)], device=bool_mask.device, dtype=torch.int64 ).to(torch.int32) bitmask = (bool_mask_view * weights).sum(dim=2) return bitmask.to(torch.int32) def _get_matcher_from_grammar_and_tokenizer_info( grammar: Union[Grammar, str], tokenizer_info: Optional[TokenizerInfo] = None, **kwargs ) -> GrammarMatcher: """Create a GrammarMatcher from a grammar and tokenizer info. Parameters ---------- grammar : Union[Grammar, str] The grammar to create the matcher from. Can be either a Grammar object or a string containing EBNF grammar. tokenizer_info : Optional[TokenizerInfo], default: None Information about the tokenizer to use with this grammar. If None, an empty TokenizerInfo will be created. **kwargs Additional keyword arguments to pass to the GrammarMatcher constructor. Returns ------- matcher : GrammarMatcher The created grammar matcher. """ if tokenizer_info is None: tokenizer_info = TokenizerInfo([]) grammar_compiler = GrammarCompiler(tokenizer_info, cache_enabled=False) compiled_grammar = grammar_compiler.compile_grammar(grammar) return GrammarMatcher(compiled_grammar, **kwargs) def _get_allow_empty_rule_ids(compiled_grammar: CompiledGrammar) -> List[int]: return _core.testing._get_allow_empty_rule_ids(compiled_grammar._handle) def _generate_range_regex(start: Optional[int] = None, end: Optional[int] = None) -> str: return _core.testing._generate_range_regex(start, end) def _generate_float_regex(start: Optional[float] = None, end: Optional[float] = None) -> str: return _core.testing._generate_float_regex(start, end) class GrammarFunctor: """A utility class for transforming grammars. These methods are called during grammar parsing. For test purposes.""" @staticmethod def structure_normalizer(grammar: Grammar) -> Grammar: """Normalize the structure of the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.structure_normalizer(grammar._handle) ) @staticmethod def rule_inliner(grammar: Grammar) -> Grammar: """Inline some rule references in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.rule_inliner(grammar._handle) ) @staticmethod def byte_string_fuser(grammar: Grammar) -> Grammar: """Fuse the byte string elements in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.byte_string_fuser(grammar._handle) ) @staticmethod def dead_code_eliminator(grammar: Grammar) -> Grammar: """Eliminate the not referenced rules in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.dead_code_eliminator(grammar._handle) ) @staticmethod def lookahead_assertion_analyzer(grammar: Grammar) -> Grammar: """Analyze and add lookahead assertions in the grammar.""" return Grammar._create_from_handle( _core.testing.grammar_functor.lookahead_assertion_analyzer(grammar._handle) ) xgrammar-0.1.19/python/xgrammar/tokenizer_info.py000066400000000000000000000365661500705317600222020ustar00rootroot00000000000000"""This module provides the tokenizer info class to handle the tokenizer information.""" import json from enum import Enum from typing import Any, Dict, List, Optional, Union import sentencepiece import tiktoken from transformers import PreTrainedTokenizerBase, PreTrainedTokenizerFast from .base import XGRObject, _core from .support import logging logging.enable_logging() logger = logging.getLogger(__name__) class VocabType(Enum): """The type of the vocabulary. Used in TokenizerInfo. XGrammar supports three types of vocabularies: RAW The vocabulary is in the raw format. The tokens in the vocabulary are kept in their original form without any processing. This kind of tokenizer includes the tiktoken tokenizer, e.g. microsoft/Phi-3-small-8k-instruct, Qwen/Qwen-7B-Chat, etc. BYTE_FALLBACK The vocabulary used in the byte fallback BPE tokenizer. The tokens are encoded through the byte-fallback conversion. E.g. "\u001b" -> "<0x1B>", " apple" -> "▁apple". This kind of tokenizer includes meta-llama/Llama-2-7b-chat, microsoft/Phi-3.5-mini-instruct, etc. BYTE_LEVEL The vocabulary used in the byte level BPE tokenizer. The tokens are encoded through the byte-to-unicode conversion, as in https://github.com/huggingface/transformers/blob/87be06ca77166e6a6215eee5a990ab9f07238a18/src/transformers/models/gpt2/tokenization_gpt2.py#L38-L59 This kind of tokenizer includes meta-llama/Meta-Llama-3-8B-Instruct, meta-llama/Meta-Llama-3.1-8B-Instruct, etc. """ RAW = 0 BYTE_FALLBACK = 1 BYTE_LEVEL = 2 class TokenizerInfo(XGRObject): """The tokenizer info contains the vocabulary, the type of the vocabulary, and necessary information for the grammar-guided generation. Note that although some tokenizers will encode the tokens in a special format, e.g. "<0x1B>" for "\u001b" in the ByteFallback tokenizer, and "Ġ" for " " in the Byte-Level BPE tokenizer, TokenizerInfo always decodes the vocabulary to the original format (e.g. "\u001b" and " "). Also note that some models (e.g. Phi-3 and Deepseek-V2) may pad the vocabulary to a multiple of 32. In this case, the model's vocab_size is larger than the tokenizer's vocabulary size. Please pass the model's vocab_size to the vocab_size parameter in the constructor, because this information is used to determine the size of the token mask. Parameters ---------- encoded_vocab : Union[List[bytes], List[str]] The encoded vocabulary of the tokenizer. vocab_type : VocabType, default: VocabType.RAW The type of the vocabulary. See also VocabType. vocab_size : Optional[int], default: None The size of the vocabulary. If not provided, the vocabulary size will be len(encoded_vocab). stop_token_ids : Optional[List[int]], default: None The stop token ids. If not provided, the stop token ids will be auto detected (but may not be correct). add_prefix_space : bool, default: False Whether the tokenizer will prepend a space before the text in the tokenization process. """ def __init__( self, encoded_vocab: Union[List[bytes], List[str]], vocab_type: VocabType = VocabType.RAW, *, vocab_size: Optional[int] = None, stop_token_ids: Optional[Union[List[int], int]] = None, add_prefix_space: bool = False, ) -> None: if isinstance(stop_token_ids, int): stop_token_ids = [stop_token_ids] self._init_handle( _core.TokenizerInfo( encoded_vocab, vocab_type.value, vocab_size, stop_token_ids, add_prefix_space ) ) @staticmethod def _is_tiktoken_tokenizer(tokenizer: PreTrainedTokenizerBase) -> bool: # helper to check if tokenizer is a tiktoken tokenizer has_tiktoken_encoding = hasattr(tokenizer, "tokenizer") and isinstance( tokenizer.tokenizer, tiktoken.Encoding ) filename_pattern = ( hasattr(tokenizer, "vocab_files_names") and "vocab_file" in tokenizer.vocab_files_names and "tiktoken" in tokenizer.vocab_files_names["vocab_file"] ) return has_tiktoken_encoding or filename_pattern @staticmethod def _is_sentencepiece_tokenizer(tokenizer: PreTrainedTokenizerBase) -> bool: # helper to check if tokenizer is a sentence piece tokenizer has_sp_model_attr = hasattr(tokenizer, "sp_model") and isinstance( tokenizer.sp_model, sentencepiece.SentencePieceProcessor ) has_nested_sp_model_attr = ( hasattr(tokenizer, "tokenizer") and hasattr(tokenizer.tokenizer, "sp_model") and isinstance(tokenizer.tokenizer.sp_model, sentencepiece.SentencePieceProcessor) ) return has_sp_model_attr or has_nested_sp_model_attr @staticmethod def from_huggingface( tokenizer: PreTrainedTokenizerBase, *, vocab_size: Optional[int] = None, stop_token_ids: Optional[Union[List[int], int]] = None, ) -> "TokenizerInfo": """Construct the tokenizer info from the huggingface tokenizer. This constructor supports various tokenizer backends, including the huggingface fast tokenizer and tiktoken tokenizer. Necessary information is automatically detected from the tokenizer. The vocab_size parameter is introduced to handle the misalignment between the model's vocab_size and the tokenizer's vocabulary size. User should pass the model's vocab_size (could be defined in the model config) here. See docs of vocab_size for more details. The stop token ids is by default the eos_token_id of the tokenizer. If there are other stop tokens, you can specify them manually. Parameters ---------- tokenizer : PreTrainedTokenizerBase The huggingface tokenizer. vocab_size : Optional[int], default: None The vocabulary size **defined by the model** (**not the tokenizer**). This equals to the vocab dimention of the model's lm_head. This is the size of the token mask. It can be: 1. the same as the tokenizer's vocabulary size. This is the most common case. 2. larger than the tokenizer's vocabulary size. This happens when the model has padding to lm_head, possibly due to aligning lm_head to the power of 2. E.g. Phi-3 and Deepseek-V2. 3. smaller than the tokenizer's vocabulary size. This happens when the tokenizer has some added tokens that will not supported by the model. E.g. Llama-3.2 Vision and Molmo-72B-0924 has padded <|image|> tokens, but they will not be considered in lm_head or generated by the model. model_vocab_size need to be provided for case 2 and 3. If not provided, it will be set to the tokenizer's vocabulary size. stop_token_ids : Optional[List[int]], default: None The stop token ids. If not provided, the eos_token_id of the tokenizer will be used. Returns ------- tokenizer_info : TokenizerInfo The tokenizer info. """ if isinstance(stop_token_ids, int): stop_token_ids = [stop_token_ids] if isinstance(stop_token_ids, list) and len(stop_token_ids) == 0: raise ValueError("stop_token_ids cannot be empty") try: vocab_dict = tokenizer.get_vocab() except AttributeError as e: msg = ( f"Cannot get the vocabulary of the tokenizer {type(tokenizer)}. The tokenizer " "should have a get_vocab method." ) raise ValueError(msg) from e # Some tokenizer don't have token id 0 or 1 or 2. So the max_id could be larger than the # number of tokens. max_id = max(vocab_dict.values()) tokenizer_vocab_size = max(len(vocab_dict), max_id + 1) vocab_size = vocab_size or tokenizer_vocab_size # maintain tokenizer's indexing encoded_vocab = [""] * vocab_size for token, idx in vocab_dict.items(): if idx < vocab_size: encoded_vocab[idx] = token if isinstance(tokenizer, PreTrainedTokenizerFast): # huggingface fast tokenizer # - the vocabulary is directly obtained from tokenizer.get_vocab() # (tokenizer.backend_tokenizer.to_str() may not contain the full vocab, special # tokens may be omitted) # - the vocab size is obtained from len(tokenizer.get_vocab()) or provided by user # - the vocab type and add_prefix_space are obtained from # tokenizer.backend_tokenizer.to_str() # - stop token id is provided by user, or auto detected. backend_str = tokenizer.backend_tokenizer.to_str() if stop_token_ids is None: if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: stop_token_ids = [tokenizer.eos_token_id] else: logger.warning( "When constructing TokenizerInfo from a huggingface tokenizer, " "stop_token_ids is neither provided by user nor found from the tokenizer. " "It will be automatically detected." ) metadata = TokenizerInfo._detect_metadata_from_hf(backend_str) return TokenizerInfo( encoded_vocab, vocab_type=metadata["vocab_type"], vocab_size=vocab_size, stop_token_ids=stop_token_ids, add_prefix_space=metadata["add_prefix_space"], ) elif TokenizerInfo._is_tiktoken_tokenizer(tokenizer): # tiktoken tokenizer # e.g. Phi-3-small-8k-instruct, Qwen-7B-Chat, stablelm-2-12b-chat (previously) if stop_token_ids is None: if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: stop_token_ids = [tokenizer.eos_token_id] else: logger.warning( "When constructing TokenizerInfo from a huggingface tokenizer, " "stop_token_ids is neither provided by user nor found from the tokenizer. " "It will be automatically detected." ) return TokenizerInfo( encoded_vocab, VocabType.RAW, vocab_size=vocab_size, stop_token_ids=stop_token_ids, add_prefix_space=False, ) elif TokenizerInfo._is_sentencepiece_tokenizer(tokenizer): # sentencepiece tokenizer # e.g. Chatglm3-6b if hasattr(tokenizer, "sp_model"): sp_model = tokenizer.sp_model elif hasattr(tokenizer, "tokenizer") and hasattr(tokenizer.tokenizer, "sp_model"): sp_model = tokenizer.tokenizer.sp_model if stop_token_ids is None: if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: stop_token_ids = [tokenizer.eos_token_id] else: eos_id = sp_model.eos_id() if eos_id != -1: stop_token_ids = [eos_id] else: logger.warning( "When constructing TokenizerInfo from a huggingface tokenizer, " "stop_token_ids is neither provided by user nor found from the tokenizer. " "It will be automatically detected." ) # detect vocab_type of tokenizer if "<0x0A>" in vocab_dict: vocab_type = VocabType.BYTE_FALLBACK else: vocab_type = VocabType.RAW return TokenizerInfo( encoded_vocab, vocab_type=vocab_type, vocab_size=vocab_size, stop_token_ids=stop_token_ids, add_prefix_space=True, ) else: # TODO(yixin): unsupported tokenizer raise ValueError(f"Unsupported tokenizer type: {type(tokenizer)}") @property def vocab_type(self) -> VocabType: """The type of the vocabulary.""" return VocabType(self._handle.vocab_type) @property def vocab_size(self) -> int: """The size of the vocabulary.""" return self._handle.vocab_size @property def add_prefix_space(self) -> bool: """Whether the tokenizer will prepend a space before the text in the tokenization process.""" return self._handle.add_prefix_space @property def prepend_space_in_tokenization(self) -> bool: """Whether the tokenizer will prepend a space before the text in the tokenization process. This property is deprecated. Use add_prefix_space instead. """ logger.warning("prepend_space_in_tokenization is deprecated. Use add_prefix_space instead.") return self.add_prefix_space @property def decoded_vocab(self) -> List[bytes]: """The decoded vocabulary of the tokenizer. This converts the tokens in the LLM's vocabulary back to the original format of the input text. E.g. for type ByteFallback, the token <0x1B> is converted back to "\u001b". """ return self._handle.decoded_vocab @property def stop_token_ids(self) -> List[int]: """The stop token ids.""" return self._handle.stop_token_ids @property def special_token_ids(self) -> List[int]: """The special token ids. Special tokens include control tokens, reserved tokens, padded tokens, etc. Now it is automatically detected from the vocabulary.""" return self._handle.special_token_ids def dump_metadata(self) -> str: """Dump the metadata of the tokenizer to a json string. It can be used to construct the tokenizer info from the vocabulary and the metadata string.""" return self._handle.dump_metadata() @staticmethod def from_vocab_and_metadata( encoded_vocab: List[Union[bytes, str]], metadata: str ) -> "TokenizerInfo": """Construct the tokenizer info from the vocabulary and the metadata string in json format. Parameters ---------- encoded_vocab : List[Union[bytes, str]] The encoded vocabulary of the tokenizer. metadata : str The metadata string in json format. """ return TokenizerInfo._create_from_handle( _core.TokenizerInfo.from_vocab_and_metadata(encoded_vocab, metadata) ) @staticmethod def _detect_metadata_from_hf(backend_str: str) -> Dict[str, Any]: """Detect the metadata from the huggingface tokenizer backend string. For implementation use only. It returns {"vocab_type": VocabType, "add_prefix_space": bool}. """ # the metadata_str should in the format of {"vocab_type": int, "add_prefix_space": bool} metadata_str = _core.TokenizerInfo._detect_metadata_from_hf(backend_str) metadata = json.loads(metadata_str) return { "vocab_type": VocabType(metadata["vocab_type"]), "add_prefix_space": metadata["add_prefix_space"], } xgrammar-0.1.19/python/xgrammar/version.py000066400000000000000000000103101500705317600206150ustar00rootroot00000000000000# pylint: disable=missing-docstring import argparse import logging import os import subprocess # Modify the following value during release # --------------------------------------------------- # Current version: # We use the version of the incoming release for code # that is under development. # # It is also fallback version to be used when --git-describe # is not invoked, or when the repository does not present the # git tags in a format that this script can use. # # Two tag formats are supported: # - vMAJ.MIN.PATCH (e.g. v0.8.0) or # - vMAJ.MIN.devN (e.g. v0.8.dev0) # --------------------------------------------------- __version__ = "0.1.19" PROJ_ROOT = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) def py_str(cstr): return cstr.decode("utf-8") def git_describe_version(): """Get PEP-440 compatible public and local version using git describe. Returns ------- pub_ver: str Public version. local_ver: str Local version (with additional label appended to pub_ver). Notes ----- - We follow PEP 440's convention of public version and local versions. - Only tags conforming to vMAJOR.MINOR.REV (e.g. "v0.7.0") are considered in order to generate the version string. See the use of `--match` in the `git` command below. Here are some examples: - pub_ver = '0.7.0', local_ver = '0.7.0': We are at the 0.7.0 release. - pub_ver = '0.8.dev94', local_ver = '0.8.dev94+g0d07a329e': We are at the 0.8 development cycle. The current source contains 94 additional commits after the most recent tag(v0.7.0), the git short hash tag of the current commit is 0d07a329e. """ cmd = [ "git", "describe", "--tags", "--match", "v[0-9]*.[0-9]*.[0-9]*", "--match", "v[0-9]*.[0-9]*.dev[0-9]*", ] with subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=PROJ_ROOT ) as proc: (out, _) = proc.communicate() if proc.returncode != 0: msg = py_str(out) logging.warning("git describe: %s", msg) return None, None describe = py_str(out).strip() arr_info = describe.split("-") # Remove the v prefix, mainly to be robust # to the case where v is not presented as well. if arr_info[0].startswith("v"): arr_info[0] = arr_info[0][1:] # hit the exact tag if len(arr_info) == 1: return arr_info[0], arr_info[0] if len(arr_info) != 3: logging.warning("Invalid output from git describe %s", describe) return None, None dev_pos = arr_info[0].find(".dev") # Development versions: # The code will reach this point in case it can't match a full release version, such as v0.7.0. # # 1. in case the last known label looks like vMAJ.MIN.devN e.g. v0.8.dev0, we use # the current behavior of just using vMAJ.MIN.devNNNN+gGIT_REV if dev_pos != -1: dev_version = arr_info[0][: arr_info[0].find(".dev")] # 2. in case the last known label looks like vMAJ.MIN.PATCH e.g. v0.8.0 # then we just carry on with a similar version to what git describe provides, which is # vMAJ.MIN.PATCH.devNNNN+gGIT_REV else: dev_version = arr_info[0] pub_ver = f"{dev_version}.dev{arr_info[1]}" local_ver = f"{pub_ver}+{arr_info[2]}" return pub_ver, local_ver def main(): logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(description="Detect and synchronize version.") parser.add_argument( "--print-version", action="store_true", help="Print version to the command line. No changes is applied to files.", ) parser.add_argument( "--git-describe", action="store_true", help="Use git describe to generate development version.", ) parser.add_argument("--dry-run", action="store_true") opt = parser.parse_args() pub_ver, local_ver = None, None if opt.git_describe: pub_ver, local_ver = git_describe_version() if pub_ver is None: pub_ver = __version__ if local_ver is None: local_ver = __version__ if opt.print_version: print(local_ver) if __name__ == "__main__": main() xgrammar-0.1.19/scripts/000077500000000000000000000000001500705317600151135ustar00rootroot00000000000000xgrammar-0.1.19/scripts/build-environment.yaml000066400000000000000000000003511500705317600214370ustar00rootroot00000000000000name: xgrammar-build channels: - pytorch - conda-forge dependencies: - llvmdev>=15 - cmake>=3.24 - zlib - zstd-static - git - conda-build - numpy - pytest - pip - cython - nanobind - pytorch - cpuonly xgrammar-0.1.19/scripts/build_site.sh000077500000000000000000000002711500705317600175750ustar00rootroot00000000000000#!/bin/bash set -euxo pipefail export PYTHONPATH=$PWD/python cd docs && make html && cd .. cd site && jekyll b && cd .. rm -rf site/_site/docs cp -r docs/_build/html site/_site/docs xgrammar-0.1.19/scripts/docker/000077500000000000000000000000001500705317600163625ustar00rootroot00000000000000xgrammar-0.1.19/scripts/docker/bash.sh000077500000000000000000000044341500705317600176430ustar00rootroot00000000000000#!/usr/bin/env bash # # Start a bash, mount /workspace to be current directory. # # Usage: docker/bash.sh # Starts an interactive session # # Usage2: docker/bash.sh [COMMAND] # Execute command in the docker image, non-interactive # if [ "$#" -lt 1 ]; then echo "Usage: docker/bash.sh [--no-gpu] [COMMAND]" exit -1 fi if [ "$1" == "--no-gpu" ]; then ENABLE_NV_DOCKER=0 shift else ENABLE_NV_DOCKER=1 fi DOCKER_IMAGE_NAME=("$1") if [ "$#" -eq 1 ]; then COMMAND="bash" if [[ $(uname) == "Darwin" ]]; then # Docker's host networking driver isn't supported on macOS. # Use default bridge network and expose port for jupyter notebook. DOCKER_EXTRA_PARAMS=("-it -p 8888:8888") else DOCKER_EXTRA_PARAMS=("-it --net=host") fi else shift 1 COMMAND=("$@") fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKSPACE="$(pwd)" # Use nvidia-docker if the container is GPU. if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then CUDA_ENV="-e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" else CUDA_ENV="" fi # If this is an wheel test command then pass the env var to docker. if [[ ! -z $WHEEL_TEST ]]; then WHEEL_TEST="-e WHEEL_TEST=${WHEEL_TEST}" fi if [[ "${DOCKER_IMAGE_NAME}" == *"cu"* ]]; then if [ "$ENABLE_NV_DOCKER" -eq 1 ]; then if ! type "nvidia-docker" 1> /dev/null 2> /dev/null then DOCKER_BINARY="docker" CUDA_ENV=" --gpus all "${CUDA_ENV} else DOCKER_BINARY="nvidia-docker" fi else DOCKER_BINARY="docker" fi else DOCKER_BINARY="docker" fi # Print arguments. echo "WORKSPACE: ${WORKSPACE}" echo "DOCKER CONTAINER NAME: ${DOCKER_IMAGE_NAME}" echo "" echo "Running '${COMMAND[@]}' inside ${DOCKER_IMAGE_NAME}..." # By default we cleanup - remove the container once it finish running (--rm) # and share the PID namespace (--pid=host) so the process inside does not have # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it). ${DOCKER_BINARY} run --rm --pid=host\ -v ${WORKSPACE}:/workspace \ -v ${SCRIPT_DIR}:/docker \ -w /workspace \ ${CUDA_ENV} \ ${WHEEL_TEST} \ ${DOCKER_EXTRA_PARAMS[@]} \ ${DOCKER_IMAGE_NAME} \ ${COMMAND[@]} xgrammar-0.1.19/scripts/gh_deploy_site.sh000077500000000000000000000006741500705317600204570ustar00rootroot00000000000000#!/bin/bash # NOTE: this script is triggered by github action automatically # when megred into main set -euxo pipefail scripts/build_site.sh git fetch git checkout -B gh-pages origin/gh-pages rm -rf docs .gitignore mkdir -p docs cp -rf site/_site/* docs touch docs/.nojekyll DATE=`date` git add docs && git commit -am "Build at ${DATE}" git push origin gh-pages git checkout main && git submodule update echo "Finish deployment at ${DATE}" xgrammar-0.1.19/scripts/lint.sh000077500000000000000000000001221500705317600164130ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -x pre-commit run --all-files ruff check . --fix xgrammar-0.1.19/scripts/local_deploy_site.sh000077500000000000000000000002731500705317600211460ustar00rootroot00000000000000#!/bin/bash # NOTE: use this script to check local site set -euxo pipefail scripts/build_site.sh cd site && jekyll serve --skip-initial-build --host localhost --baseurl / --port 8888 xgrammar-0.1.19/scripts/release_new_version.sh000077500000000000000000000004471500705317600215150ustar00rootroot00000000000000#!/bin/bash # Usage: ./scripts/release_new_version.sh set -ex if [ -z "$1" ]; then echo "Error: Version argument is required" echo "Usage: $0 " exit 1 fi # Pull and checkout main branch git pull origin main git checkout main git tag $1 HEAD git push origin $1 xgrammar-0.1.19/site/000077500000000000000000000000001500705317600143705ustar00rootroot00000000000000xgrammar-0.1.19/site/.gitignore000066400000000000000000000000311500705317600163520ustar00rootroot00000000000000dist _site .jekyll-cache xgrammar-0.1.19/site/CNAME000066400000000000000000000000201500705317600151260ustar00rootroot00000000000000xgrammar.mlc.ai xgrammar-0.1.19/site/Gemfile000066400000000000000000000002021500705317600156550ustar00rootroot00000000000000# frozen_string_literal: true source "https://rubygems.org" # gem "rails" gem "jekyll-remote-theme" gem "jekyll-sass-converter" xgrammar-0.1.19/site/_config.yml000066400000000000000000000013431500705317600165200ustar00rootroot00000000000000name: "XGrammar" short_name: "XGrammar" url: https://xgrammar.mlc.ai/ exclude: [README.md, serve_local.sh] plugins: - jekyll-remote-theme remote_theme: mlc-ai/jekyll-theme-mlc # Colorize code snippets with the rogue module if we want to deploy on GH. highlighter: rouge markdown: kramdown # The path structure for blog posts. permalink: /blog/:year/:month/:day/:title.html # Number of news stories on the front page. front_page_news: 8 # Base pathname for links. base: '' # make pages for the _projects folder collections: projects: output: true course_title: # Navigation bar links. navigation: - title: Home link: / - title: Docs link: /docs - title: Github link: https://github.com/mlc-ai/xgrammar xgrammar-0.1.19/site/_includes/000077500000000000000000000000001500705317600163355ustar00rootroot00000000000000xgrammar-0.1.19/site/_includes/arrow.svg000066400000000000000000000015201500705317600202060ustar00rootroot00000000000000 xgrammar-0.1.19/site/_includes/github.svg000066400000000000000000000017301500705317600203410ustar00rootroot00000000000000 xgrammar-0.1.19/site/_includes/head.html000066400000000000000000000014761500705317600201340ustar00rootroot00000000000000 xgrammar-0.1.19/site/_includes/hero.html000066400000000000000000000020671500705317600201650ustar00rootroot00000000000000

XGrammar: Efficient, Flexible and Portable Structured Generation

{% include github.svg %} GitHub {% include arrow.svg %} Documentation {% include arrow.svg %}
xgrammar-0.1.19/site/assets/000077500000000000000000000000001500705317600156725ustar00rootroot00000000000000xgrammar-0.1.19/site/assets/css/000077500000000000000000000000001500705317600164625ustar00rootroot00000000000000xgrammar-0.1.19/site/assets/css/hero.scss000066400000000000000000000125661500705317600203260ustar00rootroot00000000000000--- --- #hero { background: radial-gradient(100% 50rem at center 50rem, #3351cb50, #ffffff); padding: 3rem; width: 100vw; margin-left: calc(50% - 50vw); margin-top: -20px; display: flex; flex-direction: column; align-items: center; a { color: black; } .heading-container { display: flex; flex-direction: column; align-items: center; font-family: "Mona Sans", "MonaSansFallback", -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji"; margin: auto; a { min-width: fit-content; max-width: 16rem; flex-grow: 1; } h1 { text-align: center; font-size: 2rem; font-weight: 700; } .link-container { display: flex; margin-top: 2rem; align-items: center; flex-wrap: wrap; font-size: 1rem; word-break: keep-all; font-weight: 600; gap: 1rem; justify-content: center; .github-link { display: inline-flex; gap: 1rem; border-radius: 9999px; vertical-align: middle; align-items: center; justify-content: center; text-decoration: none; cursor: pointer; height: fit-content; // padding: .25rem; .github-link-content { width: 100%; height: 100%; z-index: 1; border-radius: 9999px; padding: 1rem 1.75rem; background-color: #000000; display: inline-flex; gap: .5rem; display: inline-flex; justify-content: center; color: rgb(229 229 229); .icon { display: inline-flex; align-items: center; margin-right: .5rem; svg { height: 1.5rem; } } } } .get-start-link { display: inline-flex; gap: 1rem; background-color: white; border-radius: 9999px; vertical-align: middle; align-items: center; justify-content: center; text-decoration: none; cursor: pointer; height: fit-content; padding: .25rem; .get-start-link-content { width: 100%; height: 100%; z-index: 1; border-radius: 9999px; padding: 1rem 1.75rem; background-color: white; display: inline-flex; justify-content: center; } } .arrow-container { margin-left: .25rem; display: inline-flex; align-items: center; } } } .arrow-expandable { stroke-dasharray: 10; stroke-dashoffset: 10; transition: stroke-dashoffset 200ms; } .expanded { .arrow-expandable { stroke-dashoffset: 20; } } .demo-container { position: relative; margin-top: 96px; width: calc(100% + 4rem); max-width: 1024px; flex-shrink: 0; padding: 2rem; svg { height: auto; width: 100%; border-radius: inherit; } } } .moving-border { overflow: hidden; position: relative; .border { position: absolute; inset: -1000%; animation: spin 3s linear infinite; border-radius: 1rem; background-image: conic-gradient(from 90deg at 50% 50%, #e2cbff 0, #393bb2 50%, #e2cbff 100%); } } @media screen and (min-width:640px) { #hero { padding: 6rem; .heading-container { max-width: 40rem; h1 { font-size: 3rem; } } .demo-container { width: calc(100% + 10rem); } } } @media screen and (min-width:768px) { #hero { .heading-container { max-width: 45rem; h1 { font-size: 3.2rem; } .link-container { font-size: 1.2rem; } } } } @media screen and (min-width:1024px) { #hero { padding: 8rem; .heading-container { max-width: 50rem; h1 { font-size: 3.5rem; } } .demo-container { width: 100%; } } } @media screen and (min-width:1280px) { #hero { .heading-container { max-width: 60rem; h1 { font-size: 4rem; } } } } @media screen and (min-width:1760px) { #hero { background: radial-gradient(100% 50rem at center 50rem, #3351cb50, #ffffff); gap: 4rem; padding-bottom: 12rem; } } @keyframes spin { 100% { transform: rotate(1turn); } } xgrammar-0.1.19/site/index.md000066400000000000000000000016601500705317600160240ustar00rootroot00000000000000--- layout: default title: Home notitle: true --- {% include hero.html %} ## Overview XGrammar is open-source solution for flexible, portable, and fast structured generations, aiming at bring flexible zero-overhead structure generation everywhere. It supports general context-free grammar to enable a broad range of structures while bringing careful system optimizations to enable fast executions. XGrammar features a minimal and portable C++ backend that can be easily integrated into multiple environments and frameworks, and is co-designed with the LLM inference engine and enables zero-overhead structured generation in LLM inference. ## Get Started Please visit our [documentation](https://xgrammar.mlc.ai/docs/) to get started with XGrammar. - [Installation](https://xgrammar.mlc.ai/docs/start/install) - [Quick start](https://xgrammar.mlc.ai/docs/start/quick_start) ## Links - [XGrammar Github](https://github.com/mlc-ai/xgrammar) xgrammar-0.1.19/tests/000077500000000000000000000000001500705317600145665ustar00rootroot00000000000000xgrammar-0.1.19/tests/README.md000066400000000000000000000005571500705317600160540ustar00rootroot00000000000000To test, run `pytest .` under `xgrammar` folder. You may need to do the following: ```bash pip install sentencepiece pip install protobuf pip install -U "huggingface_hub[cli]" huggingface-cli login --token YOUR_HF_TOKEN ``` Make sure you also have access to the gated models, which should only require you to agree some terms on the models' website on huggingface. xgrammar-0.1.19/tests/cpp/000077500000000000000000000000001500705317600153505ustar00rootroot00000000000000xgrammar-0.1.19/tests/cpp/test_fsm.cc000066400000000000000000000520011500705317600175010ustar00rootroot00000000000000#include #include #include "fsm.h" using namespace xgrammar; TEST(XGrammarFSMTest, BasicBuildTest) { std::cout << "--------- Basic Build Test Starts! -----------" << std::endl; std::cout << "--------- Basic Build Test1 -----------" << std::endl; auto fsm_wse = RegexToFSM("abcd\\n").Unwrap(); std::string test_str = "abcd\n"; EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Basic Build Test2 -----------" << std::endl; fsm_wse = RegexToFSM("[-a-z\\n]").Unwrap(); test_str = "abcd-\n"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.fsm.edges[0]) { if (edge.min <= int(character) && edge.max >= int(character)) { return true; } } return false; }()); } std::cout << "--------- Basic Build Test3 -----------" << std::endl; fsm_wse = RegexToFSM("[\\d]").Unwrap(); test_str = "1234567890"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.fsm.edges[0]) { if (edge.min <= int(character) && edge.max >= int(character)) { return true; } } return false; }()); } std::cout << "--------- Basic Build Test4 -----------" << std::endl; fsm_wse = RegexToFSM("[^\\d]").Unwrap(); test_str = "1234567890"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.fsm.edges[0]) { if (edge.min <= int(character) && edge.max >= int(character)) { return false; } } return true; }()); } test_str = "abz"; for (const auto& character : test_str) { EXPECT_TRUE([&]() -> bool { for (const auto& edge : fsm_wse.fsm.edges[0]) { if (edge.min <= int(character) && edge.max >= int(character)) { return true; } } std::cout << character << std::endl; return false; }()); } std::cout << "--------- Basic Build Test5 -----------" << std::endl; fsm_wse = RegexToFSM("你好a").Unwrap(); test_str = "你好a"; EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Basic Build Test6 -----------" << std::endl; fsm_wse = RegexToFSM("(())()()").Unwrap(); test_str = ""; EXPECT_FALSE(fsm_wse.Check(test_str)); std::cout << "--------- Basic Build Test7 -----------" << std::endl; fsm_wse = RegexToFSM("[abcdabcdxyzxyz]").Unwrap(); test_str = "a"; EXPECT_TRUE(fsm_wse.Check(test_str)); EXPECT_FALSE(fsm_wse.Check("e")); std::cout << fsm_wse << std::endl; EXPECT_EQ(fsm_wse.fsm.edges[0].size(), 2); std::cout << "Basic Build Test Passed!" << std::endl; } TEST(XGrammarFSMTest, ConnectionTest) { std::cout << "--------- Connection Test Starts! -----------" << std::endl; std::cout << "--------- Connection Test1 -----------" << std::endl; auto fsm_wse = RegexToFSM(" [a-zA-Z0-9]--").Unwrap(); std::string test_str = " a--"; EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Connection Test2 -----------" << std::endl; fsm_wse = RegexToFSM("aaa|[\\d]").Unwrap(); test_str = "aaa"; EXPECT_TRUE(fsm_wse.Check(test_str)); test_str = "1"; EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Connection Test3 -----------" << std::endl; if (RegexToFSM("(([\\d]|[\\w])|aaa)").IsErr()) { std::cout << RegexToFSM("(([\\d]|[\\w])|aaa)").UnwrapErr()->what() << std::endl; } fsm_wse = RegexToFSM("(([\\d]|[\\w])|aaa)").Unwrap(); test_str = "aaa"; EXPECT_TRUE(fsm_wse.Check(test_str)); test_str = "1"; EXPECT_TRUE(fsm_wse.Check(test_str)); test_str = "1a"; EXPECT_FALSE(fsm_wse.Check(test_str)); std::cout << "Connection Test Passed!" << std::endl; } TEST(XGrammarFSMTest, SymbolTest) { std::cout << "--------- Symbol Test Starts! -----------" << std::endl; std::cout << "--------- Symbol Test1 -----------" << std::endl; auto fsm_wse = RegexToFSM("1[\\d]+").Unwrap(); std::string test_str[2] = {"1111", "1"}; EXPECT_TRUE(fsm_wse.Check(test_str[0])); EXPECT_FALSE(fsm_wse.Check(test_str[1])); std::cout << "--------- Symbol Test2 -----------" << std::endl; fsm_wse = RegexToFSM("1[1]*").Unwrap(); EXPECT_TRUE(fsm_wse.Check(test_str[0])); EXPECT_TRUE(fsm_wse.Check(test_str[1])); std::cout << "--------- Symbol Test3 -----------" << std::endl; fsm_wse = RegexToFSM("1[\\d]?").Unwrap(); EXPECT_FALSE(fsm_wse.Check(test_str[0])); EXPECT_TRUE(fsm_wse.Check(test_str[1])); std::string test3 = "11"; EXPECT_TRUE(fsm_wse.Check(test3)); std::cout << "--------- Symbol Test4 -----------" << std::endl; fsm_wse = RegexToFSM(" * * + ? *").Unwrap(); test_str[0] = " "; test_str[1] = " "; for (const auto& str : test_str) { EXPECT_TRUE(fsm_wse.Check(str)); } std::cout << "Symbol Test Passed!" << std::endl; } TEST(XGrammarFSMTest, IntegratedTest) { std::cout << "--------- Integrated Test Starts! -----------" << std::endl; auto fsm_wse = RegexToFSM("((naive|bbb|[\\d]+)*[\\w])| +").Unwrap(); std::string test_str[5] = {"naive1", "bbbnaive114514W", " ", "123", "_"}; for (const auto& str : test_str) { EXPECT_TRUE(fsm_wse.Check(str)); } std::string test_str2[5] = {"naive", "bbbbbb", "naive ", "123 ", "aaa"}; for (const auto& str : test_str2) { EXPECT_FALSE(fsm_wse.Check(str)); } std::cout << "--------- Integrated Test Passed! -----------" << std::endl; } TEST(XGrammarFSMTest, FunctionTest) { std::cout << "--------- Function Test Starts! -----------" << std::endl; std::cout << "--------- Function Test1 -----------" << std::endl; auto fsm_wse = RegexToFSM("[\\d\\d\\d]+123").Unwrap(); std::string test_str = "123456123"; EXPECT_TRUE(fsm_wse.Check(test_str)); auto compact_fsm = fsm_wse.fsm.ToCompact(); CompactFSMWithStartEnd compact_fsm_wse; compact_fsm_wse.fsm = compact_fsm; compact_fsm_wse.start = fsm_wse.start; compact_fsm_wse.ends = fsm_wse.ends; EXPECT_TRUE(compact_fsm_wse.Check(test_str)); fsm_wse.fsm = compact_fsm_wse.fsm.ToFSM(); EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Function Test2 -----------" << std::endl; fsm_wse = RegexToFSM("([abc]|[\\d])+").Unwrap(); test_str = "abc3"; EXPECT_TRUE(fsm_wse.Check(test_str)); fsm_wse = fsm_wse.ToDFA(); EXPECT_TRUE(fsm_wse.Check(test_str)); EXPECT_TRUE([&]() -> bool { for (const auto& edges : fsm_wse.fsm.edges) { for (const auto& edge : edges) { if (edge.IsEpsilon()) { return false; } } } return true; }()); EXPECT_TRUE([&]() -> bool { for (const auto& edges : fsm_wse.fsm.edges) { std::unordered_set rules; std::unordered_set chars; for (const auto& edge : edges) { if (edge.IsRuleRef()) { if (rules.find(edge.GetRefRuleId()) != rules.end()) { return false; } rules.insert(edge.GetRefRuleId()); continue; } for (int i = edge.min; i <= edge.max; i++) { if (chars.find(i) != chars.end()) { return false; } chars.insert(i); } } } return true; }()); std::cout << "--------- Function Test3 -----------" << std::endl; fsm_wse = fsm_wse.MinimizeDFA(); EXPECT_TRUE(fsm_wse.Check(test_str)); EXPECT_EQ(fsm_wse.fsm.edges.size(), 3); std::cout << "--------- Function Test4 -----------" << std::endl; fsm_wse = fsm_wse.Not(); EXPECT_FALSE(fsm_wse.Check(test_str)); test_str = "abcd"; EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Function Test5 -----------" << std::endl; fsm_wse = RegexToFSM("[\\d]{1, 5}").Unwrap(); std::string test_strs[2] = {"123", "12345"}; for (const auto& str : test_strs) { EXPECT_TRUE(fsm_wse.Check(str)); } test_strs[0] = "123456"; test_strs[1] = "1234567"; for (const auto& str : test_strs) { EXPECT_FALSE(fsm_wse.Check(str)); } fsm_wse = RegexToFSM("[\\d]{6}").Unwrap(); EXPECT_TRUE(fsm_wse.Check("123456")); EXPECT_FALSE(fsm_wse.Check("1234567")); fsm_wse = RegexToFSM("[\\d]{6, }").Unwrap(); EXPECT_TRUE(fsm_wse.Check("123456")); EXPECT_TRUE(fsm_wse.Check("1234567")); std::cout << "--------- Function Test6 -----------" << std::endl; fsm_wse = RegexToFSM("[a][b][c][d]").Unwrap(); test_str = "abcd"; EXPECT_TRUE(fsm_wse.Check(test_str)); fsm_wse.SimplifyEpsilon(); EXPECT_EQ(fsm_wse.NumNodes(), 5); EXPECT_TRUE(fsm_wse.Check(test_str)); std::cout << "--------- Function Test7 -----------" << std::endl; fsm_wse = RegexToFSM("abc|abd").Unwrap(); test_str = "abc"; EXPECT_TRUE(fsm_wse.Check(test_str)); fsm_wse.SimplifyTransition(); fsm_wse.SimplifyEpsilon(); EXPECT_TRUE(fsm_wse.Check(test_str)); test_str = "abcd"; EXPECT_FALSE(fsm_wse.Check(test_str)); EXPECT_EQ(fsm_wse.NumNodes(), 4); std::cout << "--------- Function Test8 -----------" << std::endl; fsm_wse = RegexToFSM("acd|bcd").Unwrap(); test_str = "acd"; EXPECT_TRUE(fsm_wse.Check(test_str)); fsm_wse.SimplifyTransition(); fsm_wse.SimplifyEpsilon(); EXPECT_TRUE(fsm_wse.Check(test_str)); test_str = "abcd"; EXPECT_FALSE(fsm_wse.Check(test_str)); EXPECT_EQ(fsm_wse.NumNodes(), 4); std::cout << "--------- Function Test9 -----------" << std::endl; fsm_wse = RegexToFSM("ab*").Unwrap(); test_str = "abbb"; EXPECT_TRUE(fsm_wse.Check(test_str)); fsm_wse.SimplifyEpsilon(); EXPECT_TRUE(fsm_wse.Check(test_str)); EXPECT_EQ(fsm_wse.NumNodes(), 2); std::cout << "--------- Function Test Passed! -----------" << std::endl; } TEST(XGrammarFSMTest, EfficiencyTest) { std::cout << "--------- Efficiency Test Starts! -----------" << std::endl; // i.e ([a-z]0123456789){10}. Use this way to test the performance. auto fsm_wse = RegexToFSM( "(a0123456789|a0123456789|b0123456789|b0123456789|c0123456789|" "c0123456789|d0123456789|d0123456789|e0123456789|e0123456789|" "f0123456789|f0123456789|g0123456789|g0123456789|h0123456789|" "h0123456789|i0123456789|i0123456789|j0123456789|j0123456789|" "k0123456789|k0123456789|l0123456789|l0123456789|m0123456789|" "m0123456789|n0123456789|n0123456789|o0123456789|o0123456789|" "p0123456789|p0123456789|q0123456789|q0123456789|r0123456789|" "r0123456789|s0123456789|s0123456789|t0123456789|t0123456789|" "u0123456789|u0123456789|v0123456789|v0123456789|w0123456789|" "w0123456789|x0123456789|x0123456789|y0123456789|y0123456789|" "z0123456789|z0123456789)(a0123456789|a0123456789|b0123456789|" "b0123456789|c0123456789|c0123456789|d0123456789|d0123456789|" "e0123456789|e0123456789|f0123456789|f0123456789|g0123456789|" "g0123456789|h0123456789|h0123456789|i0123456789|i0123456789|" "j0123456789|j0123456789|k0123456789|k0123456789|l0123456789|" "l0123456789|m0123456789|m0123456789|n0123456789|n0123456789|" "o0123456789|o0123456789|p0123456789|p0123456789|q0123456789|" "q0123456789|r0123456789|r0123456789|s0123456789|s0123456789|" "t0123456789|t0123456789|u0123456789|u0123456789|v0123456789|" "v0123456789|w0123456789|w0123456789|x0123456789|x0123456789|" "y0123456789|y0123456789|z0123456789|z0123456789)(a0123456789|" "a0123456789|b0123456789|b0123456789|c0123456789|c0123456789|" "d0123456789|d0123456789|e0123456789|e0123456789|f0123456789|" "f0123456789|g0123456789|g0123456789|h0123456789|h0123456789|" "i0123456789|i0123456789|j0123456789|j0123456789|k0123456789|" "k0123456789|l0123456789|l0123456789|m0123456789|m0123456789|" "n0123456789|n0123456789|o0123456789|o0123456789|p0123456789|" "p0123456789|q0123456789|q0123456789|r0123456789|r0123456789|" "s0123456789|s0123456789|t0123456789|t0123456789|u0123456789|" "u0123456789|v0123456789|v0123456789|w0123456789|w0123456789|" "x0123456789|x0123456789|y0123456789|y0123456789|z0123456789|" "z0123456789)(a0123456789|a0123456789|b0123456789|b0123456789|" "c0123456789|c0123456789|d0123456789|d0123456789|e0123456789|" "e0123456789|f0123456789|f0123456789|g0123456789|g0123456789|" "h0123456789|h0123456789|i0123456789|i0123456789|j0123456789|" "j0123456789|k0123456789|k0123456789|l0123456789|l0123456789|" "m0123456789|m0123456789|n0123456789|n0123456789|o0123456789|" "o0123456789|p0123456789|p0123456789|q0123456789|q0123456789|" "r0123456789|r0123456789|s0123456789|s0123456789|t0123456789|" "t0123456789|u0123456789|u0123456789|v0123456789|v0123456789|" "w0123456789|w0123456789|x0123456789|x0123456789|y0123456789|" "y0123456789|z0123456789|z0123456789)(a0123456789|a0123456789|" "b0123456789|b0123456789|c0123456789|c0123456789|d0123456789|" "d0123456789|e0123456789|e0123456789|f0123456789|f0123456789|" "g0123456789|g0123456789|h0123456789|h0123456789|i0123456789|" "i0123456789|j0123456789|j0123456789|k0123456789|k0123456789|" "l0123456789|l0123456789|m0123456789|m0123456789|n0123456789|" "n0123456789|o0123456789|o0123456789|p0123456789|p0123456789|" "q0123456789|q0123456789|r0123456789|r0123456789|s0123456789|" "s0123456789|t0123456789|t0123456789|u0123456789|u0123456789|" "v0123456789|v0123456789|w0123456789|w0123456789|x0123456789|" "x0123456789|y0123456789|y0123456789|z0123456789|z0123456789)(" "a0123456789|a0123456789|b0123456789|b0123456789|c0123456789|" "c0123456789|d0123456789|d0123456789|e0123456789|e0123456789|" "f0123456789|f0123456789|g0123456789|g0123456789|h0123456789|" "h0123456789|i0123456789|i0123456789|j0123456789|j0123456789|" "k0123456789|k0123456789|l0123456789|l0123456789|m0123456789|" "m0123456789|n0123456789|n0123456789|o0123456789|o0123456789|" "p0123456789|p0123456789|q0123456789|q0123456789|r0123456789|" "r0123456789|s0123456789|s0123456789|t0123456789|t0123456789|" "u0123456789|u0123456789|v0123456789|v0123456789|w0123456789|" "w0123456789|x0123456789|x0123456789|y0123456789|y0123456789|" "z0123456789|z0123456789)(a0123456789|a0123456789|b0123456789|" "b0123456789|c0123456789|c0123456789|d0123456789|d0123456789|" "e0123456789|e0123456789|f0123456789|f0123456789|g0123456789|" "g0123456789|h0123456789|h0123456789|i0123456789|i0123456789|" "j0123456789|j0123456789|k0123456789|k0123456789|l0123456789|" "l0123456789|m0123456789|m0123456789|n0123456789|n0123456789|" "o0123456789|o0123456789|p0123456789|p0123456789|q0123456789|" "q0123456789|r0123456789|r0123456789|s0123456789|s0123456789|" "t0123456789|t0123456789|u0123456789|u0123456789|v0123456789|" "v0123456789|w0123456789|w0123456789|x0123456789|x0123456789|" "y0123456789|y0123456789|z0123456789|z0123456789)(a0123456789|" "a0123456789|b0123456789|b0123456789|c0123456789|c0123456789|" "d0123456789|d0123456789|e0123456789|e0123456789|f0123456789|" "f0123456789|g0123456789|g0123456789|h0123456789|h0123456789|" "i0123456789|i0123456789|j0123456789|j0123456789|k0123456789|" "k0123456789|l0123456789|l0123456789|m0123456789|m0123456789|" "n0123456789|n0123456789|o0123456789|o0123456789|p0123456789|" "p0123456789|q0123456789|q0123456789|r0123456789|r0123456789|" "s0123456789|s0123456789|t0123456789|t0123456789|u0123456789|" "u0123456789|v0123456789|v0123456789|w0123456789|w0123456789|" "x0123456789|x0123456789|y0123456789|y0123456789|z0123456789|" "z0123456789)(a0123456789|a0123456789|b0123456789|b0123456789|" "c0123456789|c0123456789|d0123456789|d0123456789|e0123456789|" "e0123456789|f0123456789|f0123456789|g0123456789|g0123456789|" "h0123456789|h0123456789|i0123456789|i0123456789|j0123456789|" "j0123456789|k0123456789|k0123456789|l0123456789|l0123456789|" "m0123456789|m0123456789|n0123456789|n0123456789|o0123456789|" "o0123456789|p0123456789|p0123456789|q0123456789|q0123456789|" "r0123456789|r0123456789|s0123456789|s0123456789|t0123456789|" "t0123456789|u0123456789|u0123456789|v0123456789|v0123456789|" "w0123456789|w0123456789|x0123456789|x0123456789|y0123456789|" "y0123456789|z0123456789|z0123456789)(a0123456789|a0123456789|" "b0123456789|b0123456789|c0123456789|c0123456789|d0123456789|" "d0123456789|e0123456789|e0123456789|f0123456789|f0123456789|" "g0123456789|g0123456789|h0123456789|h0123456789|i0123456789|" "i0123456789|j0123456789|j0123456789|k0123456789|k0123456789|" "l0123456789|l0123456789|m0123456789|m0123456789|n0123456789|" "n0123456789|o0123456789|o0123456789|p0123456789|p0123456789|" "q0123456789|q0123456789|r0123456789|r0123456789|s0123456789|" "s0123456789|t0123456789|t0123456789|u0123456789|u0123456789|" "v0123456789|v0123456789|w0123456789|w0123456789|x0123456789|" "x0123456789|y0123456789|y0123456789|z0123456789|z0123456789)" ) .Unwrap(); std::cout << "Initial Node Numbers:" << fsm_wse.NumNodes() << std::endl; auto time_start = std::chrono::high_resolution_clock::now(); fsm_wse.SimplifyEpsilon(); auto time_end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to simplify epsilon: " << duration.count() << " ms" << std::endl; std::cout << "After SimplifyEpsilon Node Numbers:" << fsm_wse.NumNodes() << std::endl; time_start = std::chrono::high_resolution_clock::now(); fsm_wse.SimplifyTransition(); time_end = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to simplify transition: " << duration.count() << " ms" << std::endl; std::cout << "After SimplifyTransition Node Numbers:" << fsm_wse.NumNodes() << std::endl; time_start = std::chrono::high_resolution_clock::now(); fsm_wse = fsm_wse.ToDFA(); time_end = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to convert to DFA: " << duration.count() << " ms" << std::endl; std::cout << "After ToDFA Node Numbers:" << fsm_wse.NumNodes() << std::endl; time_start = std::chrono::high_resolution_clock::now(); fsm_wse = fsm_wse.MinimizeDFA(); time_end = std::chrono::high_resolution_clock::now(); duration = std::chrono::duration_cast(time_end - time_start); std::cout << "Time taken to minimize DFA: " << duration.count() << " ms" << std::endl; EXPECT_EQ(fsm_wse.NumNodes(), 111); std::cout << "--------- Efficiency Test Passed! -----------" << std::endl; } TEST(XGrammarFSMTest, BuildTrieTest) { std::vector patterns = {"hello", "hi", "哈哈", "哈", "hili", "good"}; auto fsm = BuildTrie(patterns); // Test1: The printed result of FSM // Test2: The printed result of CompactFSM CompactFSMWithStartEnd compact_fsm; compact_fsm.start = fsm.StartNode(); compact_fsm.ends = fsm.ends; compact_fsm.fsm = fsm.fsm.ToCompact(); // Test3: Walk through the FSM int state = fsm.StartNode(); EXPECT_EQ(state, 0); // Test "hello" state = fsm.StartNode(); EXPECT_EQ(fsm.Transition(state, 'h'), 1); EXPECT_EQ(fsm.Transition(1, 'e'), 2); EXPECT_EQ(fsm.Transition(2, 'l'), 3); EXPECT_EQ(fsm.Transition(3, 'l'), 4); EXPECT_EQ(fsm.Transition(4, 'o'), 5); EXPECT_TRUE(fsm.IsEndNode(5)); // Test "hil" state = fsm.StartNode(); EXPECT_EQ(fsm.Transition(state, 'h'), 1); EXPECT_EQ(fsm.Transition(1, 'i'), 6); EXPECT_EQ(fsm.Transition(6, 'l'), 13); EXPECT_FALSE(fsm.IsEndNode(13)); // Test walk failure state = fsm.StartNode(); EXPECT_EQ(fsm.Transition(state, 'g'), 15); EXPECT_EQ(fsm.Transition(15, 'o'), 16); EXPECT_EQ(fsm.Transition(16, 'e'), -1); } xgrammar-0.1.19/tests/cpp/test_thread_pool.cc000066400000000000000000000036771500705317600212330ustar00rootroot00000000000000#include #include #include "support/thread_pool.h" using namespace xgrammar; TEST(XGramamrThreadPoolTest, FunctionalTest) { ThreadPool pool(4); // Example 1: Use Submit to submit tasks with return values std::vector> futures; for (int i = 0; i < 8; ++i) { auto fut = pool.Submit([i] { std::this_thread::sleep_for(std::chrono::milliseconds(100)); std::cout << "Task " << i << " is running in thread " << std::this_thread::get_id() << "\n"; return i * i; }); futures.push_back(fut); } for (auto& fut : futures) { int result = fut.get(); std::cout << "Result: " << result << "\n"; } // Example 2: Use Execute to submit tasks without return values for (int i = 0; i < 5; ++i) { pool.Execute([i] { std::this_thread::sleep_for(std::chrono::milliseconds(50)); std::cout << "Execute task " << i << " is running in thread " << std::this_thread::get_id() << "\n"; }); } // Wait for task to complete pool.Join(); } // TEST(XGramamrThreadPoolTest, PressureTest) { // const size_t num_threads = std::thread::hardware_concurrency(); // ThreadPool pool(num_threads); // const size_t num_tasks = 1000; // int counter = 0; // std::mutex counter_mutex; // auto start_time = std::chrono::high_resolution_clock::now(); // for (size_t i = 0; i < num_tasks; ++i) { // pool.Execute([&counter, &counter_mutex, i]() { // std::this_thread::sleep_for(std::chrono::milliseconds(i % 50)); // std::lock_guard lock(counter_mutex); // counter++; // }); // } // pool.Wait(); // auto end_time = std::chrono::high_resolution_clock::now(); // EXPECT_EQ(counter, static_cast(num_tasks)); // auto duration = std::chrono::duration_cast(end_time - start_time); // std::cout << "Pressure test completed, time taken: " << duration.count() << " milliseconds.\n"; // } xgrammar-0.1.19/tests/cpp/test_thread_safe_cache.cc000066400000000000000000000201311500705317600223030ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include "support/logging.h" #include "support/thread_safe_cache.h" using namespace xgrammar; namespace { // static_assert( // sizeof(CompiledGrammar) >= sizeof(std::size_t), // "Our test requires that CompiledGrammar is at least as large as std::size_t" // ); // // simulate a CompiledGrammar object // struct MockGrammar { // std::size_t uuid; // std::byte padding[sizeof(CompiledGrammar) - sizeof(std::size_t)]; // MockGrammar() = default; // MockGrammar(std::size_t uuid) : uuid(uuid) {} // }; // struct SizeEstimator { // template // std::size_t operator()(const T&) const { // return 1; // } // }; // using namespace std::chrono_literals; // struct Computer0 { // inline static auto counter = std::atomic_size_t{}; // inline static constexpr auto kSleepTime = 1000ms; // MockGrammar operator()(std::size_t key) const { // std::this_thread::sleep_for(kSleepTime); // simulate a slow operation // return MockGrammar{counter++}; // } // }; // constexpr auto kUnlimited = std::size_t(-1); // constexpr auto kOverheadRatio = 0.1; // TEST(XGrammarParallelTest, CacheContention) { // XGRAMMAR_LOG_INFO << "Testing the contention performance of the cache (no eviction)"; // constexpr auto kReadGroup = 8; // const auto kNumThreads = int(std::thread::hardware_concurrency()) * 4; // // never evict // auto cache = ThreadSafeLRUCache{kUnlimited}; // auto futures = std::vector>{}; // futures.reserve(kNumThreads); // const auto tic = std::chrono::high_resolution_clock::now(); // const auto target = tic + 1s; // for (int i = 0; i < kNumThreads; ++i) { // futures.push_back(std::async(std::launch::async, [=, &cache] { // std::this_thread::sleep_until(target); // auto sum = std::size_t{}; // // write group: they should not compete with each other // for (int j = 0; j < kNumThreads; ++j) { // sum += cache.Get((i + j) % kNumThreads).uuid; // } // // read group: they should not compete with each other // for (int k = 0; k < kReadGroup; ++k) { // for (int j = 0; j < kNumThreads; ++j) { // sum += cache.Get((i + j) % kNumThreads).uuid; // } // } // return sum; // })); // } // const auto kResult = std::size_t(kNumThreads) * (kNumThreads - 1) / 2 * (1 + kReadGroup); // for (int i = 0; i < kNumThreads; ++i) EXPECT_EQ(futures[i].get(), kResult); // const auto toc = std::chrono::high_resolution_clock::now(); // const auto dur = std::chrono::duration_cast(toc - tic); // // remove 1s sleep time and computing sleep time // const auto overhead = dur - 1s - Computer0::kSleepTime; // XGRAMMAR_LOG_INFO << "(1 write + " << kReadGroup << " reads) " // << "* " << kNumThreads << " threads | " // << "overhead = " << overhead.count() << "ms"; // if (overhead > kOverheadRatio * kNumThreads * Computer0::kSleepTime + 1s) { // XGRAMMAR_LOG(WARNING) << "The overhead is too high, maybe the cache holds the lock too // long?"; // } // } // TEST(XGrammarParallelTest, CacheEviction) { // XGRAMMAR_LOG_INFO << "Testing the eviction performance of the cache (always evict)"; // constexpr auto kInsertGroup = 8; // const auto kNumThreads = int(std::thread::hardware_concurrency()) * 4; // // always evict // auto cache = ThreadSafeLRUCache{0}; // auto futures = std::vector>{}; // futures.reserve(kNumThreads); // const auto tic = std::chrono::high_resolution_clock::now(); // const auto target = tic + 1s; // for (int i = 0; i < kNumThreads; ++i) { // futures.push_back(std::async(std::launch::async, [=, &cache] { // std::this_thread::sleep_until(target); // auto sum = std::size_t{}; // // each thread writes to a different key // for (int j = 0; j < kInsertGroup; ++j) { // sum += cache.Get(i * kInsertGroup + j).uuid; // } // return sum; // })); // } // const auto kNumInsert = std::size_t(kNumThreads) * kInsertGroup; // const auto kResult = kNumInsert * (kNumInsert - 1) / 2; // auto sum = std::size_t{}; // for (int i = 0; i < kNumThreads; ++i) sum += futures[i].get(); // EXPECT_EQ(sum, kResult); // const auto toc = std::chrono::high_resolution_clock::now(); // const auto dur = std::chrono::duration_cast(toc - tic); // // remove 1s sleep time and computing sleep time // const auto overhead = dur - 1s - Computer0::kSleepTime * kInsertGroup; // XGRAMMAR_LOG_INFO << "(" << kInsertGroup << " writes) " // << "* " << kNumThreads << " threads | " // << "overhead = " << overhead.count() << "ms"; // // shouldn't exceed compute + sleep time // if (overhead > kOverheadRatio * Computer0::kSleepTime * kNumThreads + 1s) { // XGRAMMAR_LOG(WARNING) << "The overhead is too high, maybe the cache holds the lock too // long?"; // } // } // // A hook to ensure that the object will not be accessed after its destruction // struct LifeSpanHook { // private: // inline static std::unordered_set manager{}; // inline static std::mutex mutex{}; // static void unsafe_construct(const LifeSpanHook* ptr) { // // insert will return a pair of iterator and bool // EXPECT_TRUE(manager.insert(ptr).second); // } // static void unsafe_destruct(const LifeSpanHook* ptr) { // // erase will return 1 if the element is found and removed // EXPECT_TRUE(manager.erase(ptr)); // } // static void unsafe_confirm(const LifeSpanHook* ptr) { // // ensure that the object is still alive // EXPECT_TRUE(manager.find(ptr) != manager.end()); // } // public: // LifeSpanHook() { // const auto lock = std::lock_guard{mutex}; // unsafe_construct(this); // } // LifeSpanHook(const LifeSpanHook& other) { // const auto lock = std::lock_guard{mutex}; // unsafe_construct(this); // unsafe_confirm(&other); // } // LifeSpanHook& operator=(const LifeSpanHook& other) { // const auto lock = std::lock_guard{mutex}; // unsafe_confirm(this); // unsafe_confirm(&other); // return *this; // } // ~LifeSpanHook() { // const auto lock = std::lock_guard{mutex}; // unsafe_destruct(this); // } // void check() const { // const auto lock = std::lock_guard{mutex}; // unsafe_confirm(this); // } // }; // struct TestObject : LifeSpanHook { // private: // std::string name; // public: // TestObject() = default; // TestObject(std::string name) : name(std::move(name)) {} // TestObject& operator=(std::string name) { // this->check(); // this->name = std::move(name); // return *this; // } // std::string to_string() const { // this->check(); // return this->name; // } // std::size_t MemorySize() const { // this->check(); // return 1; // } // }; // struct Computer1 { // TestObject operator()(const TestObject& key) const { // std::this_thread::sleep_for(5s); // simulate a slow operation // return TestObject{key}; // } // }; // TEST(XGrammarParallelTest, CacheCorrectness) { // auto cache = ThreadSafeLRUCache{kUnlimited}; // const auto kNumThreads = int(std::thread::hardware_concurrency()) * 16; // auto futures = std::vector>{}; // futures.reserve(kNumThreads); // for (auto i = 0; i < kNumThreads; ++i) { // futures.push_back(std::async(std::launch::async, [&cache, i] { // return cache.Get(std::to_string(-i)).to_string(); // })); // } // // Wait the futures to block // std::this_thread::sleep_for(1s); // cache.Clear(); // for (auto i = 0; i < kNumThreads; ++i) { // EXPECT_EQ(futures[i].get(), std::to_string(-i)); // } // } } // namespace xgrammar-0.1.19/tests/python/000077500000000000000000000000001500705317600161075ustar00rootroot00000000000000xgrammar-0.1.19/tests/python/test_grammar_compiler.py000066400000000000000000000277151500705317600230540ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import threading import time from typing import Dict, List, Tuple import pytest from pydantic import BaseModel from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_allow_empty_rule_ids @pytest.mark.hf_token_required def test_compiled_grammar(): grammar = xgr.Grammar.builtin_json_grammar() tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() context = compiler.compile_grammar(grammar) time_end = time.monotonic_ns() print(f"Time to get compiled grammar: {(time_end - time_start) / 1e3} us") def check_matcher(matcher: xgr.GrammarMatcher): assert not matcher.is_terminated() assert not matcher._debug_accept_string('{ name: "John" }') assert matcher._debug_accept_string('{"name": "John"}') assert matcher.is_terminated() time_start = time.monotonic_ns() matcher_1 = xgr.GrammarMatcher(context, terminate_without_stop_token=True) time_end = time.monotonic_ns() print(f"Time to init matcher 1: {(time_end - time_start) / 1e3} us") check_matcher(matcher_1) time_start = time.monotonic_ns() matcher_2 = xgr.GrammarMatcher(context, terminate_without_stop_token=True) time_end = time.monotonic_ns() print(f"Time to init matcher 2: {(time_end - time_start) / 1e3} us") check_matcher(matcher_2) # Test max_threads=1 since we have a special logic to avoid using ThreadPool and mutex @pytest.mark.hf_token_required @pytest.mark.parametrize("max_threads", (8, 1)) def test_grammar_compiler_json(max_threads): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) time_start = time.monotonic_ns() grammar_compiler = xgr.GrammarCompiler(tokenizer_info, max_threads=max_threads) time_end = time.monotonic_ns() print(f"Time to init cached grammar compiler: {(time_end - time_start) / 1e3} us") def check_matcher(matcher: xgr.GrammarMatcher): assert not matcher.is_terminated() assert not matcher._debug_accept_string('{ name: "John" }') assert matcher._debug_accept_string('{"name": "John"}') assert matcher.is_terminated() time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_builtin_json_grammar() time_end = time.monotonic_ns() print(f"Time to get compiled grammar: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher) time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_builtin_json_grammar() time_end = time.monotonic_ns() print(f"Time to get compiled grammar again: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher) grammar_compiler.clear_cache() time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_builtin_json_grammar() time_end = time.monotonic_ns() print(f"Time to get compiled grammar after clear: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher) @pytest.mark.hf_token_required def test_grammar_compiler_json_schema(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) class MainModel(BaseModel): integer_field: int number_field: float boolean_field: bool any_array_field: List array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] nested_object_field: Dict[str, Dict[str, int]] instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) def check_with_fmt(any_whitespace, indent, separators, test_id): instance_str = instance.model_dump_json(indent=indent, round_trip=True) time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_json_schema( MainModel, any_whitespace=any_whitespace, indent=indent, separators=separators ) time_end = time.monotonic_ns() print(f"Time to get compiled grammar {test_id}: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) assert not matcher.is_terminated() assert matcher._debug_accept_string(instance_str) assert matcher.is_terminated() check_with_fmt(False, None, (",", ":"), "1") check_with_fmt(False, None, (",", ":"), "2") check_with_fmt(False, 2, None, "3") check_with_fmt(False, 2, (",", ": "), "4") check_with_fmt(True, None, (",", ":"), "5") check_with_fmt(True, None, (",", ":"), "6") check_with_fmt(True, 2, None, "7") check_with_fmt(True, 2, (",", ": "), "8") grammar_compiler.clear_cache() check_with_fmt(False, None, (",", ":"), "9") grammar_expected_test_get_allow_empty_rule_ids = [ ( r"""root ::= rule1 rule2 | "abc" rule1 ::= "abc" | "" rule2 ::= "def" rule3 | "" rule3 ::= "ghi" """, [0, 1, 2], ), ( r"""root ::= rule1 rule2 [a-z]* rule1 ::= "abc" | "" rule2 ::= "def" | "" """, [0, 1, 2], ), ( r"""root ::= rule1 rule3 rule1 ::= "abc" | "" rule2 ::= "def" | "" rule3 ::= rule1 rule2 """, [0, 1, 2, 3], ), ( r"""root ::= [a]* [b]* rule1 rule1 ::= [abc]* [def]* """, [0, 1], ), ] @pytest.mark.parametrize("grammar, expected", grammar_expected_test_get_allow_empty_rule_ids) def test_get_allow_empty_rule_ids(grammar: str, expected: List[int]): grammar_compiler = xgr.GrammarCompiler(xgr.TokenizerInfo([])) compiled_grammar = grammar_compiler.compile_grammar(grammar) allow_empty_rule_ids = _get_allow_empty_rule_ids(compiled_grammar) assert allow_empty_rule_ids == expected schema_instances = [ ( '{"type": "object","properties":{"username":{"type": "string"}},"required":["username"]}', '{"username":"Alice"}', ), ( '{"type": "object","properties":{"age":{"type": "integer"}},"required":["age"]}', '{"age":30}', ), ( '{"type": "object","properties":{"city":{"type": "string"}},"required":["city"]}', '{"city":"Paris"}', ), ( '{"type": "object","properties":{"isActive":{"type": "boolean"}},"required":["isActive"]}', '{"isActive":true}', ), ( '{"type": "object","properties":{"rating":{"type": "number"}},"required":["rating"]}', '{"rating":4.5}', ), ( '{"type": "object","properties":{"name":{"type": "string"}},"required":["name"]}', '{"name":"Bob"}', ), ( '{"type": "object","properties":{"quantity":{"type": "integer"}},"required":["quantity"]}', '{"quantity":10}', ), ( '{"type": "object","properties":{"color":{"type": "string"}},"required":["color"]}', '{"color":"blue"}', ), ( '{"type": "object","properties":{"temperature":{"type": "number"}},"required":["temperature"]}', '{"temperature":22.5}', ), ( '{"type": "object","properties":{"isCompleted":{"type": "boolean"}},"required":["isCompleted"]}', '{"isCompleted":false}', ), ] @pytest.mark.hf_token_required def test_grammar_compiler_json_schema_concurrent(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info) def check_matcher(matcher: xgr.GrammarMatcher, instance_str: str): assert not matcher.is_terminated() assert matcher._debug_accept_string(instance_str) assert matcher.is_terminated() num_schemas = len(schema_instances) thread_cnt = 100 threads = [] def compile_grammar(id: int, schema: str, instance_str: str): schema_id = id % num_schemas time_mid = time.monotonic_ns() print(f"Thread {id} start compile grammar {schema_id}: {(time_mid - time_start) / 1e3} us") compiled_grammar = grammar_compiler.compile_json_schema( schema, indent=None, separators=(",", ":"), strict_mode=True ) time_end = time.monotonic_ns() print(f"Thread {id} end compile grammar {schema_id}: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) check_matcher(matcher, instance_str) time_start = time.monotonic_ns() for i in range(thread_cnt): t = threading.Thread(target=compile_grammar, args=(i, *schema_instances[i % num_schemas])) threads.append(t) t.start() for t in threads: t.join() @pytest.mark.hf_token_required def test_grammar_compiler_cache_unlimited(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) def make_schema(name_str: str): return { "properties": {name_str: {"type": "string"}}, "required": [name_str], "type": "object", } MB = 1024 * 1024 # Default no limit grammar_compiler = xgr.GrammarCompiler(tokenizer_info) assert grammar_compiler.cache_limit_bytes == -1 # No limit (default, -1) assert grammar_compiler.get_cache_size_bytes() == 0 # No memory usage sum_single = 0 for i in range(10): schema = make_schema(f"name_{i}") compiled_grammar = grammar_compiler.compile_json_schema(schema, strict_mode=True) sum_single += compiled_grammar.memory_size_bytes memory_usage = grammar_compiler.get_cache_size_bytes() assert memory_usage == sum_single print(f"Cache memory usage after {i + 1} schemas: {memory_usage / MB:.3f} MB / unlimited") old_size = grammar_compiler.get_cache_size_bytes() grammar_compiler.compile_json_schema(make_schema("name_0"), strict_mode=True) assert grammar_compiler.get_cache_size_bytes() == old_size @pytest.mark.hf_token_required def test_grammar_compiler_cache_limited(): tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) def make_schema(name_str: str): return { "properties": {name_str: {"type": "string"}}, "required": [name_str], "type": "object", } MB = 1024 * 1024 # with a 2MB limit limit = int(2 * MB) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_limit_bytes=limit) assert grammar_compiler.cache_limit_bytes == limit assert grammar_compiler.get_cache_size_bytes() == 0 sum_single = 0 for i in range(10): schema = make_schema(f"name_{i}") compiled_grammar = grammar_compiler.compile_json_schema(schema, strict_mode=True) sum_single += compiled_grammar.memory_size_bytes memory_usage = grammar_compiler.get_cache_size_bytes() assert 0 <= memory_usage <= min(sum_single, limit * 2) # this is a rough estimate print( f"Cache memory usage after {i + 1} schemas: {memory_usage / MB:.3f} MB / {limit / MB:.3f} MB" ) # Test clear_cache grammar_compiler.clear_cache() assert grammar_compiler.get_cache_size_bytes() == 0 if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_matcher.py000066400000000000000000000277731500705317600226710ustar00rootroot00000000000000"""This test tests the token-based operations for the grammar matcher.""" import sys from typing import List, Optional import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import ( _get_masked_tokens_from_bitmask, _get_matcher_from_grammar_and_tokenizer_info, _is_grammar_accept_string, ) json_grammar = xgr.Grammar.builtin_json_grammar() input_accepted = ['{"name": "John"}', '{ "name" : "John" }'] @pytest.mark.parametrize("input_accepted", input_accepted) def test_accept(input_accepted: str): assert _is_grammar_accept_string(json_grammar, input_accepted) input_refused = ('{ name: "John" }', '{ "name": "John" } ') @pytest.mark.parametrize("input_refused", input_refused) def test_refuse(input_refused: str): assert not _is_grammar_accept_string(json_grammar, input_refused) tokenizer_path__input_str__expected_rejected_sizes = [ ( "meta-llama/Llama-2-7b-chat-hf", '{"id": 1,"name": "Example"}', [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ], ), ( # test for llama 3 "meta-llama/Meta-Llama-3-8B-Instruct", '{"id": 1,"name": "Example哈哈"}', [ # fmt: off 128235, 127497, 4744, 4744, 4744, 127849, 126399, 126399, 126760, 127499, 4744, 4744, 4744, 4744, 4744, 127849, 126399, 126399, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 128066, 128111, 4694, 128066, 128111, 4694, 127873, 128255, # fmt: on ], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, input_str, expected_rejected_sizes", tokenizer_path__input_str__expected_rejected_sizes, ) def test_fill_next_token_bitmask( tokenizer_path: str, input_str: str, expected_rejected_sizes: Optional[List[int]] ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = input_str.encode("utf-8") rejected_sizes = [] for i, c in enumerate(input_bytes): matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) rejected_sizes.append(len(rejected_token_ids)) if expected_rejected_sizes is not None: assert rejected_sizes[-1] == expected_rejected_sizes[i], ( rejected_sizes[-1], expected_rejected_sizes[i], ) assert matcher._debug_accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) rejected_sizes.append(len(rejected_token_ids)) if expected_rejected_sizes is not None: assert rejected_sizes[-1] == expected_rejected_sizes[-1] def test_token_operations(): """Test accepting token and finding the next token mask.""" vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) expected = [ ["{"], ['"', "}", "\n", " ", '"a":true'], ["", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], ["", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], [":", "\n", " ", ':"'], ['"', "{", "6", "\n", " "], ["}", ", ", "6", "\n", " "], [" ", "\n", '"', '"a":true'], [" ", "\n", '"', '"a":true'], ["}", ", ", "\n", " "], [""], ] result = [] for id in input_ids: matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) accepted = list(set(range(len(vocab))) - set(rejected_token_ids)) accepted_tokens = [vocab[i] for i in accepted] result.append(accepted_tokens) assert id in accepted, vocab[id] assert matcher.accept_token(id) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) accepted = list(set(range(len(vocab))) - set(rejected_token_ids)) accepted_tokens = [vocab[i] for i in accepted] result.append(accepted_tokens) assert result == expected def test_rollback(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) assert matcher.max_rollback_tokens == 5 input_ids_splitted = [input_ids[i : i + 2] for i in range(0, len(input_ids), 2)] for i_1, i_2 in input_ids_splitted: orig_result = [] token_bitmask1 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask1) orig_result.append(token_bitmask1) assert matcher.accept_token(i_1) token_bitmask2 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask2) orig_result.append(token_bitmask2) assert matcher.accept_token(i_2) matcher.rollback(2) result_after_rollback = [] new_token_bitmask1 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(new_token_bitmask1) result_after_rollback.append(new_token_bitmask1) assert matcher.accept_token(i_1) new_token_bitmask2 = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(new_token_bitmask2) result_after_rollback.append(new_token_bitmask2) assert matcher.accept_token(i_2) for l, r in zip(orig_result, result_after_rollback): torch.testing.assert_close(l, r) def test_graceful_rollback_failure(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", "6:", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", '"', ":"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) for i in input_ids: assert matcher.accept_token(i) assert not matcher.accept_token(vocab.index("6:")) # The matching should have accepted char '6' but failed to accept char ':' # A graceful revert should then occur, where char '6' is rolled back and # the state of the matcher is the same as before the failed call to accept_token for i in map(vocab.index, ['"', "abc", '"', " ", "}"]): assert matcher.accept_token(i) def test_reset(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) orig_result = [] for i in input_ids: token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) orig_result.append(token_bitmask) assert matcher.accept_token(i) matcher.reset() result_after_reset = [] for i in input_ids: token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) result_after_reset.append(token_bitmask) assert matcher.accept_token(i) for l, r in zip(orig_result, result_after_reset): torch.testing.assert_close(l, r) def test_termination(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", " }", ", ", "6", ":", "\n", " ", '"a"', ':true', # fmt: on ] input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a"', ":true", " }", ""] input_ids = [vocab.index(t) for t in input_splitted] tokenizer_info = xgr.TokenizerInfo(vocab) matcher = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info, max_rollback_tokens=5 ) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for i in input_ids: matcher.fill_next_token_bitmask(token_bitmask) assert matcher.accept_token(i) assert matcher.is_terminated() assert matcher.accept_token(0) is False with pytest.raises(RuntimeError): matcher.fill_next_token_bitmask(token_bitmask) matcher.rollback(2) assert not matcher.is_terminated() assert matcher.accept_token(input_ids[-2]) def test_get_jump_forward_string(): grammar_ebnf = r"""root ::= "abb" | "abbd" | other_rule other_rule ::= "a" sub_rule "b" sub_rule ::= "b" """ grammar = xgr.Grammar.from_ebnf(grammar_ebnf) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar) assert matcher._debug_accept_string("a") assert matcher.find_jump_forward_string() == "bb" def test_vocab_size(): vocab = [ # fmt: off "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', # fmt: on ] tokenizer_info = xgr.TokenizerInfo(vocab, vocab_size=64) matcher = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) matcher.fill_next_token_bitmask(token_bitmask) assert token_bitmask.shape == (1, 2) rejected_tokens = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert rejected_tokens == [i for i in range(64) if i != 7] tokenizer_path_override_stop_tokens = [ ("meta-llama/Llama-2-7b-chat-hf", [2]), ("meta-llama/Meta-Llama-3-8B-Instruct", [128001, 128009]), ("deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct", [100001]), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, override_stop_tokens", tokenizer_path_override_stop_tokens ) def test_override_stop_tokens(tokenizer_path: str, override_stop_tokens: List[int]): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info_1 = xgr.TokenizerInfo.from_huggingface( tokenizer, stop_token_ids=override_stop_tokens ) matcher_1 = _get_matcher_from_grammar_and_tokenizer_info(json_grammar, tokenizer_info_1) assert tokenizer_info_1.stop_token_ids == override_stop_tokens assert matcher_1.stop_token_ids == override_stop_tokens tokenizer_info_2 = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher_2 = _get_matcher_from_grammar_and_tokenizer_info( json_grammar, tokenizer_info_2, override_stop_tokens=override_stop_tokens ) assert matcher_2.stop_token_ids == override_stop_tokens if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_matcher_ebnf.py000066400000000000000000000376661500705317600236650ustar00rootroot00000000000000"""This test is adopted from test_builtin_grammar_json.py, but the grammar is parsed from a unoptimized, non-simplified EBNF string. This is to test the robustness of the grammar matcher. """ import sys import time from typing import List import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string def test_simple(): grammar_str = """root ::= rule1 rule2 rule1 ::= (rule2 | rule3) "a" rule2 ::= "b" rule3 ::= "c" """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "bab") assert not _is_grammar_accept_string(grammar, "abb") assert _is_grammar_accept_string(grammar, "cab") input_accepted_test_repetition = ( ("aaa", True), ("abcbc", True), ("bcbcbcbcbc", True), ("bcbcbcbcbcbcbcb", True), ("d", False), ("aaaa", False), ) @pytest.mark.parametrize("input, accepted", input_accepted_test_repetition) def test_repetition(input: str, accepted: bool): grammar_str = """ root ::= rule {2, 3} rule ::= ("a" | [bc] {4,}) """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, input) == accepted def test_utf8(): # Test utf8-encoded string with EBNF grammar ebnf_grammar_str = "root ::= [，]+" grammar = xgr.Grammar.from_ebnf(ebnf_grammar_str) accepted_inputs = ["，", "，，，", "，，，，，，，，，，，，，，，，，，，，，，"] for input_str in accepted_inputs: assert _is_grammar_accept_string(grammar, input_str, print_time=True) def test_custom_root_rule(): json_grammar_simple_ebnf = r""" root ::= basic_object basic_any ::= basic_string | basic_object basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\r\n] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [ \n\t]* """ grammar = xgr.Grammar.from_ebnf(json_grammar_simple_ebnf, root_rule_name="basic_string") assert _is_grammar_accept_string(grammar, r'"abc\r\n"') assert not _is_grammar_accept_string(grammar, r'{"name": "John" }') json_grammar_ebnf = r""" root ::= basic_array | basic_object basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= (([\"] basic_string_1 [\"])) basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1 escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]" basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" ws ::= [ \n\t]* """ json_grammar = xgr.Grammar.from_ebnf(json_grammar_ebnf) json_input_accepted = [ '{"name": "John"}', '{ "name" : "John" }', "{}", "[]", '{"name": "Alice", "age": 30, "city": "New York"}', '{"name": "Mike", "hobbies": ["reading", "cycling", "hiking"]}', '{"name": "Emma", "address": {"street": "Maple Street", "city": "Boston"}}', '[{"name": "David"}, {"name": "Sophia"}]', ( '{"name": "William", "age": null, "married": true, "children": ["Liam", "Olivia"],' ' "hasPets": false}' ), ( '{"name": "Olivia", "contact": {"email": "olivia@example.com", "address": ' '{"city": "Chicago", "zipcode": "60601"}}}' ), ( '{"name": "Liam", "skills": ["Java", "Python"], "experience": ' '[{"company": "CompanyA", "years": 5}, {"company": "CompanyB", "years": 3}]}' ), ( '{"person": {"name": "Ethan", "age": 40}, "education": {"degree": "Masters", ' '"university": "XYZ University"}, "work": [{"company": "ABC Corp", "position": ' '"Manager"}, {"company": "DEF Corp", "position": "Senior Manager"}]}' ), ( '{"name": "Charlotte", "details": {"personal": {"age": 35, "hobbies": ["gardening", ' '"painting"]}, "professional": {"occupation": "Engineer", "skills": ' '["CAD", "Project Management"], "projects": [{"name": "Project A", ' '"status": "Completed"}, {"name": "Project B", "status": "In Progress"}]}}}' ), ] @pytest.mark.parametrize("json_input_accepted", json_input_accepted) def test_json_accept(json_input_accepted: str): assert _is_grammar_accept_string(json_grammar, json_input_accepted) json_input_refused = ( r'{ name: "John" }', r'{ "name": "John" } ', # trailing space is not accepted r'{ "name": "John", "age": 30, }', r'{ "name": "John", "address": { "street": "123 Main St", "city": "New York" }', r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling",], }', r'{ "name": "John", "age": 30.5.7 }', r'{ "name": "John, "age": 30, "hobbies": ["reading", "traveling"] }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", { "type": "outdoor", "list": ' r'["hiking", "swimming",]}] }' ), r'{ "name": "John", "age": 30, "status": "\P\J" }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling"], "address": ' r'{ "street": "123 Main St", "city": "New York", "coordinates": { "latitude": 40.7128, ' r'"longitude": -74.0060 }}}, "work": { "company": "Acme", "position": "developer" }}' ), ) @pytest.mark.parametrize("json_input_refused", json_input_refused) def test_json_refuse(json_input_refused: str): assert not _is_grammar_accept_string(json_grammar, json_input_refused) json_input_pressure = ( # Extra long string: 1k chars ( '["Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. Praesent ' "libero. Sed cursus ante dapibus diam. Sed nisi. Nulla quis sem at nibh elementum " "imperdiet. Duis sagittis ipsum. Praesent mauris. Fusce nec tellus sed augue semper " "porta. Mauris massa. Vestibulum lacinia arcu eget nulla. Class aptent taciti sociosqu " "ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur sodales ligula " "in libero. Sed dignissim lacinia nunc. Curabitur tortor. Pellentesque nibh. Aenean quam. " "In scelerisque sem at dolor. Maecenas mattis. Sed convallis tristique sem. Proin ut " "ligula vel nunc egestas porttitor. Morbi lectus risus, iaculis vel, suscipit quis, " "luctus non, massa. Fusce ac turpis quis ligula lacinia aliquet. Mauris ipsum. Nulla " "metus metus, ullamcorper vel, tincidunt sed, euismod in, nibh. Quisque volutpat " "condimentum velit. Class aptent taciti sociosqu ad litora torquent per conubia nostra, " "per inceptos himenaeos. Nam nec ante. Sed lacinia, urna non tincidunt mattis, tortor " "neque adipiscing diam, a cursus ipsum ante quis turpis. Nulla facilisi. Ut fringilla. " "Suspendisse potenti. Nunc feugiat mi a tellus consequat imperdiet. Vestibulum sapien. " "Proin quam. Etiam ultrices. Suspendisse in justo eu magna luctus suscipit. Sed lectus. " "Integer euismod lacus luctus magna. Quisque cursus, metus vitae pharetra auctor, sem " 'massa mattis sem, at interdum magna augue eget diam."]' ), # long and complex json: 3k chars ( r"""{ "web-app": { "servlet": [ { "servlet-name": "cofaxCDS", "servlet-class": "org.cofax.cds.CDSServlet", "init-param": { "configGlossary:installationAt": "Philadelphia, PA", "configGlossary:adminEmail": "ksm@pobox.com", "configGlossary:poweredBy": "Cofax", "configGlossary:poweredByIcon": "/images/cofax.gif", "configGlossary:staticPath": "/content/static", "templateProcessorClass": "org.cofax.WysiwygTemplate", "templateLoaderClass": "org.cofax.FilesTemplateLoader", "templatePath": "templates", "templateOverridePath": "", "defaultListTemplate": "listTemplate.htm", "defaultFileTemplate": "articleTemplate.htm", "useJSP": false, "jspListTemplate": "listTemplate.jsp", "jspFileTemplate": "articleTemplate.jsp", "cachePackageTagsTrack": 200, "cachePackageTagsStore": 200, "cachePackageTagsRefresh": 60, "cacheTemplatesTrack": 100, "cacheTemplatesStore": 50, "cacheTemplatesRefresh": 15, "cachePagesTrack": 200, "cachePagesStore": 100, "cachePagesRefresh": 10, "cachePagesDirtyRead": 10, "searchEngineListTemplate": "forSearchEnginesList.htm", "searchEngineFileTemplate": "forSearchEngines.htm", "searchEngineRobotsDb": "WEB-INF/robots.db", "useDataStore": true, "dataStoreClass": "org.cofax.SqlDataStore", "redirectionClass": "org.cofax.SqlRedirection", "dataStoreName": "cofax", "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", "dataStoreUser": "sa", "dataStorePassword": "dataStoreTestQuery", "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", "dataStoreInitConns": 10, "dataStoreMaxConns": 100, "dataStoreConnUsageLimit": 100, "dataStoreLogLevel": "debug", "maxUrlLength": 500 } }, { "servlet-name": "cofaxEmail", "servlet-class": "org.cofax.cds.EmailServlet", "init-param": { "mailHost": "mail1", "mailHostOverride": "mail2" } }, { "servlet-name": "cofaxAdmin", "servlet-class": "org.cofax.cds.AdminServlet" }, { "servlet-name": "fileServlet", "servlet-class": "org.cofax.cds.FileServlet" }, { "servlet-name": "cofaxTools", "servlet-class": "org.cofax.cms.CofaxToolsServlet", "init-param": { "templatePath": "toolstemplates/", "log": 1, "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", "logMaxSize": "", "dataLog": 1, "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", "dataLogMaxSize": "", "removePageCache": "/content/admin/remove?cache=pages&id=", "removeTemplateCache": "/content/admin/remove?cache=templates&id=", "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", "lookInContext": 1, "adminGroupID": 4, "betaServer": true } } ], "servlet-mapping": { "cofaxCDS": "/", "cofaxEmail": "/cofaxutil/aemail/*", "cofaxAdmin": "/admin/*", "fileServlet": "/static/*", "cofaxTools": "/tools/*" }, "taglib": { "taglib-uri": "cofax.tld", "taglib-location": "/WEB-INF/tlds/cofax.tld" } } }""" ), ) @pytest.mark.parametrize("json_input_pressure", json_input_pressure) def test_json_pressure(json_input_pressure: str): assert _is_grammar_accept_string(json_grammar, json_input_pressure, print_time=True) tokenizer_path__input_str__expected_rejected_sizes = [ ( # short test "meta-llama/Llama-2-7b-chat-hf", '{"id": 1,"name": "Example"}', [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ], ), ( # long test "meta-llama/Llama-2-7b-chat-hf", """{ "id": 1, "na": "ex", "ac": true, "t": ["t1", "t2"], "ne": {"lv2": {"val": "dp"}, "arr": [1, 2, 3]}, "res": "res" }""", [ # fmt: off 31989, 31912, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31997, 31997, 31998, 31974, 31915, 31915, 270, 270, 31973, 31846, 31846, 31840, 262, 262, 262, 31969, 31846, 31846, 262, 262, 262, 31969, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31908, 270, 270, 270, 270, 31973, 31846, 31846, 31906, 270, 270, 270, 270, 31973, 31846, 31846, 262, 262, 262, 31968, 31970, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 31840, 31943, 31846, 31846, 31943, 31846, 31846, 31943, 31970, 31974, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 31974, 31974, 31999, # fmt: on ], ), ( # test for llama 3 "meta-llama/Meta-Llama-3-8B-Instruct", '{"id": 1,"name": "Example哈哈"}', [ # fmt: off 128235, 127497, 4744, 4744, 4744, 127849, 126399, 126399, 126760, 127499, 4744, 4744, 4744, 4744, 4744, 127849, 126399, 126399, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 128066, 128111, 4694, 128066, 128111, 4694, 127873, 128255, # fmt: on ], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, input_str, expected_rejected_sizes", tokenizer_path__input_str__expected_rejected_sizes, ) def test_fill_next_token_bitmask( tokenizer_path: str, input_str: str, expected_rejected_sizes: List[int] ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() matcher = xgr.GrammarMatcher(compiler.compile_grammar(json_grammar_ebnf)) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) device = "cuda" if torch.cuda.is_available() else "cpu" logits_gpu = torch.zeros(tokenizer_info.vocab_size, dtype=torch.float32, device=device) input_bytes = input_str.encode("utf-8") for i, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. Correctness verification rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert len(rejected_token_ids) == expected_rejected_sizes[i] # 3. apply_token_bitmask_inplace if torch.cuda.is_available(): torch.cuda.synchronize() time_start = time.monotonic_ns() xgr.apply_token_bitmask_inplace(logits_gpu, token_bitmask.to(device)) if torch.cuda.is_available(): torch.cuda.synchronize() time_end = time.monotonic_ns() print(f"Time to apply_token_bitmask_inplace: {(time_end - time_start) / 1e3} us") # 4. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher._debug_accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 5. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert len(rejected_token_ids) == expected_rejected_sizes[-1] if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_matcher_json.py000066400000000000000000000322721500705317600237100ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import time from typing import List import pytest import torch from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string json_grammar = xgr.Grammar.builtin_json_grammar() json_input_accepted = [ '{"name": "John"}', '{ "name" : "John" }', "{}", "[]", '{"name": "Alice", "age": 30, "city": "New York"}', '{"name": "Mike", "hobbies": ["reading", "cycling", "hiking"]}', '{"name": "Emma", "address": {"street": "Maple Street", "city": "Boston"}}', '[{"name": "David"}, {"name": "Sophia"}]', ( '{"name": "William", "age": null, "married": true, "children": ["Liam", "Olivia"],' ' "hasPets": false}' ), ( '{"name": "Olivia", "contact": {"email": "olivia@example.com", "address": ' '{"city": "Chicago", "zipcode": "60601"}}}' ), ( '{"name": "Liam", "skills": ["Java", "Python"], "experience": ' '[{"company": "CompanyA", "years": 5}, {"company": "CompanyB", "years": 3}]}' ), ( '{"person": {"name": "Ethan", "age": 40}, "education": {"degree": "Masters", ' '"university": "XYZ University"}, "work": [{"company": "ABC Corp", "position": ' '"Manager"}, {"company": "DEF Corp", "position": "Senior Manager"}]}' ), ( '{"name": "Charlotte", "details": {"personal": {"age": 35, "hobbies": ["gardening", ' '"painting"]}, "professional": {"occupation": "Engineer", "skills": ' '["CAD", "Project Management"], "projects": [{"name": "Project A", ' '"status": "Completed"}, {"name": "Project B", "status": "In Progress"}]}}}' ), ] @pytest.mark.parametrize("json_input_accepted", json_input_accepted) def test_json_accept(json_input_accepted: str): assert _is_grammar_accept_string(json_grammar, json_input_accepted) json_input_refused = ( r'{ name: "John" }', r'{ "name": "John" } ', # trailing space is not accepted r'{ "name": "John", "age": 30, }', r'{ "name": "John", "address": { "street": "123 Main St", "city": "New York" }', r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling",], }', r'{ "name": "John", "age": 30.5.7 }', r'{ "name": "John, "age": 30, "hobbies": ["reading", "traveling"] }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", { "type": "outdoor", "list": ' r'["hiking", "swimming",]}] }' ), r'{ "name": "John", "age": 30, "status": "\P\J" }', ( r'{ "name": "John", "age": 30, "hobbies": ["reading", "traveling"], "address": ' r'{ "street": "123 Main St", "city": "New York", "coordinates": { "latitude": 40.7128, ' r'"longitude": -74.0060 }}}, "work": { "company": "Acme", "position": "developer" }}' ), ) @pytest.mark.parametrize("json_input_refused", json_input_refused) def test_json_refuse(json_input_refused: str): assert not _is_grammar_accept_string(json_grammar, json_input_refused) json_input_pressure = ( # Extra long string: 1k chars ( '["Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer nec odio. Praesent ' "libero. Sed cursus ante dapibus diam. Sed nisi. Nulla quis sem at nibh elementum " "imperdiet. Duis sagittis ipsum. Praesent mauris. Fusce nec tellus sed augue semper " "porta. Mauris massa. Vestibulum lacinia arcu eget nulla. Class aptent taciti sociosqu " "ad litora torquent per conubia nostra, per inceptos himenaeos. Curabitur sodales ligula " "in libero. Sed dignissim lacinia nunc. Curabitur tortor. Pellentesque nibh. Aenean quam. " "In scelerisque sem at dolor. Maecenas mattis. Sed convallis tristique sem. Proin ut " "ligula vel nunc egestas porttitor. Morbi lectus risus, iaculis vel, suscipit quis, " "luctus non, massa. Fusce ac turpis quis ligula lacinia aliquet. Mauris ipsum. Nulla " "metus metus, ullamcorper vel, tincidunt sed, euismod in, nibh. Quisque volutpat " "condimentum velit. Class aptent taciti sociosqu ad litora torquent per conubia nostra, " "per inceptos himenaeos. Nam nec ante. Sed lacinia, urna non tincidunt mattis, tortor " "neque adipiscing diam, a cursus ipsum ante quis turpis. Nulla facilisi. Ut fringilla. " "Suspendisse potenti. Nunc feugiat mi a tellus consequat imperdiet. Vestibulum sapien. " "Proin quam. Etiam ultrices. Suspendisse in justo eu magna luctus suscipit. Sed lectus. " "Integer euismod lacus luctus magna. Quisque cursus, metus vitae pharetra auctor, sem " 'massa mattis sem, at interdum magna augue eget diam."]' ), # long and complex json: 3k chars ( r"""{ "web-app": { "servlet": [ { "servlet-name": "cofaxCDS", "servlet-class": "org.cofax.cds.CDSServlet", "init-param": { "configGlossary:installationAt": "Philadelphia, PA", "configGlossary:adminEmail": "ksm@pobox.com", "configGlossary:poweredBy": "Cofax", "configGlossary:poweredByIcon": "/images/cofax.gif", "configGlossary:staticPath": "/content/static", "templateProcessorClass": "org.cofax.WysiwygTemplate", "templateLoaderClass": "org.cofax.FilesTemplateLoader", "templatePath": "templates", "templateOverridePath": "", "defaultListTemplate": "listTemplate.htm", "defaultFileTemplate": "articleTemplate.htm", "useJSP": false, "jspListTemplate": "listTemplate.jsp", "jspFileTemplate": "articleTemplate.jsp", "cachePackageTagsTrack": 200, "cachePackageTagsStore": 200, "cachePackageTagsRefresh": 60, "cacheTemplatesTrack": 100, "cacheTemplatesStore": 50, "cacheTemplatesRefresh": 15, "cachePagesTrack": 200, "cachePagesStore": 100, "cachePagesRefresh": 10, "cachePagesDirtyRead": 10, "searchEngineListTemplate": "forSearchEnginesList.htm", "searchEngineFileTemplate": "forSearchEngines.htm", "searchEngineRobotsDb": "WEB-INF/robots.db", "useDataStore": true, "dataStoreClass": "org.cofax.SqlDataStore", "redirectionClass": "org.cofax.SqlRedirection", "dataStoreName": "cofax", "dataStoreDriver": "com.microsoft.jdbc.sqlserver.SQLServerDriver", "dataStoreUrl": "jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon", "dataStoreUser": "sa", "dataStorePassword": "dataStoreTestQuery", "dataStoreTestQuery": "SET NOCOUNT ON;select test='test';", "dataStoreLogFile": "/usr/local/tomcat/logs/datastore.log", "dataStoreInitConns": 10, "dataStoreMaxConns": 100, "dataStoreConnUsageLimit": 100, "dataStoreLogLevel": "debug", "maxUrlLength": 500 } }, { "servlet-name": "cofaxEmail", "servlet-class": "org.cofax.cds.EmailServlet", "init-param": { "mailHost": "mail1", "mailHostOverride": "mail2" } }, { "servlet-name": "cofaxAdmin", "servlet-class": "org.cofax.cds.AdminServlet" }, { "servlet-name": "fileServlet", "servlet-class": "org.cofax.cds.FileServlet" }, { "servlet-name": "cofaxTools", "servlet-class": "org.cofax.cms.CofaxToolsServlet", "init-param": { "templatePath": "toolstemplates/", "log": 1, "logLocation": "/usr/local/tomcat/logs/CofaxTools.log", "logMaxSize": "", "dataLog": 1, "dataLogLocation": "/usr/local/tomcat/logs/dataLog.log", "dataLogMaxSize": "", "removePageCache": "/content/admin/remove?cache=pages&id=", "removeTemplateCache": "/content/admin/remove?cache=templates&id=", "fileTransferFolder": "/usr/local/tomcat/webapps/content/fileTransferFolder", "lookInContext": 1, "adminGroupID": 4, "betaServer": true } } ], "servlet-mapping": { "cofaxCDS": "/", "cofaxEmail": "/cofaxutil/aemail/*", "cofaxAdmin": "/admin/*", "fileServlet": "/static/*", "cofaxTools": "/tools/*" }, "taglib": { "taglib-uri": "cofax.tld", "taglib-location": "/WEB-INF/tlds/cofax.tld" } } }""" ), ) @pytest.mark.parametrize("json_input_pressure", json_input_pressure) def test_json_pressure(json_input_pressure: str): assert _is_grammar_accept_string(json_grammar, json_input_pressure, print_time=True) tokenizer_path__input_str__expected_rejected_sizes = [ ( # short test "meta-llama/Llama-2-7b-chat-hf", '{"id": 1,"name": "Example"}', [ # fmt: off 31989, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 270, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 263, 263, 263, 263, 31974, 31999, # fmt: on ], ), ( # long test "meta-llama/Llama-2-7b-chat-hf", """{ "id": 1, "na": "ex", "ac": true, "t": ["t1", "t2"], "ne": {"lv2": {"val": "dp"}, "arr": [1, 2, 3]}, "res": "res" }""", [ # fmt: off 31989, 31912, 31912, 270, 270, 270, 31973, 31846, 31846, 31948, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31997, 31997, 31998, 31974, 31915, 31915, 270, 270, 31973, 31846, 31846, 31840, 262, 262, 262, 31969, 31846, 31846, 262, 262, 262, 31969, 31974, 31915, 31915, 270, 270, 270, 31973, 31846, 31846, 31908, 270, 270, 270, 270, 31973, 31846, 31846, 31906, 270, 270, 270, 270, 31973, 31846, 31846, 262, 262, 262, 31968, 31970, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 31840, 31943, 31846, 31846, 31943, 31846, 31846, 31943, 31970, 31974, 31915, 31915, 270, 270, 270, 270, 31973, 31846, 31846, 263, 263, 263, 263, 31974, 31974, 31999, # fmt: on ], ), ( # test for llama 3 "meta-llama/Meta-Llama-3-8B-Instruct", '{"id": 1,"name": "Example哈哈"}', [ # fmt: off 128235, 127497, 4744, 4744, 4744, 127849, 126399, 126399, 126760, 127499, 4744, 4744, 4744, 4744, 4744, 127849, 126399, 126399, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 4694, 128066, 128111, 4694, 128066, 128111, 4694, 127873, 128255, # fmt: on ], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, input_str, expected_rejected_sizes", tokenizer_path__input_str__expected_rejected_sizes, ) def test_fill_next_token_bitmask( tokenizer_path: str, input_str: str, expected_rejected_sizes: List[int] ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() matcher = xgr.GrammarMatcher(compiler.compile_builtin_json_grammar()) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) device = "cuda" if torch.cuda.is_available() else "cpu" logits_gpu = torch.zeros(1, tokenizer_info.vocab_size, dtype=torch.float32, device=device) input_bytes = input_str.encode("utf-8") for i, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() assert matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. Correctness verification rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert len(rejected_token_ids) == expected_rejected_sizes[i] # 3. apply_token_bitmask_inplace if torch.cuda.is_available(): torch.cuda.synchronize() time_start = time.monotonic_ns() xgr.apply_token_bitmask_inplace(logits_gpu, token_bitmask.to(device)) if torch.cuda.is_available(): torch.cuda.synchronize() time_end = time.monotonic_ns() print(f"Time to apply_token_bitmask_inplace: {(time_end - time_start) / 1e3} us") # 4. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher._debug_accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 5. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert len(rejected_token_ids) == expected_rejected_sizes[-1] if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_matcher_json_schema.py000066400000000000000000000340211500705317600252220ustar00rootroot00000000000000import json import sys import time from typing import Dict, List, Tuple import pytest from pydantic import BaseModel, Field from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import ( _get_masked_tokens_from_bitmask, _get_matcher_from_grammar_and_tokenizer_info, ) class MainModel(BaseModel): integer_field: int number_field: float boolean_field: bool any_array_field: List array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] nested_object_field: Dict[str, Dict[str, int]] instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) instance_str = instance.model_dump_json(indent=2, round_trip=True) @pytest.mark.hf_token_required def test_json_schema_debug_accept_string(): grammar = xgr.Grammar.from_json_schema(MainModel, indent=2) instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) instance_str = instance.model_dump_json(indent=2, round_trip=True) tokenizer_path = "meta-llama/Llama-2-7b-chat-hf" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, tokenizer_info) for c in instance_str: assert matcher._debug_accept_string(c) assert matcher.accept_token(2) assert matcher.is_terminated() def test_json_schema_find_jump_forward_string(): grammar = xgr.Grammar.from_json_schema(MainModel, indent=2) matcher = _get_matcher_from_grammar_and_tokenizer_info(grammar, xgr.TokenizerInfo([])) for i, c in enumerate(instance_str): jump_forward_str = matcher.find_jump_forward_string() assert instance_str[i : i + len(jump_forward_str)] == jump_forward_str assert matcher._debug_accept_string(c) assert matcher.find_jump_forward_string() == "" tokenizer_path = ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Meta-Llama-3-8B-Instruct"] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_path) def test_fill_next_token_bitmask(tokenizer_path: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(MainModel, indent=2) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for _, c in enumerate(input_bytes): # 1. fill_next_token_bitmask time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") # 2. accept_string print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher._debug_accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # 3. Final correctness verification matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids class RangeSchema(BaseModel): value: int = Field(ge=1, le=100) class ExtendedRangeSchema(BaseModel): value: int = Field(ge=-128, le=256) class NegativeRangeSchema(BaseModel): value: int = Field(ge=-1000, le=-1) class LargeRangeSchema(BaseModel): value: int = Field(ge=-99999, le=99999) class FloatRangeSchema(BaseModel): value: float = Field(ge=0.0, le=1.0) class NegativeFloatRangeSchema(BaseModel): value: float = Field(ge=-10.0, le=-0.1) class ComplexFloatRangeSchema(BaseModel): value: float = Field(ge=-12345.12345, le=56789.56789) class LargeFloatRangeSchema(BaseModel): value: float = Field(ge=-1000.0, le=1000.0) class MultipleBoundariesSchema(BaseModel): small_value: int = Field(ge=-10, le=10) medium_value: int = Field(ge=-100, le=100) large_value: int = Field(ge=-1000, le=1000) class MixedTypeRangeSchema(BaseModel): int_value: int = Field(ge=-100, le=100) float_value: float = Field(ge=-10.0, le=10.0) @pytest.mark.parametrize("tokenizer_path", tokenizer_path) @pytest.mark.parametrize( "schema_class,test_value", [ # Integer test cases (RangeSchema, 42), (ExtendedRangeSchema, -128), (ExtendedRangeSchema, 0), (ExtendedRangeSchema, 256), (ExtendedRangeSchema, 14), (NegativeRangeSchema, -1000), (NegativeRangeSchema, -500), (NegativeRangeSchema, -1), (LargeRangeSchema, -99999), (LargeRangeSchema, -5678), (LargeRangeSchema, 0), (LargeRangeSchema, 5678), (LargeRangeSchema, 99999), # Float test cases (FloatRangeSchema, 0.0), (FloatRangeSchema, 0.5), (FloatRangeSchema, 1.0), (NegativeFloatRangeSchema, -10.0), (NegativeFloatRangeSchema, -5.5), (NegativeFloatRangeSchema, -0.1), (LargeFloatRangeSchema, -1000.0), (LargeFloatRangeSchema, -500.5), (LargeFloatRangeSchema, 0.0), (LargeFloatRangeSchema, 500.5), (LargeFloatRangeSchema, 1000.0), (ComplexFloatRangeSchema, (-1234.1234)), (ComplexFloatRangeSchema, (0)), (ComplexFloatRangeSchema, (5671.123456)), ], ) @pytest.mark.hf_token_required def test_fill_next_token_bitmask_intfloat_range(tokenizer_path: str, schema_class, test_value): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) instance = schema_class(value=test_value) instance_str = instance.model_dump_json() print(f"Testing {schema_class.__name__} with value {test_value}") time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(schema_class) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for c in input_bytes: time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher._debug_accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_path) def test_mixed_type_range_schema(tokenizer_path: str): """Test the MixedTypeRangeSchema with both integer and float fields""" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) test_instances = [ MixedTypeRangeSchema(int_value=-100, float_value=-10.0), MixedTypeRangeSchema(int_value=100, float_value=10.0), MixedTypeRangeSchema(int_value=0, float_value=0.0), MixedTypeRangeSchema(int_value=-50, float_value=5.5), ] for instance in test_instances: instance_str = instance.model_dump_json() print(f"Testing MixedTypeRangeSchema with values: {instance}") time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(MixedTypeRangeSchema) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for c in input_bytes: time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher._debug_accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert tokenizer.eos_token_id not in rejected_token_ids @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_path) def test_multiple_boundaries_schema(tokenizer_path: str): """Test the complex MultipleBoundariesSchema with multiple integer fields""" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) test_instances = [ MultipleBoundariesSchema( small_value=-10, medium_value=-100, large_value=-1000 ), # All lower bounds MultipleBoundariesSchema( small_value=10, medium_value=100, large_value=1000 ), # All upper bounds MultipleBoundariesSchema(small_value=0, medium_value=0, large_value=0), MultipleBoundariesSchema(small_value=-5, medium_value=50, large_value=-500), ] for instance in test_instances: instance_str = instance.model_dump_json() print(f"Testing MultipleBoundariesSchema with values: {instance}") time_start = time.monotonic_ns() compiled_grammar = compiler.compile_json_schema(MultipleBoundariesSchema) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) input_bytes = instance_str.encode("utf-8") for c in input_bytes: time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher._debug_accept_string(bytes([c])) matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) assert tokenizer.eos_token_id not in rejected_token_ids string_format_instances = [ (r"long.email-address-with-hyphens@and.subdomains.example.com", "email"), (r'"very.(),:;<>[]\".VERY.\"very@\\ \"very\".unusual"@strange.example.com', "email"), (r"128.255.000.222", "ipv4"), (r"2001:db8:3:4::192.0.2.33", "ipv6"), (r"P1Y23M456DT9H87M654S", "duration"), (r"2025-01-01T12:34:56.7+08:09", "date-time"), (r"123--abc.efgh---789-xyz.rst-uvw", "hostname"), (r"01234567-89AB-CDEF-abcd-ef0123456789", "uuid"), ( r"http://azAZ09-._~%Ff!$&'()*+,;=:@xyz:987/-/./+/*?aA0-._~%Ff!$&'()@#zZ9-._~%Aa!$&,;=:", "uri", ), ] # not frequently used string_format_instances_skipped = [ ( r"//azAZ09-._~%Ff!$&'()*+,;=:@xyz:987/-/./+/*?aA0-._~%Ff!$&'()@#zZ9-._~%Aa!$&,;=:", "uri-reference", ), (r"!#$&()*+,-./{+abc}{#def}{.ghi}{/jkl}{;mno:2468}", "uri-template"), (r"/a/bc/def/ghij/~0~1//", "json-pointer"), (r"1234/a/bc/def/ghij/~0~1//", "relative-json-pointer"), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("value, format", string_format_instances) def test_mask_generation_format(value: str, format: str): class MainModel(BaseModel): name: str = Field(json_schema_extra={"format": format}) instance = json.dumps(MainModel(name=value).model_dump(mode="json")) tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B-Instruct") tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) time_start = time.monotonic_ns() compiled_grammar = grammar_compiler.compile_json_schema(MainModel) time_end = time.monotonic_ns() print(f"Time for preprocessing: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in instance.encode("utf-8"): time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() delta_us = (time_end - time_start) / 1e3 print(f"Time for fill_next_token_bitmask: {delta_us} us before accepting char {bytes([c])}") accepted = matcher._debug_accept_string(bytes([c])) assert accepted time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time for fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_token(tokenizer.eos_token_id) assert matcher.is_terminated() if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_matcher_regex.py000066400000000000000000000123551500705317600240510ustar00rootroot00000000000000import sys import time import pytest from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string def test_simple(): regex_str = "abc" grammar = xgr.Grammar.from_regex(regex_str) assert _is_grammar_accept_string(grammar, "abc") assert not _is_grammar_accept_string(grammar, "ab") assert not _is_grammar_accept_string(grammar, "abcd") test_repetition_input_accepted_test_repetition = ( ("aaa", True), ("abcbc", True), ("bcbcbcbcbc", True), ("bcbcbcbcbcbcbcb", True), ("d", False), ("aaaa", False), ) @pytest.mark.parametrize("input, accepted", test_repetition_input_accepted_test_repetition) def test_repetition(input: str, accepted: bool): regex_str = "(a|[bc]{4,}){2,3}" grammar = xgr.Grammar.from_regex(regex_str) assert _is_grammar_accept_string(grammar, input) == accepted test_regex_accept_regex_input_accepted = [ r"abc", r"[abc]+", r"[a-z0-9]+", r"[^abc]+", r"a*b+c?", r"(abc|def)+", r"a{2,4}", r"\d+", r"\w+", r"[A-Z][a-z]*", r"[0-9]{3}-[0-9]{3}-[0-9]{4}", r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", ] @pytest.mark.parametrize("regex_input_accepted", test_regex_accept_regex_input_accepted) def test_regex_accept(regex_input_accepted: str): grammar = xgr.Grammar.from_regex(regex_input_accepted) assert grammar is not None test_regex_refuse_regex_input_refused = ( r"a{,3}", # Invalid range r"a{3,2}", # Invalid range (max < min) r"[z-a]", # Invalid range (max < min) r"a++", # Invalid repetition r"(?=a)", # Lookahead not supported r"(?!a)", # Negative lookahead not supported ) @pytest.mark.parametrize("regex_input_refused", test_regex_refuse_regex_input_refused) def test_regex_refuse(regex_input_refused: str): with pytest.raises(RuntimeError): xgr.Grammar.from_regex(regex_input_refused) test_advanced_regex_string_instance_is_accepted = [ # Basic patterns (r"abc", "abc", True), (r"abc", "def", False), # Character classes (r"[abc]+", "aabbcc", True), (r"[abc]+", "abcd", False), (r"[a-z0-9]+", "abc123", True), (r"[a-z0-9]+", "ABC", False), (r"[^abc]+", "def", True), (r"[^abc]+", "aaa", False), # Lazy character class (r"[abc]+?abc", "aabc", True), # Quantifiers (r"a*b+c?", "b", True), (r"a*b+c?", "aaabbc", True), (r"a*b+c?", "c", False), # Alternation (r"(abc|def)+", "abcdef", True), (r"(abc|def)+", "abcabc", True), (r"(abc|def)+", "ab", False), # Repetition ranges (r"a{2,4}", "aa", True), (r"a{2,4}", "aaaa", True), (r"a{2,4}", "a", False), (r"a{2,4}", "aaaaa", False), # Common patterns (r"\d+", "123", True), (r"\d+", "abc", False), (r"\w+", "abc123", True), (r"\w+", "!@#", False), (r"[A-Z][a-z]*", "Hello", True), (r"[A-Z][a-z]*", "hello", False), # Complex patterns (r"[0-9]{3}-[0-9]{3}-[0-9]{4}", "123-456-7890", True), (r"[0-9]{3}-[0-9]{3}-[0-9]{4}", "12-34-567", False), (r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}", "test@email.com", True), (r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}", "invalid.email", False), ] @pytest.mark.parametrize( "regex_string, instance, is_accepted", test_advanced_regex_string_instance_is_accepted ) def test_advanced(regex_string: str, instance: str, is_accepted: bool): grammar = xgr.Grammar.from_regex(regex_string) assert _is_grammar_accept_string(grammar, instance) == is_accepted regex_input_str_test_fill_next_token_bitmask = [ (r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}", "test@email.com"), (r"[0-9]{3}-[0-9]{3}-[0-9]{4}", "123-456-7890"), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("regex, input_str", regex_input_str_test_fill_next_token_bitmask) def test_fill_next_token_bitmask(regex: str, input_str: str): tokenizer_path = "meta-llama/Meta-Llama-3-8B-Instruct" tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) compiler = xgr.GrammarCompiler(tokenizer_info) time_start = time.monotonic_ns() compiled_grammar = compiler.compile_regex(regex) matcher = xgr.GrammarMatcher(compiled_grammar) time_end = time.monotonic_ns() print(f"Time to init GrammarMatcher: {(time_end - time_start) / 1e3} us") input_bytes = input_str.encode("utf-8") token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in input_bytes: time_start = time.monotonic_ns() assert matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") time_start = time.monotonic_ns() assert matcher._debug_accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept char {chr(c)}: {(time_end - time_start) / 1e3} us") matcher.fill_next_token_bitmask(token_bitmask) rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_matcher_structural_tag.py000066400000000000000000000332741500705317600260050ustar00rootroot00000000000000import json import sys import time from typing import List import pytest from pydantic import BaseModel from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _get_masked_tokens_from_bitmask, _is_grammar_accept_string def test_simple(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "abcd" rule2 ::= "efg" """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcd") assert _is_grammar_accept_string(grammar, "tag1abcdtag2efg") assert _is_grammar_accept_string(grammar, "tag1abcdqqqqtag2efg") assert not _is_grammar_accept_string(grammar, "tag1abc") assert not _is_grammar_accept_string(grammar, "tag1abce") assert not _is_grammar_accept_string(grammar, "ttag1abd") def test_complex_rule(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "abcd" [p]* rule2 ::= "efg" [t]* """ grammar = xgr.Grammar.from_ebnf(grammar_str) assert _is_grammar_accept_string(grammar, "tag1abcd") assert _is_grammar_accept_string(grammar, "tag1abcdppppptag2efg") assert _is_grammar_accept_string(grammar, "tag2efgtttttag1abc") def test_utf8(): # Test utf8-encoded string with structural tags class Schema(BaseModel): arg1: str arg2: int tags = [ xgr.StructuralTagItem(begin="，，", schema=Schema, end="。"), xgr.StructuralTagItem(begin="，！", schema=Schema, end="。。"), xgr.StructuralTagItem(begin="，，？", schema=Schema, end="。。。"), xgr.StructuralTagItem(begin="｜｜？", schema=Schema, end="｜？｜"), ] triggers = ["，", "｜｜"] grammar = xgr.Grammar.from_structural_tag(tags, triggers) accepted_inputs = [ '这是无用的内容，，{"arg1": "你好，世界！", "arg2": 0}。这是无用的内容', '这是无用的内容，！{"arg1": "こんにちは！", "arg2": 1}。。这是无用的内容', '这是无用的内容，，？{"arg1": "안녕하세요！", "arg2": 2}。。。这是无用的内容，！{"arg1": "안녕하세요！", "arg2": 3}。。', '这是无用的内容｜｜？{"arg1": "။စ်န, ်ပြ！", "arg2": 0}｜？｜｜｜？{"arg1": "။စ်န, ်ပြ", "arg2": 0}｜？｜', ] for input_str in accepted_inputs: assert _is_grammar_accept_string(grammar, input_str, print_time=True) def test_tag_dispatch_mask_generation_correctness(): grammar_str = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2)) rule1 ::= "abc" rule2 ::= "dg" """ tokens = [ # fmt: off "a", "b", "c", "d", "g", "t", "1", "2", "1a", "2d", "2a", "2dgt", "2dgtag1a", "2dgtag1b", "tag1a", "tag1b", "c哈哈t", "q", "abcdef" # fmt: on ] input_str = "tag1abcqqtag2dgq" expected_accepted_tokens = [ # fmt: off ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'abcdef'], ['b'], ['c哈哈t', 'c'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['d'], ['g'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'], ['a', 'b', 'c', 'd', 'g', 't', '1', '2', '1a', '2d', '2a', '2dgt', '2dgtag1a', 'tag1a', 'c哈哈t', 'q', 'abcdef'] # fmt: on ] grammar = xgr.Grammar.from_ebnf(grammar_str) tokenizer_info = xgr.TokenizerInfo(tokens) compiler = xgr.GrammarCompiler(tokenizer_info) compiled_grammar = compiler.compile_grammar(grammar) matcher = xgr.GrammarMatcher(compiled_grammar, terminate_without_stop_token=True) mask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # pad a dummy char to check the final bitmask after accepting the input string for i, c in enumerate(input_str + "0"): matcher.fill_next_token_bitmask(mask) rejected_indices = _get_masked_tokens_from_bitmask(mask, tokenizer_info.vocab_size) accepted_indices = list(set(range(tokenizer_info.vocab_size)) - set(rejected_indices)) accepted_tokens = [tokens[id] for id in accepted_indices] if i < len(input_str): assert matcher._debug_accept_string(c) assert accepted_tokens == expected_accepted_tokens[i] expected_grammar_test_structural_tag = r"""root ::= TagDispatch(("" root_1 "") | ("2>" root_2 "")) basic_escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub)) basic_string_sub ::= (("\"") | ([^\"\\\r\n] basic_string_sub) | ("\\" basic_escape basic_string_sub)) (=([ \n\t]* [,}\]:])) basic_integer ::= (("0") | (basic_integer_1 [1-9] [0-9]*)) (=([ \n\t]* "}")) basic_string ::= (("\"" basic_string_sub)) (=([ \n\t]* "," [ \n\t]* "\"arg2\"" [ \n\t]* ":" [ \n\t]* basic_integer [ \n\t]* "}")) root_1 ::= (("{" [ \n\t]* "\"arg1\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "," [ \n\t]* "\"arg2\"" [ \n\t]* ":" [ \n\t]* basic_integer [ \n\t]* "}")) basic_integer_1 ::= ("" | ("-")) (=([1-9] [0-9]*)) basic_escape_1 ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub_1)) basic_string_sub_1 ::= (("\"") | ([^\"\\\r\n] basic_string_sub_1) | ("\\" basic_escape_1 basic_string_sub_1)) (=([ \n\t]* [,}\]:])) basic_integer_2 ::= (("0") | (basic_integer_1_1 [1-9] [0-9]*)) (=([ \n\t]* "}")) basic_string_1 ::= (("\"" basic_string_sub_1)) (=([ \n\t]* "," [ \n\t]* "\"arg2\"" [ \n\t]* ":" [ \n\t]* basic_integer_2 [ \n\t]* "}")) root_2 ::= (("{" [ \n\t]* "\"arg1\"" [ \n\t]* ":" [ \n\t]* basic_string_1 [ \n\t]* "," [ \n\t]* "\"arg2\"" [ \n\t]* ":" [ \n\t]* basic_integer_2 [ \n\t]* "}")) basic_integer_1_1 ::= ("" | ("-")) (=([1-9] [0-9]*)) trigger_rule_1 ::= ((">" root_3 "")) basic_escape_2 ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) (=(basic_string_sub_2)) basic_string_sub_2 ::= (("\"") | ([^\"\\\r\n] basic_string_sub_2) | ("\\" basic_escape_2 basic_string_sub_2)) (=([ \n\t]* [,}\]:])) basic_number ::= ((basic_number_choice basic_number_3 basic_number_6)) (=([ \n\t]* "," [ \n\t]* "\"arg4\"" [ \n\t]* ":" [ \n\t]* root_prop_1 [ \n\t]* "}")) basic_string_2 ::= (("\"" basic_string_sub_2)) root_prop_1 ::= (("[" [ \n\t]* basic_string_2 root_prop_1_1 [ \n\t]* "]") | ("[" [ \n\t]* "]")) (=([ \n\t]* "}")) root_3 ::= (("{" [ \n\t]* "\"arg3\"" [ \n\t]* ":" [ \n\t]* basic_number [ \n\t]* "," [ \n\t]* "\"arg4\"" [ \n\t]* ":" [ \n\t]* root_prop_1 [ \n\t]* "}")) basic_number_1 ::= ("" | ("-")) (=([1-9] [0-9]*)) basic_number_2 ::= (([0-9] basic_number_2) | ([0-9])) basic_number_3 ::= ("" | ("." basic_number_2)) (=(basic_number_6)) basic_number_4 ::= ("" | ([+\-])) (=(basic_number_5)) basic_number_5 ::= (([0-9] basic_number_5) | ([0-9])) basic_number_6 ::= ("" | ([eE] basic_number_4 basic_number_5)) root_prop_1_1 ::= ("" | ([ \n\t]* "," [ \n\t]* basic_string_2 root_prop_1_1)) (=([ \n\t]* "]")) basic_number_choice ::= (("0") | (basic_number_1 [1-9] [0-9]*)) (=(basic_number_3 basic_number_6)) """ def test_structural_tag(): class Schema1(BaseModel): arg1: str arg2: int class Schema2(BaseModel): arg3: float arg4: List[str] tags = [ xgr.StructuralTagItem(begin="", schema=Schema1, end=""), xgr.StructuralTagItem(begin="", schema=Schema1, end=""), xgr.StructuralTagItem(begin="", schema=Schema2, end=""), ] # in real cases, we should use one trigger: "{"arg1": "abc", "arg2": 1}', '{"arg3": 1.23, "arg4": ["a", "b", "c"]}', '{"arg1": "abc", "arg2": 1}{"arg3": 1.23, "arg4": ["a", "b", "c"]}', 'hhhh{"arg3": 1.23, "arg4": ["a", "b", "c"]}haha{"arg1": "abc", "arg2": 1}123', ] for input in accepted_inputs: assert _is_grammar_accept_string(grammar, input, print_time=True) def test_structural_tag_compiler(): class Schema1(BaseModel): arg1: str arg2: int class Schema2(BaseModel): arg3: float arg4: List[str] tags = [ xgr.StructuralTagItem(begin="", schema=Schema1, end=""), xgr.StructuralTagItem(begin="", schema=Schema1, end=""), xgr.StructuralTagItem(begin="", schema=Schema2, end=""), ] # in real cases, we should use one trigger: "{"arg3": 1.23, "arg4": ["a", "b", "c"]}' 'haha{"arg1": "abc", "arg2": 1}123' ) dont_apply_mask_indices = [ # fmt: off 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 119, 120, 121, 122 # fmt: on ] input_bytes = accepted_input.encode("utf-8") # Set up token bitmask for validation token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) # Process input character by character for i, c in enumerate(input_bytes): # 1. Test token bitmask generation time_start = time.monotonic_ns() need_apply = matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert need_apply == (i not in dont_apply_mask_indices) # 2. Verify token bitmask correctness rejected_token_ids = _get_masked_tokens_from_bitmask( token_bitmask, tokenizer_info.vocab_size ) # This checking does not support non-ascii characters for now token_id_for_next_char = tokenizer.convert_tokens_to_ids(chr(c)) assert token_id_for_next_char not in rejected_token_ids # 3. Test character acceptance # print("Accepting char:", bytes([c])) time_start = time.monotonic_ns() assert matcher._debug_accept_string(bytes([c])) time_end = time.monotonic_ns() print(f"Time to accept_token: {(time_end - time_start) / 1e3} us") # Final verification - check that EOS token is allowed time_start = time.monotonic_ns() need_apply = matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() assert need_apply == (len(input_bytes) not in dont_apply_mask_indices) print(f"Time to fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") rejected_token_ids = _get_masked_tokens_from_bitmask(token_bitmask, tokenizer_info.vocab_size) assert tokenizer.eos_token_id not in rejected_token_ids if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_parser.py000066400000000000000000000511151500705317600225250ustar00rootroot00000000000000import sys import pytest import xgrammar as xgr from xgrammar.testing import GrammarFunctor, _ebnf_to_grammar_no_normalization def test_bnf_simple(): before = """root ::= b c b ::= "b" c ::= "c" """ expected = """root ::= ((b c)) b ::= (("b")) c ::= (("c")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_bnf_comment(): before = """# top comment root ::= a b # inline comment a ::= "a" b ::= "b" # bottom comment """ expected = """root ::= ((a b)) a ::= (("a")) b ::= (("b")) """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_ebnf(): before = """root ::= b c | b root b ::= "ab"* c ::= [acep-z]+ d ::= "d"? """ expected = """root ::= ((b c) | (b root)) b ::= ((b_1)) c ::= ((c_1)) d ::= ((d_1)) b_1 ::= ("" | ("ab" b_1)) c_1 ::= (([acep-z] c_1) | [acep-z]) d_1 ::= ("" | "d") """ grammar = _ebnf_to_grammar_no_normalization(before) after = str(grammar) assert after == expected def test_star_quantifier(): before = """root ::= b c d b ::= [b]* c ::= "b"* d ::= ([b] [c] [d] | ([p] [q]))* e ::= [e]* [f]* | [g]* """ expected = """root ::= ((b c d)) b ::= (([b]*)) c ::= ((c_1)) d ::= ((d_1)) e ::= (([e]* [f]*) | ([g]*)) c_1 ::= ("" | ("b" c_1)) d_1 ::= ("" | (d_1_choice d_1)) d_1_choice ::= (("bcd") | ("pq")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) after = str(grammar) assert after == expected # Here rule1 can be empty before = """root ::= [a]* [b]* rule1 rule1 ::= [abc]* [def]* """ expected = """root ::= (([a]* [b]* rule1)) rule1 ::= (([abc]* [def]*)) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) after = str(grammar) assert after == expected def test_consecutive_quantifiers(): grammar_str = """root ::= "a"{1,3}{1,3} """ with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 18: Expect element, but got character: {", ): xgr.Grammar.from_ebnf(grammar_str) grammar_str = """root ::= "a"++ """ with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 14: Expect element, but got character: +", ): xgr.Grammar.from_ebnf(grammar_str) grammar_str = """root ::= "a"?? """ with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 14: Expect element, but got character: ?", ): xgr.Grammar.from_ebnf(grammar_str) def test_repetition_range(): before = """root ::= a b c d e f g a ::= [a]{1,2} b ::= (a | "b"){1, 5} c ::= "c" {0 , 2} d ::= "d" {0,} e ::= "e" {2, } f ::= "f" {3} g ::= "g" {0} """ expected = """root ::= ((a b c d e f g)) a ::= (("a" a_1)) b ::= ((b_choice b_1)) c ::= ((c_1)) d ::= ((d_1)) e ::= (("ee" e_1)) f ::= (("fff")) g ::= (()) a_1 ::= ("" | ("a")) b_1 ::= ("" | (b_1_choice b_2)) b_2 ::= ("" | (b_2_choice b_3)) b_3 ::= ("" | (b_3_choice b_4)) b_4 ::= ("" | (a) | ("b")) c_1 ::= ("" | ("c" c_2)) c_2 ::= ("" | ("c")) d_1 ::= ("" | ("d" d_1)) e_1 ::= ("" | ("e" e_1)) b_choice ::= ((a) | ("b")) b_1_choice ::= ((a) | ("b")) b_2_choice ::= ((a) | ("b")) b_3_choice ::= ((a) | ("b")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) after = str(grammar) assert after == expected def test_lookahead_assertion(): before = """root ::= ((b c d)) b ::= (("abc" [a-z])) (=("abc")) c ::= (("a") | ("b")) (=([a-z] "b")) d ::= (("ac") | ("b" d_choice)) (=("abc")) d_choice ::= (("e") | ("d")) """ expected = """root ::= ((b c d)) b ::= (("abc" [a-z])) (=("abc")) c ::= (("a") | ("b")) (=([a-z] "b")) d ::= (("ac") | ("b" d_choice)) (=("abc")) d_choice ::= (("e") | ("d")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) after = str(grammar) assert after == expected def test_char(): before = r"""root ::= [a-z] [A-z] "\u0234" "\U00000345\xff" [-A-Z] [--] [^a] rest rest ::= [a-zA-Z0-9-] [\u0234-\U00000345] [测-试] [\--\]] rest1 rest1 ::= "\?\"\'测试あc" "👀" "" [a-a] [b-b] """ expected = r"""root ::= (([a-z] [A-z] "\u0234\u0345\xff" [\-A-Z] [\-\-] [^a] rest)) rest ::= (([a-zA-Z0-9\-] [\u0234-\u0345] [\u6d4b-\u8bd5] [\--\]] rest1)) rest1 ::= (("\?\"\'\u6d4b\u8bd5\u3042c\U0001f440ab")) """ # Disable unwrap_nesting_rules to expose the result before unwrapping. grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) after = str(grammar) assert after == expected def test_space(): before = """ root::="a" "b" ("c""d" "e") | "f" | "g" """ expected = """root ::= (("abcde") | ("f") | ("g")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_nest(): before = """root::= "a" ("b" | "c" "d") | (("e" "f")) """ expected = """root ::= (("a" root_choice) | ("ef")) root_choice ::= (("b") | ("cd")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_empty_parentheses(): before = """root ::= "a" ( ) "b" """ expected = """root ::= (("ab")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected before = """root ::= "a" rule1 rule1 ::= ( ) """ expected = """root ::= (("a" rule1)) rule1 ::= ("") """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_lookahead_assertion_analyzer(): before = r"""root ::= "a" rule1 "b" rule3 rule5 rule2 rule1 ::= "b" rule2 ::= "c" rule3 ::= "" | "d" rule3 rule4 ::= "" | "e" rule4 "f" rule5 ::= "" | "g" rule5 "h" """ expected = r"""root ::= (("a" rule1 "b" rule3 rule5 rule2)) rule1 ::= (("b")) (=("b" rule3 rule5 rule2)) rule2 ::= (("c")) rule3 ::= (("") | ("d" rule3)) (=(rule5 rule2)) rule4 ::= (("") | ("e" rule4 "f")) (=("f")) rule5 ::= (("") | ("g" rule5 "h")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.lookahead_assertion_analyzer(grammar) after = str(grammar) assert after == expected def test_lookahead_assertion_analyzer_tag_dispatch(): # tag dispatch disables lookahead assertion detection before = r"""root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3), ("tag4", rule4), ("tag5", rule5)) rule1 ::= "b" rule2 ::= "c" rule3 ::= "" | "d" rule3 rule4 ::= "" | "e" rule4 "f" rule5 ::= "" | "g" rule5 "h" """ expected = r"""root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3), ("tag4", rule4), ("tag5", rule5)) rule1 ::= (("b")) rule2 ::= (("c")) rule3 ::= (("") | ("d" rule3)) rule4 ::= (("") | ("e" rule4 "f")) rule5 ::= (("") | ("g" rule5 "h")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.lookahead_assertion_analyzer(grammar) after = str(grammar) assert after == expected def test_tag_dispatch(): before = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3)) rule1 ::= "a" rule2 ::= "b" rule3 ::= "c" """ expected = """root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3)) rule1 ::= (("a")) rule2 ::= (("b")) rule3 ::= (("c")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_flatten(): before = """root ::= or_test sequence_test nested_test empty_test or_test ::= ([a] | "b") | "de" | "" | or_test | [^a-z] sequence_test ::= [a] "a" ("b" ("c" | "d")) ("d" "e") sequence_test "" nested_test ::= ("a" ("b" ("c" "d"))) | ("a" | ("b" | "c")) | nested_rest nested_rest ::= ("a" | ("b" "c" | ("d" | "e" "f"))) | ((("g"))) empty_test ::= "d" | (("" | "" "") "" | "a" "") | ("" ("" | "")) "" "" """ expected = """root ::= ((or_test sequence_test nested_test empty_test)) or_test ::= ("" | ("a") | ("b") | ("de") | (or_test) | ([^a-z])) sequence_test ::= (("aab" sequence_test_choice "de" sequence_test)) nested_test ::= (("abcd") | ("a") | ("b") | ("c") | (nested_rest)) nested_rest ::= (("a") | ("bc") | ("d") | ("ef") | ("g")) empty_test ::= ("" | ("d") | ("a")) sequence_test_choice ::= (("c") | ("d")) """ grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.structure_normalizer(grammar) grammar = GrammarFunctor.byte_string_fuser(grammar) after = str(grammar) assert after == expected before__expected__test_rule_inliner = [ ( r"""root ::= rule1 | rule2 rule1 ::= "a" | "b" rule2 ::= "b" | "c" """, r"""root ::= (("a") | ("b") | ("b") | ("c")) rule1 ::= (("a") | ("b")) rule2 ::= (("b") | ("c")) """, ), ( r"""root ::= rule1 "a" [a-z]* | rule2 "b" "c" rule1 ::= "a" [a-z]* | "b" rule2 ::= "b" | "c" [b-c] """, r"""root ::= (("a" [a-z]* "a" [a-z]*) | ("b" "a" [a-z]*) | ("b" "b" "c") | ("c" [b-c] "b" "c")) rule1 ::= (("a" [a-z]*) | ("b")) rule2 ::= (("b") | ("c" [b-c])) """, ), ] @pytest.mark.parametrize("before, expected", before__expected__test_rule_inliner) def test_rule_inliner(before: str, expected: str): grammar = _ebnf_to_grammar_no_normalization(before) grammar = GrammarFunctor.rule_inliner(grammar) after = str(grammar) assert after == expected before__expected__test_dead_code_eliminator = [ # Test basic dead code elimination ( r"""root ::= rule1 | rule2 rule1 ::= "a" | "b" rule2 ::= "b" | "c" unused ::= "x" | "y" """, r"""root ::= ((rule1) | (rule2)) rule1 ::= (("a") | ("b")) rule2 ::= (("b") | ("c")) """, ), # Test recursive rule references ( r"""root ::= rule1 | rule2 unused1 ::= unused2 | "x" unused2 ::= unused1 | "y" rule1 ::= "a" rule2 | "b" rule2 ::= "c" rule1 | "d" """, r"""root ::= ((rule1) | (rule2)) rule1 ::= (("a" rule2) | ("b")) rule2 ::= (("c" rule1) | ("d")) """, ), # Test complex nested rules with unused branches ( r"""root ::= rule1 "x" | rule2 rule1 ::= "a" rule3 | "b" rule2 ::= "c" | "d" rule4 rule3 ::= "e" | "f" rule4 ::= "g" | "h" unused1 ::= "i" unused2 unused2 ::= "j" unused3 unused3 ::= "k" | "l" """, r"""root ::= ((rule1 "x") | (rule2)) rule1 ::= (("a" rule3) | ("b")) rule2 ::= (("c") | ("d" rule4)) rule3 ::= (("e") | ("f")) rule4 ::= (("g") | ("h")) """, ), ] @pytest.mark.parametrize("before, expected", before__expected__test_dead_code_eliminator) def test_dead_code_eliminator(before: str, expected: str): grammar = _ebnf_to_grammar_no_normalization(before) after = xgr.testing.GrammarFunctor.dead_code_eliminator(grammar) assert str(after) == expected def test_e2e_json_grammar(): before = r"""root ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace ) value_non_str ::= ( "{" [ \n\t]* members_and_embrace | "[" [ \n\t]* elements_or_embrace | "0" fraction exponent | [1-9] [0-9]* fraction exponent | "-" [0-9] fraction exponent | "-" [1-9] [0-9]* fraction exponent | "true" | "false" | "null" ) (= [ \n\t,}\]]) members_and_embrace ::= ("\"" characters_and_colon [ \n\t]* members_suffix | "}") (= [ \n\t,}\]]) members_suffix ::= ( value_non_str [ \n\t]* member_suffix_suffix | "\"" characters_and_embrace | "\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) member_suffix_suffix ::= ( "}" | "," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix ) (= [ \n\t,}\]]) elements_or_embrace ::= ( "{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]" | "[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]" | "\"" characters_item elements_rest [ \n\t]* "]" | "0" fraction exponent elements_rest [ \n\t]* "]" | [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "-" "0" fraction exponent elements_rest [ \n\t]* "]" | "-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" | "true" elements_rest [ \n\t]* "]" | "false" elements_rest [ \n\t]* "]" | "null" elements_rest [ \n\t]* "]" | "]" ) elements ::= ( "{" [ \n\t]* members_and_embrace elements_rest | "[" [ \n\t]* elements_or_embrace elements_rest | "\"" characters_item elements_rest | "0" fraction exponent elements_rest | [1-9] [0-9]* fraction exponent elements_rest | "-" [0-9] fraction exponent elements_rest | "-" [1-9] [0-9]* fraction exponent elements_rest | "true" elements_rest | "false" elements_rest | "null" elements_rest ) elements_rest ::= ( "" | [ \n\t]* "," [ \n\t]* elements ) characters_and_colon ::= ( "\"" [ \n\t]* ":" | [^"\\\x00-\x1F] characters_and_colon | "\\" escape characters_and_colon ) (=[ \n\t]* [\"{[0-9tfn-]) characters_and_comma ::= ( "\"" [ \n\t]* "," | [^"\\\x00-\x1F] characters_and_comma | "\\" escape characters_and_comma ) (=[ \n\t]* "\"") characters_and_embrace ::= ( "\"" [ \n\t]* "}" | [^"\\\x00-\x1F] characters_and_embrace | "\\" escape characters_and_embrace ) (=[ \n\t]* [},]) characters_item ::= ( "\"" | [^"\\\x00-\x1F] characters_item | "\\" escape characters_item ) (= [ \n\t]* [,\]]) escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] fraction ::= "" | "." [0-9] [0-9]* exponent ::= "" | "e" sign [0-9] [0-9]* | "E" sign [0-9] [0-9]* sign ::= "" | "+" | "-" """ expected = r"""root ::= (("{" [ \n\t]* members_and_embrace) | ("[" [ \n\t]* elements_or_embrace)) value_non_str ::= (("{" [ \n\t]* members_and_embrace) | ("[" [ \n\t]* elements_or_embrace) | ("0" fraction exponent) | ([1-9] [0-9]* fraction exponent) | ("-" [0-9] fraction exponent) | ("-" [1-9] [0-9]* fraction exponent) | ("true") | ("false") | ("null")) (=([ \n\t,}\]])) members_and_embrace ::= (("\"" characters_and_colon [ \n\t]* members_suffix) | ("}")) (=([ \n\t,}\]])) members_suffix ::= ((value_non_str [ \n\t]* member_suffix_suffix) | ("\"" characters_and_embrace) | ("\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix)) (=([ \n\t,}\]])) member_suffix_suffix ::= (("}") | ("," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix)) (=([ \n\t,}\]])) elements_or_embrace ::= (("{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]") | ("[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]") | ("\"" characters_item elements_rest [ \n\t]* "]") | ("0" fraction exponent elements_rest [ \n\t]* "]") | ([1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]") | ("-0" fraction exponent elements_rest [ \n\t]* "]") | ("-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]") | ("true" elements_rest [ \n\t]* "]") | ("false" elements_rest [ \n\t]* "]") | ("null" elements_rest [ \n\t]* "]") | ("]")) elements ::= (("{" [ \n\t]* members_and_embrace elements_rest) | ("[" [ \n\t]* elements_or_embrace elements_rest) | ("\"" characters_item elements_rest) | ("0" fraction exponent elements_rest) | ([1-9] [0-9]* fraction exponent elements_rest) | ("-" [0-9] fraction exponent elements_rest) | ("-" [1-9] [0-9]* fraction exponent elements_rest) | ("true" elements_rest) | ("false" elements_rest) | ("null" elements_rest)) elements_rest ::= ("" | ([ \n\t]* "," [ \n\t]* elements)) characters_and_colon ::= (("\"" [ \n\t]* ":") | ([^\"\\\0-\x1f] characters_and_colon) | ("\\" escape characters_and_colon)) (=([ \n\t]* [\"{[0-9tfn\-])) characters_and_comma ::= (("\"" [ \n\t]* ",") | ([^\"\\\0-\x1f] characters_and_comma) | ("\\" escape characters_and_comma)) (=([ \n\t]* "\"")) characters_and_embrace ::= (("\"" [ \n\t]* "}") | ([^\"\\\0-\x1f] characters_and_embrace) | ("\\" escape characters_and_embrace)) (=([ \n\t]* [},])) characters_item ::= (("\"") | ([^\"\\\0-\x1f] characters_item) | ("\\" escape characters_item)) (=([ \n\t]* [,\]])) escape ::= (([\"\\/bfnrt]) | ("u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9])) fraction ::= ("" | ("." [0-9] [0-9]*)) exponent ::= ("" | ("e" sign [0-9] [0-9]*) | ("E" sign [0-9] [0-9]*)) sign ::= ("" | ("+") | ("-")) """ grammar = xgr.Grammar.from_ebnf(before) after = str(grammar) assert after == expected def test_e2e_to_string_roundtrip(): """Checks the printed result can be parsed, and the parsing-printing process is idempotent.""" before = r"""root ::= ((b c) | (b root)) b ::= ((b_1 d)) c ::= ((c_1)) d ::= ((d_1)) b_1 ::= ("" | ("b" b_1)) (=(d)) c_1 ::= (([acep-z] c_1) | ([acep-z])) (=("d")) d_1 ::= ("" | ("d")) """ grammar_1 = xgr.Grammar.from_ebnf(before) output_string_1 = str(grammar_1) grammar_2 = xgr.Grammar.from_ebnf(output_string_1) output_string_2 = str(grammar_2) assert before == output_string_1 assert output_string_1 == output_string_2 def test_e2e_tag_dispatch_roundtrip(): """Checks the printed result can be parsed, and the parsing-printing process is idempotent.""" before = r"""root ::= TagDispatch(("tag1", rule1), ("tag2", rule2), ("tag3", rule3)) rule1 ::= (("a")) rule2 ::= (("b")) rule3 ::= (("c")) """ grammar_1 = xgr.Grammar.from_ebnf(before) output_string_1 = str(grammar_1) grammar_2 = xgr.Grammar.from_ebnf(output_string_1) output_string_2 = str(grammar_2) assert before == output_string_1 assert output_string_1 == output_string_2 def test_error(): with pytest.raises( RuntimeError, match='EBNF parse error at line 1, column 11: Rule "a" is not defined' ): xgr.Grammar.from_ebnf("root ::= a b") with pytest.raises(RuntimeError, match="EBNF parse error at line 1, column 15: Expect element"): xgr.Grammar.from_ebnf('root ::= "a" |') with pytest.raises(RuntimeError, match='EBNF parse error at line 1, column 15: Expect "'): xgr.Grammar.from_ebnf('root ::= "a" "') with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 1: Expect rule name" ): xgr.Grammar.from_ebnf('::= "a"') with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 12: Character class should not contain newline", ): xgr.Grammar.from_ebnf("root ::= [a\n]") with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 11: Invalid escape sequence" ): xgr.Grammar.from_ebnf(r'root ::= "\@"') with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 11: Invalid escape sequence" ): xgr.Grammar.from_ebnf(r'root ::= "\uFF"') with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 14: Invalid character class: " "lower bound is larger than upper bound", ): xgr.Grammar.from_ebnf(r"root ::= [Z-A]") with pytest.raises(RuntimeError, match="EBNF parse error at line 1, column 6: Expect ::="): xgr.Grammar.from_ebnf(r'root := "a"') with pytest.raises( RuntimeError, match='EBNF parse error at line 2, column 9: Rule "root" is defined multiple times', ): xgr.Grammar.from_ebnf('root ::= "a"\nroot ::= "b"') with pytest.raises( RuntimeError, match='EBNF parse error at line 1, column 10: The root rule with name "root" is not found.', ): xgr.Grammar.from_ebnf('a ::= "a"') with pytest.raises( RuntimeError, match="EBNF parse error at line 1, column 21: Unexpected lookahead assertion" ): xgr.Grammar.from_ebnf('root ::= "a" (="a") (="b")') def test_error_tag_dispatch(): # Test empty tag with pytest.raises(RuntimeError): xgr.Grammar.from_ebnf( """root ::= TagDispatch(("", rule1)) rule1 ::= "a" """ ) # Test undefined rule with pytest.raises(RuntimeError): xgr.Grammar.from_ebnf( """root ::= TagDispatch(("tag1", undefined_rule)) """ ) # Test using root rule as tag target with pytest.raises(RuntimeError): xgr.Grammar.from_ebnf( """root ::= TagDispatch(("tag1", root)) """ ) # Test invalid TagDispatch syntax with pytest.raises(RuntimeError): xgr.Grammar.from_ebnf( """root ::= TagDispatch("tag1", rule1) rule1 ::= "a" """ ) with pytest.raises(RuntimeError): xgr.Grammar.from_ebnf( """root ::= TagDispatch(("tag1" rule1)) rule1 ::= "a" """ ) # Test TagDispatch in non-root rule with pytest.raises(RuntimeError): xgr.Grammar.from_ebnf( """root ::= rule1 rule1 ::= TagDispatch(("tag1", rule2)) rule2 ::= "a" """ ) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_grammar_union_concat.py000066400000000000000000000033711500705317600237110ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import pytest import xgrammar as xgr def test_grammar_union(): grammar1 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 r1 ::= "true" | "" r2 ::= "false" | "" """ ) grammar2 = xgr.Grammar.from_ebnf( """root ::= "abc" | r1 r1 ::= "true" | r1 """ ) grammar3 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 | r3 r1 ::= "true" | r3 r2 ::= "false" | r3 r3 ::= "abc" | "" """ ) expected = """root ::= ((root_1) | (root_2) | (root_3)) root_1 ::= ((r1) | (r2)) r1 ::= ("" | ("true")) r2 ::= ("" | ("false")) root_2 ::= (("abc") | (r1_1)) r1_1 ::= (("true") | (r1_1)) root_3 ::= ((r1_2) | (r2_1) | (r3)) r1_2 ::= (("true") | (r3)) r2_1 ::= (("false") | (r3)) r3 ::= ("" | ("abc")) """ union_grammar = xgr.Grammar.union(grammar1, grammar2, grammar3) assert str(union_grammar) == expected def test_grammar_concat(): grammar1 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 r1 ::= "true" | "" r2 ::= "false" | "" """ ) grammar2 = xgr.Grammar.from_ebnf( """root ::= "abc" | r1 r1 ::= "true" | r1 """ ) grammar3 = xgr.Grammar.from_ebnf( """root ::= r1 | r2 | r3 r1 ::= "true" | r3 r2 ::= "false" | r3 r3 ::= "abc" | "" """ ) expected = """root ::= ((root_1 root_2 root_3)) root_1 ::= ((r1) | (r2)) r1 ::= ("" | ("true")) r2 ::= ("" | ("false")) root_2 ::= (("abc") | (r1_1)) r1_1 ::= (("true") | (r1_1)) root_3 ::= ((r1_2) | (r2_1) | (r3)) r1_2 ::= (("true") | (r3)) r2_1 ::= (("false") | (r3)) r3 ::= ("" | ("abc")) """ concat_grammar = xgr.Grammar.concat(grammar1, grammar2, grammar3) assert str(concat_grammar) == expected if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_json_schema_converter.py000066400000000000000000002015751500705317600241120ustar00rootroot00000000000000import json import sys from enum import Enum from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union import pytest from pydantic import BaseModel, Field, TypeAdapter, create_model import xgrammar as xgr from xgrammar.testing import ( _generate_float_regex, _generate_range_regex, _is_grammar_accept_string, _json_schema_to_ebnf, ) basic_json_rules_ebnf = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) basic_object ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}") | "{" [ \n\t]* "}" """ basic_json_rules_ebnf_no_space = r"""basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= (("[" "" basic_any (", " basic_any)* "" "]") | ("[" "" "]")) basic_object ::= ("{" "" basic_string ": " basic_any (", " basic_string ": " basic_any)* "" "}") | "{" "}" """ def check_schema_with_grammar( schema: Dict[str, Any], expected_grammar_ebnf: str, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, ): json_schema_ebnf = _json_schema_to_ebnf( schema, any_whitespace=any_whitespace, indent=indent, separators=separators, strict_mode=strict_mode, ) assert json_schema_ebnf == expected_grammar_ebnf def check_schema_with_instance( schema: Dict[str, Any], instance: Union[str, BaseModel, Any], is_accepted: bool = True, any_whitespace: bool = True, indent: Optional[int] = None, separators: Optional[Tuple[str, str]] = None, strict_mode: bool = True, debug_print: bool = False, ): json_schema_grammar = xgr.Grammar.from_json_schema( json.dumps(schema), any_whitespace=any_whitespace, indent=indent, separators=separators, strict_mode=strict_mode, ) # instance: pydantic model, json string, or any other object (dumped to json string) if isinstance(instance, BaseModel): instance = json.dumps( instance.model_dump(mode="json", round_trip=True), indent=indent, separators=separators ) elif not isinstance(instance, str): instance = json.dumps(instance, indent=indent, separators=separators) accepted = _is_grammar_accept_string(json_schema_grammar, instance, debug_print=debug_print) assert accepted == is_accepted def test_basic(): class MainModel(BaseModel): integer_field: int number_field: float boolean_field: bool any_array_field: List array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] nested_object_field: Dict[str, Dict[str, int]] ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_3 ::= (("[" "" basic_any (", " basic_any)* "" "]") | ("[" "" "]")) root_prop_4 ::= (("[" "" basic_string (", " basic_string)* "" "]") | ("[" "" "]")) root_prop_5_item_2 ::= (("[" "" basic_string (", " basic_string)* "" "]") | ("[" "" "]")) root_prop_5 ::= ("[" "" (basic_string ", " basic_integer ", " root_prop_5_item_2) "" "]") root_prop_6 ::= ("{" "" basic_string ": " basic_integer (", " basic_string ": " basic_integer)* "" "}") | "{" "}" root_prop_7_addl ::= ("{" "" basic_string ": " basic_integer (", " basic_string ": " basic_integer)* "" "}") | "{" "}" root_prop_7 ::= ("{" "" basic_string ": " root_prop_7_addl (", " basic_string ": " root_prop_7_addl)* "" "}") | "{" "}" root ::= "{" "" "\"integer_field\"" ": " basic_integer ", " "\"number_field\"" ": " basic_number ", " "\"boolean_field\"" ": " basic_boolean ", " "\"any_array_field\"" ": " root_prop_3 ", " "\"array_field\"" ": " root_prop_4 ", " "\"tuple_field\"" ": " root_prop_5 ", " "\"object_field\"" ": " root_prop_6 ", " "\"nested_object_field\"" ": " root_prop_7 "" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel( integer_field=42, number_field=3.14e5, boolean_field=True, any_array_field=[3.14, "foo", None, True], array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, nested_object_field={"foo": {"bar": 42}}, ) check_schema_with_instance(schema, instance, any_whitespace=False) def test_indent(): class MainModel(BaseModel): array_field: List[str] tuple_field: Tuple[str, int, List[str]] object_field: Dict[str, int] ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_0 ::= (("[" "\n " basic_string (",\n " basic_string)* "\n " "]") | ("[" "" "]")) root_prop_1_item_2 ::= (("[" "\n " basic_string (",\n " basic_string)* "\n " "]") | ("[" "" "]")) root_prop_1 ::= ("[" "\n " (basic_string ",\n " basic_integer ",\n " root_prop_1_item_2) "\n " "]") root_prop_2 ::= ("{" "\n " basic_string ": " basic_integer (",\n " basic_string ": " basic_integer)* "\n " "}") | "{" "}" root ::= "{" "\n " "\"array_field\"" ": " root_prop_0 ",\n " "\"tuple_field\"" ": " root_prop_1 ",\n " "\"object_field\"" ": " root_prop_2 "\n" "}" """ ) instance = MainModel( array_field=["foo", "bar"], tuple_field=("foo", 42, ["bar", "baz"]), object_field={"foo": 42, "bar": 43}, ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False, indent=2) check_schema_with_instance(schema, instance, any_whitespace=False, indent=2) check_schema_with_instance( schema, instance, any_whitespace=False, indent=None, separators=(",", ":") ) schema__grammar__accepted_instances__rejected_instances__test_non_strict = [ ( {"type": "array", "prefixItems": [{"type": "integer"}, {"type": "integer"}]}, basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_integer [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* basic_any)* [ \n\t]* "]") """, [[1, 2], [1, 2, 3], [1, 2, 3, "123"]], [[1]], ), ( { "type": "object", "properties": {"foo": {"type": "integer"}, "bar": {"type": "integer"}}, "required": ["foo", "bar"], }, basic_json_rules_ebnf + r"""root ::= "{" [ \n\t]* "\"foo\"" [ \n\t]* ":" [ \n\t]* basic_integer [ \n\t]* "," [ \n\t]* "\"bar\"" [ \n\t]* ":" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}" """, [{"foo": 1, "bar": 2}, {"foo": 1, "bar": 2, "baz": 3}], [{"foo": 1}], ), ] @pytest.mark.parametrize( "schema, expected_grammar, accepted_instances, rejected_instances", schema__grammar__accepted_instances__rejected_instances__test_non_strict, ) def test_non_strict( schema: Dict[str, Any], expected_grammar: str, accepted_instances: List[Any], rejected_instances: List[Any], ): check_schema_with_grammar(schema, expected_grammar, strict_mode=False) for instance in accepted_instances: check_schema_with_instance(schema, instance, is_accepted=True, strict_mode=False) for instance in rejected_instances: check_schema_with_instance(schema, instance, is_accepted=False, strict_mode=False) def test_enum_const(): class Field(Enum): FOO = "foo" BAR = "bar" class MainModel(BaseModel): bars: Literal["a"] str_values: Literal['a\n\r"'] foo: Literal["a", "b", "c"] values: Literal[1, "a", True] field: Field ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_0 ::= "\"a\"" root_prop_1 ::= "\"a\\n\\r\\\"\"" root_prop_2 ::= ("\"a\"") | ("\"b\"") | ("\"c\"") root_prop_3 ::= ("1") | ("\"a\"") | ("true") defs_Field ::= ("\"foo\"") | ("\"bar\"") root_prop_4 ::= defs_Field root ::= "{" "" "\"bars\"" ": " root_prop_0 ", " "\"str_values\"" ": " root_prop_1 ", " "\"foo\"" ": " root_prop_2 ", " "\"values\"" ": " root_prop_3 ", " "\"field\"" ": " root_prop_4 "" "}" """ ) schema = MainModel.model_json_schema() instance = MainModel(foo="a", values=1, bars="a", str_values='a\n\r"', field=Field.FOO) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) check_schema_with_instance(schema, instance, any_whitespace=False) def test_optional(): class MainModel(BaseModel): num: int = 0 opt_bool: Optional[bool] = None size: Optional[float] name: str = "" ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_prop_1 ::= basic_boolean | basic_null root_prop_2 ::= basic_number | basic_null root ::= "{" "" ("\"num\"" ": " basic_integer ", ")? ("\"opt_bool\"" ": " root_prop_1 ", ")? "\"size\"" ": " root_prop_2 (", " "\"name\"" ": " basic_string)? "" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel(num=42, opt_bool=True, size=3.14, name="foo") check_schema_with_instance(schema, instance, any_whitespace=False) instance = MainModel(size=None) check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, '{"size": null}', any_whitespace=False) check_schema_with_instance(schema, '{"size": null, "name": "foo"}', any_whitespace=False) check_schema_with_instance( schema, '{"num": 1, "size": null, "name": "foo"}', any_whitespace=False ) def test_all_optional(): class MainModel(BaseModel): size: int = 0 state: bool = False num: float = 0 ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root_part_1 ::= "" | ", " "\"num\"" ": " basic_number "" root_part_0 ::= root_part_1 | ", " "\"state\"" ": " basic_boolean root_part_1 root ::= ("{" "" (("\"size\"" ": " basic_integer root_part_0) | ("\"state\"" ": " basic_boolean root_part_1) | ("\"num\"" ": " basic_number "")) "" "}") | "{" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel(size=42, state=True, num=3.14) check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, '{"state": false}', any_whitespace=False) check_schema_with_instance(schema, '{"size": 1, "num": 1.5}', any_whitespace=False) def test_all_optional_non_strict(): class MainModel(BaseModel): size: int = 0 state: bool = False num: float = 0 ebnf_grammar_non_strict = basic_json_rules_ebnf_no_space + ( r"""root_part_2 ::= (", " basic_string ": " basic_any)* root_part_1 ::= root_part_2 | ", " "\"num\"" ": " basic_number root_part_2 root_part_0 ::= root_part_1 | ", " "\"state\"" ": " basic_boolean root_part_1 root ::= ("{" "" (("\"size\"" ": " basic_integer root_part_0) | ("\"state\"" ": " basic_boolean root_part_1) | ("\"num\"" ": " basic_number root_part_2) | basic_string ": " basic_any root_part_2) "" "}") | "{" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar( schema, ebnf_grammar_non_strict, any_whitespace=False, strict_mode=False ) check_schema_with_instance( schema, '{"size": 1, "num": 1.5, "other": false}', any_whitespace=False, strict_mode=False ) check_schema_with_instance(schema, '{"other": false}', any_whitespace=False, strict_mode=False) def test_empty(): class MainModel(BaseModel): pass ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root ::= "{" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) instance = MainModel() check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, '{"tmp": 123}', any_whitespace=False, strict_mode=False) def test_reference(): class Foo(BaseModel): count: int size: Optional[float] = None class Bar(BaseModel): apple: str = "x" banana: str = "y" class MainModel(BaseModel): foo: Foo bars: List[Bar] instance = MainModel( foo=Foo(count=42, size=3.14), bars=[Bar(apple="a", banana="b"), Bar(apple="c", banana="d")] ) ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""defs_Foo_prop_1 ::= basic_number | basic_null defs_Foo ::= "{" "" "\"count\"" ": " basic_integer (", " "\"size\"" ": " defs_Foo_prop_1)? "" "}" root_prop_0 ::= defs_Foo defs_Bar_part_0 ::= "" | ", " "\"banana\"" ": " basic_string "" defs_Bar ::= ("{" "" (("\"apple\"" ": " basic_string defs_Bar_part_0) | ("\"banana\"" ": " basic_string "")) "" "}") | "{" "}" root_prop_1_additional ::= defs_Bar root_prop_1 ::= (("[" "" root_prop_1_additional (", " root_prop_1_additional)* "" "]") | ("[" "" "]")) root ::= "{" "" "\"foo\"" ": " root_prop_0 ", " "\"bars\"" ": " root_prop_1 "" "}" """ ) schema = MainModel.model_json_schema() check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=False) check_schema_with_instance(schema, instance, any_whitespace=False) def test_reference_schema(): # Test simple reference with $defs schema = { "type": "object", "properties": {"value": {"$ref": "#/$defs/nested"}}, "required": ["value"], "$defs": { "nested": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } }, } instance = {"value": {"name": "John", "age": 30}} instance_rejected = {"value": {"name": "John"}} check_schema_with_instance(schema, instance, any_whitespace=False) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=False) # Test simple reference with definitions schema_def = { "type": "object", "properties": {"value": {"$ref": "#/definitions/nested"}}, "required": ["value"], "definitions": { "nested": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } }, } check_schema_with_instance(schema_def, instance, any_whitespace=False) check_schema_with_instance( schema_def, instance_rejected, is_accepted=False, any_whitespace=False ) # Test multi-level reference path schema_multi = { "type": "object", "properties": {"value": {"$ref": "#/$defs/level1/level2/nested"}}, "required": ["value"], "$defs": { "level1": { "level2": { "nested": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } } } }, } check_schema_with_instance(schema_multi, instance, any_whitespace=False) check_schema_with_instance( schema_multi, instance_rejected, is_accepted=False, any_whitespace=False ) # Test nested reference schema_nested = { "type": "object", "properties": {"value": {"$ref": "#/definitions/node_a"}}, "required": ["value"], "definitions": { "node_a": { "type": "object", "properties": { "name": {"type": "string"}, "child": {"$ref": "#/definitions/node_b"}, }, "required": ["name"], }, "node_b": { "type": "object", "properties": {"id": {"type": "integer"}}, "required": ["id"], }, }, } instance_nested = {"value": {"name": "first", "child": {"id": 1}}} instance_nested_rejected = {"value": {"name": "first", "child": {}}} check_schema_with_instance(schema_nested, instance_nested, any_whitespace=False) check_schema_with_instance( schema_nested, instance_nested_rejected, is_accepted=False, any_whitespace=False ) # Test schema with self-recursion through $defs schema_self_recursive = { "type": "object", "properties": {"value": {"$ref": "#/$defs/node"}}, "required": ["value"], "$defs": { "node": { "type": "object", "properties": {"id": {"type": "integer"}, "next": {"$ref": "#/$defs/node"}}, "required": ["id"], } }, } instance_self_recursive = {"value": {"id": 1, "next": {"id": 2, "next": {"id": 3}}}} instance_self_recursive_1 = {"value": {"id": 1}} instance_self_recursive_rejected = {"value": {"id": 1, "next": {"next": {"id": 3}}}} check_schema_with_instance(schema_self_recursive, instance_self_recursive, any_whitespace=False) check_schema_with_instance( schema_self_recursive, instance_self_recursive_1, any_whitespace=False ) check_schema_with_instance( schema_self_recursive, instance_self_recursive_rejected, is_accepted=False, any_whitespace=False, ) # Test schema with circular references between multiple schemas schema_circular = { "type": "object", "properties": {"value": {"$ref": "#/$defs/schema_a"}}, "required": ["value"], "$defs": { "schema_a": { "type": "object", "properties": {"name": {"type": "string"}, "next": {"$ref": "#/$defs/schema_b"}}, "required": ["name", "next"], }, "schema_b": { "type": "object", "properties": {"id": {"type": "integer"}, "child": {"$ref": "#/$defs/schema_a"}}, "required": ["id"], }, }, } instance_circular = { "value": { "name": "first", "next": {"id": 1, "child": {"name": "second", "next": {"id": 2}}}, } } instance_circular_complex = { # fmt: off "value": {"name": "root", "next": { "id": 1, "child": {"name": "level1", "next": { "id": 2, "child": {"name": "level2", "next": { "id": 3, "child": {"name": "level3", "next": { "id": 4, "child": {"name": "level4", "next": {"id": 5}} }} }} }} }} # fmt: on } instance_circular_rejected = { "value": {"name": "first", "next": {"child": {"name": "second", "next": {"id": 2}}}} } check_schema_with_instance(schema_circular, instance_circular, any_whitespace=False) check_schema_with_instance(schema_circular, instance_circular_complex, any_whitespace=False) check_schema_with_instance( schema_circular, instance_circular_rejected, is_accepted=False, any_whitespace=False ) # Test self-referential schema schema_recursive = { "type": "object", "properties": { "name": {"type": "string"}, "children": {"type": "array", "items": {"$ref": "#"}}, }, "required": ["name"], } instance_recursive = { "name": "root", "children": [{"name": "child1", "children": [{"name": "grandchild1"}]}, {"name": "child2"}], } instance_recursive_rejected = {"children": [{"name": "child1"}]} check_schema_with_instance(schema_recursive, instance_recursive, any_whitespace=False) check_schema_with_instance( schema_recursive, instance_recursive_rejected, is_accepted=False, any_whitespace=False ) def test_union(): class Cat(BaseModel): name: str color: str class Dog(BaseModel): name: str breed: str ta = TypeAdapter(Union[Cat, Dog]) model_schema = ta.json_schema() ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""defs_Cat ::= "{" "" "\"name\"" ": " basic_string ", " "\"color\"" ": " basic_string "" "}" root_case_0 ::= defs_Cat defs_Dog ::= "{" "" "\"name\"" ": " basic_string ", " "\"breed\"" ": " basic_string "" "}" root_case_1 ::= defs_Dog root ::= root_case_0 | root_case_1 """ ) check_schema_with_grammar(model_schema, ebnf_grammar, any_whitespace=False) check_schema_with_instance(model_schema, Cat(name="kitty", color="black"), any_whitespace=False) check_schema_with_instance( model_schema, Dog(name="doggy", breed="bulldog"), any_whitespace=False ) check_schema_with_instance( model_schema, '{"name": "kitty", "test": "black"}', False, any_whitespace=False ) def test_anyof_oneof(): schema = { "type": "object", "properties": {"name": {"anyOf": [{"type": "string"}, {"type": "integer"}]}}, } schema_accepted_1 = '{"name": "John"}' schema_accepted_2 = '{"name": 123}' schema_rejected = '{"name": {"a": 1}}' check_schema_with_instance(schema, schema_accepted_1, any_whitespace=False) check_schema_with_instance(schema, schema_accepted_2, any_whitespace=False) check_schema_with_instance(schema, schema_rejected, is_accepted=False, any_whitespace=False) schema = { "type": "object", "properties": {"name": {"oneOf": [{"type": "string"}, {"type": "integer"}]}}, } schema_accepted_1 = '{"name": "John"}' schema_accepted_2 = '{"name": 123}' schema_rejected = '{"name": {"a": 1}}' check_schema_with_instance(schema, schema_accepted_1, any_whitespace=False) check_schema_with_instance(schema, schema_accepted_2, any_whitespace=False) check_schema_with_instance(schema, schema_rejected, is_accepted=False, any_whitespace=False) def test_alias(): class MainModel(BaseModel): test: str = Field(..., alias="name") ebnf_grammar = basic_json_rules_ebnf_no_space + ( r"""root ::= "{" "" "\"name\"" ": " basic_string "" "}" """ ) check_schema_with_grammar(MainModel.model_json_schema(), ebnf_grammar, any_whitespace=False) instance = MainModel(name="kitty") instance_str = json.dumps(instance.model_dump(mode="json", round_trip=True, by_alias=False)) check_schema_with_instance( MainModel.model_json_schema(by_alias=False), instance_str, any_whitespace=False ) instance_str = json.dumps(instance.model_dump(mode="json", round_trip=True, by_alias=True)) check_schema_with_instance( MainModel.model_json_schema(by_alias=True), instance_str, any_whitespace=False ) # property name contains space class MainModelSpace(BaseModel): test: Literal["abc"] = Field(..., alias="name 1") ebnf_grammar_space = basic_json_rules_ebnf_no_space + ( r"""root_prop_0 ::= "\"abc\"" root ::= "{" "" "\"name 1\"" ": " root_prop_0 "" "}" """ ) check_schema_with_grammar( MainModelSpace.model_json_schema(), ebnf_grammar_space, any_whitespace=False ) instance_space = MainModelSpace(**{"name 1": "abc"}) instance_space_str = json.dumps( instance_space.model_dump(mode="json", round_trip=True, by_alias=True) ) check_schema_with_instance( MainModelSpace.model_json_schema(by_alias=True), instance_space_str, any_whitespace=False ) def test_restricted_string(): class MainModel(BaseModel): restricted_string: str = Field(..., pattern=r"[a-f]") instance = MainModel(restricted_string="a") instance_str = json.dumps(instance.model_dump(mode="json")) check_schema_with_instance(MainModel.model_json_schema(), instance_str, any_whitespace=False) check_schema_with_instance( MainModel.model_json_schema(), '{"restricted_string": "j"}', is_accepted=False, any_whitespace=False, ) def test_complex_restrictions(): class RestrictedModel(BaseModel): restricted_string: str = Field(..., pattern=r"[^\"]*") restricted_value: int = Field(..., strict=True, ge=0, lt=44) # working instance instance = RestrictedModel(restricted_string="abd", restricted_value=42) instance_str = json.dumps(instance.model_dump(mode="json")) check_schema_with_instance( RestrictedModel.model_json_schema(), instance_str, any_whitespace=False ) instance_err = RestrictedModel(restricted_string='"', restricted_value=42) instance_str = json.dumps(instance_err.model_dump(mode="json")) check_schema_with_instance( RestrictedModel.model_json_schema(), instance_str, is_accepted=False, any_whitespace=False ) check_schema_with_instance( RestrictedModel.model_json_schema(), '{"restricted_string": "j", "restricted_value": 45}', is_accepted=False, any_whitespace=False, ) def test_dynamic_model(): class MainModel(BaseModel): restricted_string: str = Field(..., pattern=r"[a-f]") additional_fields = {"restricted_string_dynamic": (str, Field(..., pattern=r"[a-x]"))} CompleteModel: Type[BaseModel] = create_model( "CompleteModel", __base__=MainModel, **additional_fields ) instance = CompleteModel(restricted_string="a", restricted_string_dynamic="j") instance_str = json.dumps(instance.model_dump(mode="json")) check_schema_with_instance( CompleteModel.model_json_schema(), instance_str, any_whitespace=False ) def test_any_whitespace(): class SimpleModel(BaseModel): value: str arr: List[int] obj: Dict[str, int] schema = SimpleModel.model_json_schema() ebnf_grammar = basic_json_rules_ebnf + ( r"""root_prop_1 ::= (("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) root_prop_2 ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer)* [ \n\t]* "}") | "{" [ \n\t]* "}" root ::= "{" [ \n\t]* "\"value\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "," [ \n\t]* "\"arr\"" [ \n\t]* ":" [ \n\t]* root_prop_1 [ \n\t]* "," [ \n\t]* "\"obj\"" [ \n\t]* ":" [ \n\t]* root_prop_2 [ \n\t]* "}" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True, strict_mode=True) ebnf_grammar = basic_json_rules_ebnf + ( r"""root_prop_1 ::= (("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) root_prop_2 ::= ("{" [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_integer)* [ \n\t]* "}") | "{" [ \n\t]* "}" root ::= "{" [ \n\t]* "\"value\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "," [ \n\t]* "\"arr\"" [ \n\t]* ":" [ \n\t]* root_prop_1 [ \n\t]* "," [ \n\t]* "\"obj\"" [ \n\t]* ":" [ \n\t]* root_prop_2 ([ \n\t]* "," [ \n\t]* basic_string [ \n\t]* ":" [ \n\t]* basic_any)* [ \n\t]* "}" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True, strict_mode=False) # Test that different whitespace variations are accepted when any_whitespace=True instances = [ '{"value": "test", "arr": [1, 2], "obj": {"a": 1}}', '{ "value" : "test", "arr": [1, 2], "obj": {"a": 1} }', '{\n "value" : "test",\n "arr" : [1, 2],\n "obj" : {"a": 1}\n}', '{\t"value"\t:\t"test",\t"arr":\t[1,\t2],\t"obj":\t{"a":\t1}\t}', ] for instance in instances: check_schema_with_instance(schema, instance, any_whitespace=True) schema__err_message__test_array_schema_error_cases = [ ({"type": "array", "prefixItems": {"type": "string"}}, "prefixItems must be an array"), ( {"type": "array", "prefixItems": ["not an object"]}, "prefixItems must be an array of objects or booleans", ), ({"type": "array", "prefixItems": [False]}, "prefixItems contains false"), ({"type": "array", "items": "not an object"}, "items must be a boolean or an object"), ( {"type": "array", "unevaluatedItems": "not an object"}, "unevaluatedItems must be a boolean or an object", ), ({"type": "array", "minItems": "not an integer"}, "minItems must be an integer"), ({"type": "array", "maxItems": -1}, "maxItems must be a non-negative integer"), ({"type": "array", "minItems": 5, "maxItems": 3}, "minItems is greater than maxItems: 5 > 3"), ( {"type": "array", "prefixItems": [{}, {}, {}], "maxItems": 2}, "maxItems is less than the number of prefixItems: 2 < 3", ), ( {"type": "array", "prefixItems": [{}, {}], "minItems": 3, "items": False}, "minItems is greater than the number of prefixItems, but additional items are not " "allowed: 3 > 2", ), ] @pytest.mark.parametrize("schema, err_message", schema__err_message__test_array_schema_error_cases) def test_array_schema_error_cases(schema: Dict[str, Any], err_message: str): with pytest.raises(Exception) as e: _json_schema_to_ebnf(schema) assert err_message in str(e.value) schema__expected_grammar__instances__test_array_schema = [ ( { "type": "array", "items": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, }, ( basic_json_rules_ebnf + r"""root_additional ::= "{" [ \n\t]* "\"name\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "," [ \n\t]* "\"age\"" [ \n\t]* ":" [ \n\t]* basic_integer [ \n\t]* "}" root ::= (("[" [ \n\t]* root_additional ([ \n\t]* "," [ \n\t]* root_additional)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) """ ), [ ([{"name": "John", "age": 30}, {"name": "Jane", "age": 25}], True), ([{"name": "John"}], False), ], ), ( { "type": "array", "prefixItems": [ { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, {"type": "integer"}, {"type": "string"}, ], "additionalItems": False, }, ( basic_json_rules_ebnf + r"""root_item_0 ::= "{" [ \n\t]* "\"name\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "," [ \n\t]* "\"age\"" [ \n\t]* ":" [ \n\t]* basic_integer [ \n\t]* "}" root ::= ("[" [ \n\t]* (root_item_0 [ \n\t]* "," [ \n\t]* basic_integer [ \n\t]* "," [ \n\t]* basic_string) [ \n\t]* "]") """ ), [ ([{"name": "John", "age": 30}, 42, "test"], True), ([{"name": "John", "age": 30}, 42], False), ([{"name": "John", "age": 30}, "test", 42], False), ([{"name": "John"}, 42, "test"], False), ], ), ( { "type": "array", "prefixItems": [ { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, {"type": "integer"}, ], "unevaluatedItems": { "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], }, }, ( basic_json_rules_ebnf + r"""root_item_0 ::= "{" [ \n\t]* "\"name\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "," [ \n\t]* "\"age\"" [ \n\t]* ":" [ \n\t]* basic_integer [ \n\t]* "}" root_additional ::= "{" [ \n\t]* "\"name\"" [ \n\t]* ":" [ \n\t]* basic_string [ \n\t]* "}" root ::= ("[" [ \n\t]* (root_item_0 [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* root_additional)* [ \n\t]* "]") """ ), [ ([{"name": "John", "age": 30}, 42, {"name": "Jane"}], True), ([{"name": "John", "age": 30}, 42], True), ([{"name": "John", "age": 30}, 42, 123], False), ([{"name": "John", "age": 30}, {"name": "Jane"}], False), ], ), ] @pytest.mark.parametrize( "schema, expected_grammar, instances", schema__expected_grammar__instances__test_array_schema ) def test_array_schema( schema: Dict[str, Any], expected_grammar: str, instances: List[Tuple[Any, bool]] ): grammar_ebnf = _json_schema_to_ebnf(schema) print(grammar_ebnf) assert grammar_ebnf == expected_grammar for instance, is_accepted in instances: check_schema_with_instance(schema, instance, is_accepted=is_accepted) schema__expected_grammar__instances__test_array_schema_min_max = [ # prefix empty, additional items not allowed ( {"type": "array", "items": False, "prefixItems": []}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* "]") """ ), [([], True), ([1], False), ([1, 2], False)], ), # prefix empty, additional items allowed, min=0 max=0 ( {"type": "array", "items": {"type": "integer"}, "minItems": 0, "maxItems": 0}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* "]") """ ), [([], True), ([1], False), ([1, 2], False)], ), # prefix empty, additional items allowed, min=0 max>0 ( {"type": "array", "items": {"type": "integer"}, "minItems": 0, "maxItems": 2}, ( basic_json_rules_ebnf + r"""root ::= (("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer)? [ \n\t]* "]") | ("[" [ \n\t]* "]")) """ ), [([], True), ([1], True), ([1, 2], True), ([1, 2, 3], False)], ), # prefix empty, additional items allowed, min>0 ( {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 3}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* basic_integer ([ \n\t]* "," [ \n\t]* basic_integer){1,2} [ \n\t]* "]") """ ), [([], False), ([1], False), ([1, 2], True), ([1, 2, 3], True), ([1, 2, 3, 4], False)], ), # prefix non-empty, additional items not allowed ( {"type": "array", "items": False, "prefixItems": [{"type": "string"}, {"type": "integer"}]}, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_string [ \n\t]* "," [ \n\t]* basic_integer) [ \n\t]* "]") """ ), [(["foo", 42], True), (["foo", 42, "bar"], False), (["foo"], False), ([42, "foo"], False)], ), # prefix non-empty, additional items allowed ( { "type": "array", "prefixItems": [{"type": "string"}, {"type": "integer"}], "items": {"type": "boolean"}, "minItems": 3, "maxItems": 4, }, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_string [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* basic_boolean){1,2} [ \n\t]* "]") """ ), [ (["foo", 42, True], True), (["foo", 42, True, False], True), (["foo", 42], False), (["foo", 42, True, False, True], False), (["foo", 42, "bar"], False), ], ), # prefix non-empty, additional items allowed, maxItems not set ( { "type": "array", "prefixItems": [{"type": "string"}, {"type": "integer"}], "items": {"type": "boolean"}, "minItems": 3, }, ( basic_json_rules_ebnf + r"""root ::= ("[" [ \n\t]* (basic_string [ \n\t]* "," [ \n\t]* basic_integer) ([ \n\t]* "," [ \n\t]* basic_boolean)+ [ \n\t]* "]") """ ), [ (["foo", 42, True], True), (["foo", 42, True, False], True), (["foo", 42, True, False, True], True), (["foo", 42], False), (["foo", 42, "bar"], False), ], ), ] @pytest.mark.parametrize( "schema, expected_grammar, instances", schema__expected_grammar__instances__test_array_schema_min_max, ) def test_array_schema_min_max( schema: Dict[str, Any], expected_grammar: str, instances: List[Tuple[Any, bool]] ): grammar_ebnf = _json_schema_to_ebnf(schema) assert grammar_ebnf == expected_grammar for instance, is_accepted in instances: check_schema_with_instance(schema, instance, is_accepted=is_accepted) def test_array_with_only_items_keyword(): schema = { "items": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } } instance_accepted = [{"name": "John", "age": 30}, {"name": "Jane", "age": 25}] instance_rejected = [{"name": "John"}] check_schema_with_instance(schema, instance_accepted, any_whitespace=False) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=False) schema_prefix_items = { "prefixItems": [ { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], }, ] } check_schema_with_instance(schema_prefix_items, instance_accepted, any_whitespace=False) check_schema_with_instance( schema_prefix_items, instance_rejected, is_accepted=False, any_whitespace=False ) schema_unevaluated_items = { "unevaluatedItems": { "type": "object", "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } } check_schema_with_instance(schema_unevaluated_items, instance_accepted, any_whitespace=False) check_schema_with_instance( schema_unevaluated_items, instance_rejected, is_accepted=False, any_whitespace=False ) def test_object_with_only_properties_keyword(): schema = { "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, "required": ["name", "age"], } instance_accepted = {"name": "John", "age": 30} instance_rejected = {"name": "John"} check_schema_with_instance(schema, instance_accepted, any_whitespace=False) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=False) schema_additional_properties = {"additionalProperties": {"type": "string"}} instance_accepted = {"name": "John"} instance_rejected = {"name": "John", "age": 30} check_schema_with_instance( schema_additional_properties, instance_accepted, any_whitespace=False ) check_schema_with_instance( schema_additional_properties, instance_rejected, is_accepted=False, any_whitespace=False ) schema_unevaluated_properties = {"unevaluatedProperties": {"type": "string"}} check_schema_with_instance( schema_unevaluated_properties, instance_accepted, any_whitespace=False ) check_schema_with_instance( schema_unevaluated_properties, instance_rejected, is_accepted=False, any_whitespace=False ) def test_generate_range_regex(): # Basic range tests assert _generate_range_regex(12, 16) == r"^((1[2-6]))$" assert _generate_range_regex(1, 10) == r"^(([1-9]|10))$" assert ( _generate_range_regex(2134, 3459) == r"^((2[2-9]\d{2}|2[2-9]\d{2}|21[4-9]\d{1}|213[5-9]|2134|3[0-3]\d{2}|3[0-3]\d{2}|34[0-4]\d{1}|345[0-8]|3459))$" ) # Negative to positive range assert _generate_range_regex(-5, 10) == r"^(-([1-5])|0|([1-9]|10))$" # Pure negative range assert _generate_range_regex(-15, -10) == r"^(-(1[0-5]))$" # Large ranges assert ( _generate_range_regex(-1999, -100) == r"^(-([1-9]\d{2}|1[0-8]\d{2}|19[0-8]\d{1}|199[0-8]|1999))$" ) assert _generate_range_regex(1, 9999) == r"^(([1-9]|[1-9]\d{1}|[1-9]\d{2}|[1-9]\d{3}))$" # Unbounded ranges (None cases) assert _generate_range_regex(None, None) == r"^-?\d+$" assert _generate_range_regex(5, None) == r"^([5-9]|[1-9]\d*)$" assert _generate_range_regex(None, 0) == r"^(-[1-9]\d*|0)$" # Medium range assert ( _generate_range_regex(78, 1278) == r"^(([8-9]\d{1}|79|78|[1-9]\d{2}|1[0-1]\d{2}|12[0-6]\d{1}|127[0-7]|1278))$" ) # Symmetric range around zero assert ( _generate_range_regex(-100, 100) == r"^(-([1-9]|[1-9]\d{1}|100)|0|([1-9]|[1-9]\d{1}|100))$" ) # Upper bound negative assert _generate_range_regex(None, -123) == r"^(-123|-1[0-1]\d{1}|-12[0-2]|-[1-9]\d{3,})$" # Additional edge cases # Single number assert _generate_range_regex(5, 5) == r"^((5))$" # Zero-inclusive ranges assert _generate_range_regex(-10, 0) == r"^(-([1-9]|10)|0)$" assert _generate_range_regex(0, 10) == r"^(0|([1-9]|10))$" instance__accepted__test_email_format = [ (r"simple@example.com", True), (r"very.common@example.com", True), (r"FirstName.LastName@EasierReading.org", True), (r"x@example.com", True), (r"long.email-address-with-hyphens@and.subdomains.example.com", True), (r"user.name+tag+sorting@example.com", True), (r"name/surname@example.com", True), (r"admin@example", True), (r"example@s.example", True), (r"\" \"@example.org", True), # (r"\"john..doe\"@example.org", True), # (r"mailhost!username@example.org", True), (r"\"very.(),:;<>[]\\\".VERY.\\\"very@\\\\ \\\"very\\\".unusual\"@strange.example.com", True), (r"user%example.com@example.org", True), (r"user-@example.org", True), (r"abc.example.com", False), (r"a@b@c@example.com", False), (r'a"b(c)d,e:f;gi[j\k]l@example.com', False), (r'just"not"right@example.com', False), (r'this is"not\allowed@example.com', False), (r"this\ still\"not\\allowed@example.com", False), (r"i.like.underscores@but_they_are_not_allowed_in_this_part", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_email_format) def test_email_format(instance: str, accepted: bool): schema = {"type": "string", "format": "email"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ ( "." [a-zA-Z0-9_!#$%&'*+/=?^`{|}~-]+ )* ) """ r"""| "\\" "\"" ( "\\" [ -~] | [ !#-[\]-~] )* "\\" "\"" ) "@" ( [A-Za-z0-9] ( [\-A-Za-z0-9]* [A-Za-z0-9] )? ) """ r"""( ( "." [A-Za-z0-9] [\-A-Za-z0-9]* [A-Za-z0-9] )* ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_date_format = [ (r"0000-01-01", True), (r"9999-12-31", True), (r"10-01-01", False), (r"2025-00-01", False), (r"2025-13-01", False), (r"2025-01-00", False), (r"2025-01-32", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_date_format) def test_date_format(instance: str, accepted: bool): schema = {"type": "string", "format": "date"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [01] ) ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_time_format = [ (r"00:00:00Z", True), (r"23:59:60Z", True), (r"12:34:56Z", True), (r"12:34:56+07:08", True), (r"12:34:56-07:08", True), (r"12:34:56.7Z", True), (r"12:34:56.7+08:09", True), (r"12:34:56.7-08:09", True), (r"00:00:00", False), (r"23:59:60", False), (r"12:34:56.7", False), (r"12:34:56.7890", False), (r"24:00:00", False), (r"00:60:00", False), (r"00:00:61", False), (r"00:00:00.", False), (r"12:34:56+07:", False), (r"12:34:56-07:", False), (r"12:34:56.7+-08:09", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_time_format) def test_time_format(instance: str, accepted: bool): schema = {"type": "string", "format": "time"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ":" ( [0-5] [0-9] | "6" "0" ) ( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_duration_format = [ (r"P0Y", True), (r"P12M", True), (r"P345D", True), (r"P6789W", True), (r"P01234D", True), (r"PT9H", True), (r"PT87M", True), (r"PT654S", True), (r"P1Y23M456D", True), (r"P23M456D", True), (r"P1Y0M456D", True), (r"P1Y23M", True), (r"PT9H87M654S", True), (r"PT87M654S", True), (r"PT9H0M654S", True), (r"PT9H87M", True), (r"P1Y23M456DT9H87M654S", True), (r"P", False), (r"PD", False), (r"P1", False), (r"PT", False), (r"P1Y456D", False), (r"PT9H654S", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_duration_format) def test_duration_format(instance: str, accepted: bool): schema = {"type": "string", "format": "duration"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" "P" ( ( [0-9]+ "D" | [0-9]+ "M" ( [0-9]+ "D" )? | [0-9]+ "Y" ( [0-9]+ "M" ( [0-9]+ "D" )? )? ) ( "T" ( [0-9]+ "S" | [0-9]+ "M" ( [0-9]+ "S" )? | [0-9]+ "H" ( [0-9]+ "M" ( [0-9]+ "S" )? )? ) )? | "T" ( [0-9]+ "S" | [0-9]+ "M" ( [0-9]+ "S" )? | [0-9]+ "H" ( [0-9]+ "M" ( [0-9]+ "S" )? )? ) | [0-9]+ "W" ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_ipv6_format = [ (r"0123:4567:890a:bced:fABC:DEF0:1234:5678", True), (r"::6666:6666:6666:6666:6666:6666", True), (r"::6666:6666:6666:6666:6666", True), (r"::6666:6666:6666:6666", True), (r"::6666:6666:6666", True), (r"::6666:6666", True), (r"::6666", True), (r"::", True), (r"8888:8888:8888:8888:8888:8888::", True), (r"8888:8888:8888:8888:8888::", True), (r"8888:8888:8888:8888::", True), (r"8888:8888:8888::", True), (r"8888:8888::", True), (r"8888::", True), (r"1111::2222", True), (r"1111:1111::2222", True), (r"1111::2222:2222", True), (r"1111:1111:1111::2222", True), (r"1111:1111::2222:2222", True), (r"1111::2222:2222:2222", True), (r"1111:1111:1111:1111::2222", True), (r"1111:1111:1111::2222:2222", True), (r"1111:1111::2222:2222:2222", True), (r"1111::2222:2222:2222:2222", True), (r"1111:1111:1111:1111:1111::2222", True), (r"1111:1111:1111:1111::2222:2222", True), (r"1111:1111:1111::2222:2222:2222", True), (r"1111:1111::2222:2222:2222:2222", True), (r"1111::2222:2222:2222:2222:2222", True), (r"1111:1111:1111:1111:1111:1111::2222", True), (r"1111:1111:1111:1111:1111::2222:2222", True), (r"1111:1111:1111:1111::2222:2222:2222", True), (r"1111:1111:1111::2222:2222:2222:2222", True), (r"1111:1111::2222:2222:2222:2222:2222", True), (r"1111::2222:2222:2222:2222:2222:2222", True), (r"2001:db8:3:4::192.0.2.33", True), (r"64:ff9b::192.0.2.33", True), (r"::ffff:0:255.255.255.255", True), (r"::111.111.222.222", True), (r":", False), (r":::", False), (r"::5555:5555:5555:5555:5555:5555:5555:5555", False), (r"5555::5555:5555:5555:5555:5555:5555:5555", False), (r"5555:5555::5555:5555:5555:5555:5555:5555", False), (r"5555:5555:5555::5555:5555:5555:5555:5555", False), (r"5555:5555:5555:5555::5555:5555:5555:5555", False), (r"5555:5555:5555:5555:5555::5555:5555:5555", False), (r"5555:5555:5555:5555:5555:5555::5555:5555", False), (r"5555:5555:5555:5555:5555:5555:5555::5555", False), (r"5555:5555:5555:5555:5555:5555:5555:5555::", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_ipv6_format) def test_ipv6_format(instance: str, accepted: bool): schema = {"type": "string", "format": "ipv6"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( [0-9a-fA-F]{1,4} ":" ){7,7} [0-9a-fA-F]{1,4} | ( [0-9a-fA-F]{1,4} ":" ){1,7} ":" """ r"""| ( [0-9a-fA-F]{1,4} ":" ){1,6} ":" [0-9a-fA-F]{1,4} | ( [0-9a-fA-F]{1,4} ":" ){1,5} ( ":" [0-9a-fA-F]{1,4} ){1,2} """ r"""| ( [0-9a-fA-F]{1,4} ":" ){1,4} ( ":" [0-9a-fA-F]{1,4} ){1,3} | ( [0-9a-fA-F]{1,4} ":" ){1,3} """ r"""( ":" [0-9a-fA-F]{1,4} ){1,4} | ( [0-9a-fA-F]{1,4} ":" ){1,2} ( ":" [0-9a-fA-F]{1,4} ){1,5} | """ r"""[0-9a-fA-F]{1,4} ":" ( ( ":" [0-9a-fA-F]{1,4} ){1,6} ) | ":" ( ( ":" [0-9a-fA-F]{1,4} ){1,7} | ":" ) """ r"""| ":" ":" ( "f" "f" "f" "f" ( ":" "0"{1,4} ){0,1} ":" ){0,1} ( ( "2" "5" [0-5] | ( "2" [0-4] """ r"""| "1"{0,1} [0-9] ){0,1} [0-9] ) "." ){3,3} ( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) """ r"""| ( [0-9a-fA-F]{1,4} ":" ){1,4} ":" ( ( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) "." ){3,3} """ r"""( "2" "5" [0-5] | ( "2" [0-4] | "1"{0,1} [0-9] ){0,1} [0-9] ) ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_ipv4_format = [ # (r"0.0.0.0", True), (r"00.00.00.00", True), (r"000.000.000.000", True), (r"255.255.255.255", True), (r"1", False), (r"1.", False), (r"1.1", False), (r"1.1.", False), (r"1.1.1", False), (r"1.1.1.", False), (r"0001.0001.0001.0001", False), (r"256.256.256.256", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_ipv4_format) def test_ipv4_format(instance: str, accepted: bool): schema = {"type": "string", "format": "ipv4"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [0-1]? [0-9]? [0-9] ) "." ){3} ( "2" "5" [0-5] | "2" [0-4] [0-9] | [0-1]? [0-9]? [0-9] ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_hostname_format = [ (r"0", True), (r"9", True), (r"a", True), (r"z", True), (r"www.github.com", True), (r"w-w-w.g-i-t-h-u-b.c-o-m", True), (r"ww-w.gi-th-ub.co-m", True), (r"w--ww.git---hub.co----m", True), (r".", False), (r"-", False), (r"-.", False), (r".-", False), (r"_", False), (r"a.", False), (r"-b", False), (r"c-", False), (r"d.-", False), (r"e-.", False), (r"-f.", False), (r"g-.h", False), (r"-i.j", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_hostname_format) def test_hostname_format(instance: str, accepted: bool): schema = {"type": "string", "format": "hostname"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( [a-z0-9] ( [a-z0-9-]* [a-z0-9] )? ) ( "." [a-z0-9] ( [a-z0-9-]* [a-z0-9] )? )* "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uuid_format = [ (r"00000000-0000-0000-0000-000000000000", True), (r"FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", True), (r"01234567-89AB-CDEF-abcd-ef0123456789", True), (r"-", False), (r"----", False), (r"AAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", False), (r"BBBBBBBB-BBB-BBBB-BBBB-BBBBBBBBBBBB", False), (r"CCCCCCCC-CCCC-CCC-CCCC-CCCCCCCCCCCC", False), (r"DDDDDDDD-DDDD-DDDD-DDD-DDDDDDDDDDDD", False), (r"EEEEEEEE-EEEE-EEEE-EEEE-EEEEEEEEEEE", False), (r"AAAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", False), (r"BBBBBBBB-BBBBB-BBBB-BBBB-BBBBBBBBBBBB", False), (r"CCCCCCCC-CCCC-CCCCC-CCCC-CCCCCCCCCCCC", False), (r"DDDDDDDD-DDDD-DDDD-DDDDD-DDDDDDDDDDDD", False), (r"EEEEEEEE-EEEE-EEEE-EEEE-EEEEEEEEEEEEE", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uuid_format) def test_uuid_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uuid"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" [0-9A-Fa-f]{8} "-" [0-9A-Fa-f]{4} "-" [0-9A-Fa-f]{4} "-" [0-9A-Fa-f]{4} "-" [0-9A-Fa-f]{12} "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uri_format = [ (r"aaa:?azAZ09-._~%Ff!$&'()*+,;=:@#azAZ09-._~%Aa!$&'()*+,;=:@", True), (r"z+.-:", True), (r"abc:", True), (r"abc:a", True), (r"abc:/", True), (r"abc:/a", True), (r"abc://", True), (r"abc://///////", True), (r"abc://azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"abc://:", True), (r"abc://:0123", True), (r"abc://azAZ09-._~%Ff!$&'()*+,;=", True), (r"xyz:/a", True), (r"xyz:/azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"aaa:?[#]", False), (r"abc://@@", False), (r"abc://::", False), (r"abc:/[]", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uri_format) def test_uri_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uri"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" [a-zA-Z] [a-zA-Z+.-]* ":" ( "/" "/" ( ( [a-zA-Z0-9_.~!$&'()*+,;=:-] | "%" """ r"""[0-9A-Fa-f] [0-9A-Fa-f] )* "@" )? ( [a-zA-Z0-9_.~!$&'()*+,;=-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* """ r"""( ":" [0-9]* )? ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* | "/"? ( """ r"""( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )+ ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] """ r"""| "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* )? ) ( "\?" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] """ r"""[0-9A-Fa-f] )* )? ( "#" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uri_reference_format = [ (r"?azAZ09-._~%Ff!$&'()*+,;=:@#azAZ09-._~%Aa!$&'()*+,;=:@", True), (r"", True), (r"a", True), (r"/", True), (r"/a", True), (r"//", True), (r"/////////", True), (r"//azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"//:", True), (r"//:0123", True), (r"//azAZ09-._~%Ff!$&'()*+,;=", True), (r"/a", True), (r"/azAZ09-._~%Ff!$&'()*+,;=:@", True), (r"?[#]", False), (r"//@@", False), (r"//::", False), (r"/[]", False), (r":", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uri_reference_format) def test_uri_reference_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uri-reference"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( "/" "/" ( ( [a-zA-Z0-9_.~!$&'()*+,;=:-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] """ r""")* "@" )? ( [a-zA-Z0-9_.~!$&'()*+,;=-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* ( ":" [0-9]* )? """ r"""( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* | "/" ( """ r"""( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )+ ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] """ r"""| "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* )? | ( [a-zA-Z0-9_.~!$&'()*+,;=@-] | "%" """ r"""[0-9A-Fa-f] [0-9A-Fa-f] )+ ( "/" ( [a-zA-Z0-9_.~!$&'()*+,;=:@-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )* )? """ r"""( "\?" ( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? ( "#" """ r"""( [a-zA-Z0-9_.~!$&'()*+,;=:@/\?-] | "%" [0-9A-Fa-f] [0-9A-Fa-f] )* )? "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_uri_template_format = [ (r"", True), (r"!#$&()*+,-./09:;=?@AZ[]_az~%Ff", True), (r"{+a}{#a}{.a}{/a}{;a}{?a}{&a}{=a}{,a}{!a}{@a}{|a}", True), (r"{%Ff}", True), (r"{i.j.k}", True), (r"{a_b_c:1234}", True), (r"{x_y_z*}", True), (r'"', False), (r"'", False), (r"%", False), (r"<", False), (r">", False), (r"\\\\", False), (r"^", False), (r"`", False), (r"{", False), (r"|", False), (r"}", False), (r"{n.}", False), (r"{m:100001}", False), (r"%1", False), (r"%Gg", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_uri_template_format) def test_uri_template_format(instance: str, accepted: bool): schema = {"type": "string", "format": "uri-template"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( ( [!#-$&(-;=\?-[\]_a-z~] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) | "{" """ r"""( [+#./;\?&=,!@|] )? ( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) ( "."? """ r"""( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) )* ( ":" [1-9] [0-9]? [0-9]? [0-9]? """ r"""| "*" )? ( "," ( [a-zA-Z0-9_] | "%" [0-9A-Fa-f] [0-9A-Fa-f] ) ( "."? ( [a-zA-Z0-9_] """ r"""| "%" [0-9A-Fa-f] [0-9A-Fa-f] ) )* ( ":" [1-9] [0-9]? [0-9]? [0-9]? | "*" )? )* "}" )* "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_json_pointer_format = [ (r"/", True), (r"//", True), (r"/a/bc/def/ghij", True), (r"/~0/~1/", True), (r"abc", False), (r"/~", False), (r"/~2", False), ] @pytest.mark.parametrize("instance, accepted", instance__accepted__test_json_pointer_format) def test_json_pointer_format(instance: str, accepted: bool): schema = {"type": "string", "format": "json-pointer"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( "/" ( [\0-.] | [0-}] | [\x7f-\U0010ffff] | "~" [01] )* )* "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) instance__accepted__test_relative_json_pointer_format = [ (r"0/", True), (r"123/a/bc/def/ghij", True), (r"45/~0/~1/", True), (r"6789#", True), (r"#", False), (r"abc", False), (r"/", False), (r"9/~2", False), ] @pytest.mark.parametrize( "instance, accepted", instance__accepted__test_relative_json_pointer_format ) def test_relative_json_pointer_format(instance: str, accepted: bool): schema = {"type": "string", "format": "relative-json-pointer"} expected_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" ( "0" | [1-9] [0-9]* ) ( "#" | ( "/" ( [\0-.] | [0-}] | [\x7f-\U0010ffff] | "~" [01] )* )* ) "\"" """ ) check_schema_with_grammar(schema, expected_grammar) check_schema_with_instance(schema, '"' + instance + '"', is_accepted=accepted) def test_min_max_length(): schema = {"type": "string", "minLength": 1, "maxLength": 10} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= "\"" [^"\\\r\n]{1,10} "\"" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = '"abcdefghij"' instance_rejected = '"abcdefghijk"' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) def test_type_array(): schema = { "type": ["integer", "string"], "minLength": 1, "maxLength": 10, "minimum": 1, "maximum": 10, } ebnf_grammar = basic_json_rules_ebnf + ( r"""root_integer ::= ( ( [1-9] | "1" "0" ) ) root_string ::= "\"" [^"\\\r\n]{1,10} "\"" root ::= root_integer | root_string """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = "1" instance_accepted_2 = '"1234567890"' instance_rejected = "11" instance_rejected_2 = '"12345678901"' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_accepted_2, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) check_schema_with_instance(schema, instance_rejected_2, is_accepted=False, any_whitespace=True) def test_type_array_empty(): schema = {"type": []} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) def test_empty_array(): schema = {"items": {"type": "string"}, "type": "array"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= (("[" [ \n\t]* basic_string ([ \n\t]* "," [ \n\t]* basic_string)* [ \n\t]* "]") | ("[" [ \n\t]* "]")) """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = "[]" instance_accepted_2 = '["a"]' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_accepted_2, any_whitespace=True) def test_empty_object(): schema = {"properties": {"name": {"type": "string"}}, "type": "object"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= ("{" [ \n\t]* (("\"name\"" [ \n\t]* ":" [ \n\t]* basic_string "")) [ \n\t]* "}") | "{" [ \n\t]* "}" """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = "{}" instance_accepted_2 = '{"name": "test"}' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_accepted_2, any_whitespace=True) def test_primitive_type_string(): schema = {"type": "string"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= basic_string """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = '"test"' instance_rejected = "123" check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) def test_primitive_type_object(): schema = {"type": "object"} ebnf_grammar = basic_json_rules_ebnf + ( r"""root ::= basic_object """ ) check_schema_with_grammar(schema, ebnf_grammar, any_whitespace=True) instance_accepted = '{"name": "test"}' instance_rejected = '"test"' check_schema_with_instance(schema, instance_accepted, any_whitespace=True) check_schema_with_instance(schema, instance_rejected, is_accepted=False, any_whitespace=True) def test_generate_float_regex(): assert _generate_float_regex(1.0, 5.0) == r"^(1|5|(([2-4]))(\.\d{1,6})?|1\.\d{1,6}|5\.\d{1,6})$" assert ( _generate_float_regex(1.5, 5.75) == r"^(1.5|5.75|(([2-4]))(\.\d{1,6})?|1\.6\d{0,5}|1\.7\d{0,5}|1\.8\d{0,5}|1\.9\d{0,5}|5\.0\d{0,5}|5\.1\d{0,5}|5\.2\d{0,5}|5\.3\d{0,5}|5\.4\d{0,5}|5\.5\d{0,5}|5\.6\d{0,5}|5\.70\d{0,4}|5\.71\d{0,4}|5\.72\d{0,4}|5\.73\d{0,4}|5\.74\d{0,4})$" ) assert ( _generate_float_regex(-3.14, 2.71828) == r"^(-3.14|2.71828|(-([1-3])|0|(1))(\.\d{1,6})?|-3\.0\d{0,5}|-3\.10\d{0,4}|-3\.11\d{0,4}|-3\.12\d{0,4}|-3\.13\d{0,4}|2\.0\d{0,5}|2\.1\d{0,5}|2\.2\d{0,5}|2\.3\d{0,5}|2\.4\d{0,5}|2\.5\d{0,5}|2\.6\d{0,5}|2\.70\d{0,4}|2\.710\d{0,3}|2\.711\d{0,3}|2\.712\d{0,3}|2\.713\d{0,3}|2\.714\d{0,3}|2\.715\d{0,3}|2\.716\d{0,3}|2\.717\d{0,3}|2\.7180\d{0,2}|2\.7181\d{0,2}|2\.71820\d{0,1}|2\.71821\d{0,1}|2\.71822\d{0,1}|2\.71823\d{0,1}|2\.71824\d{0,1}|2\.71825\d{0,1}|2\.71826\d{0,1}|2\.71827\d{0,1})$" ) assert ( _generate_float_regex(0.5, None) == r"^(0.5|0\.6\d{0,5}|0\.7\d{0,5}|0\.8\d{0,5}|0\.9\d{0,5}|([1-9]|[1-9]\d*)(\.\d{1,6})?)$" ) assert ( _generate_float_regex(None, -1.5) == r"^(-1.5|-1\.6\d{0,5}|-1\.7\d{0,5}|-1\.8\d{0,5}|-1\.9\d{0,5}|(-[3-9]|-[1-9]\d*)(\.\d{1,6})?)$" ) assert _generate_float_regex(None, None) == r"^-?\d+(\.\d{1,6})?$" assert _generate_float_regex(3.14159, 3.14159) == r"^(3.14159)$" assert _generate_float_regex(10.5, 2.5) == r"^()$" assert _generate_float_regex(5.123456, 5.123457) == r"^(5.123456|5.123457)$" assert ( _generate_float_regex(-0.000001, 0.000001) == r"^(-0.000001|0.000001|-0\.000000\d{0,0}|0\.000000\d{0,0})$" ) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_regex_converter.py000066400000000000000000000373071500705317600227330ustar00rootroot00000000000000import sys import time import pytest from transformers import AutoTokenizer import xgrammar as xgr from xgrammar.testing import _is_grammar_accept_string, _regex_to_ebnf def test_basic(): regex = "123" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "1" "2" "3" """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "123") assert not _is_grammar_accept_string(grammar_str, "1234") def test_unicode(): regex = "ww我😁" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "w" "w" "\u6211" "\U0001f601" """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, regex) regex_expected_grammar_instance = [ ( r"\^\$\.\*\+\?\\\(\)\[\]\{\}\|\/", r"""root ::= "^" "$" "." "*" "+" "\?" "\\" "(" ")" "[" "]" "{" "}" "|" "/" """, "^$.*+?\\()[]{}|/", ), ( r"\"\'\a\f\n\r\t\v\0\e", r"""root ::= "\"" "\'" "\a" "\f" "\n" "\r" "\t" "\v" "\0" "\e" """, "\"'\a\f\n\r\t\v\0\x1b", ), ( r"\u{20BB7}\u0300\x1F\cJ", r"""root ::= "\U00020bb7" "\u0300" "\x1f" "\n" """, "\U00020bb7\u0300\x1f\n", ), ( r"[\r\n\$\u0010-\u006F\]\--]+", r"""root ::= [\r\n$\x10-o\]\--]+ """, "\r\n$\u0020-", # TODO(yixin): add unicode tests ), ] @pytest.mark.parametrize("regex, expected_grammar, instance", regex_expected_grammar_instance) def test_escape(regex: str, expected_grammar: str, instance: str): grammar_str = _regex_to_ebnf(regex) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_escaped_char_class(): # TODO(yixin): add unicode tests # TODO(yixin): add tests for escaped char class nested in char class regex = r"\w\w\W\d\D\s\S" instance = "A_ 1b 0" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= [a-zA-Z0-9_] [a-zA-Z0-9_] [^a-zA-Z0-9_] [0-9] [^0-9] [\f\n\r\t\v\u0020\u00a0] [^[\f\n\r\t\v\u0020\u00a0] """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_char_class(): regex = r"[-a-zA-Z+--]+" instance = "a-+" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= [-a-zA-Z+--]+ """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_boundary(): regex = r"^abc$" instance = "abc" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "a" "b" "c" """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_disjunction(): regex = r"abc|de(f|g)" instance = "deg" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= "a" "b" "c" | "d" "e" ( "f" | "g" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_space(): regex = r" abc | df | g " instance = " df " grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= " " "a" "b" "c" " " | " " "d" "f" " " | " " "g" " " """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_quantifier(): regex = r"(a|b)?[a-z]+(abc)*" instance = "adddabcabc" instance1 = "z" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" | "b" )? [a-z]+ ( "a" "b" "c" )* """ # TODO(yixin): add tests for repetition range assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) assert _is_grammar_accept_string(grammar_str, instance1) def test_consecutive_quantifiers(): regex = "a{1,3}?{1,3}" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) regex = "a???" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) regex = "a++" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) regex = "a+?{1,3}" with pytest.raises(RuntimeError, match="Two consecutive repetition modifiers are not allowed."): _regex_to_ebnf(regex) def test_group(): regex = r"(a|b)(c|d)" instance = "ac" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" | "b" ) ( "c" | "d" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_any(): regex = r".+a.+" instance = "bbbabb" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= [\u0000-\U0010FFFF]+ "a" [\u0000-\U0010FFFF]+ """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) def test_ipv4(): regex = r"((25[0-5]|2[0-4]\d|[01]?\d\d?).)((25[0-5]|2[0-4]\d|[01]?\d\d?).)((25[0-5]|2[0-4]\d|[01]?\d\d?).)(25[0-5]|2[0-4]\d|[01]?\d\d?)" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] [0-9]? ) """ r"""[\u0000-\U0010FFFF] ) ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] """ r"""[0-9]? ) [\u0000-\U0010FFFF] ) ( ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] """ r"""[0-9]? ) [\u0000-\U0010FFFF] ) ( "2" "5" [0-5] | "2" [0-4] [0-9] | [01]? [0-9] [0-9]? ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "123.45.67.89") date_time_instances_accepted = [ ("2024-05-19T14:23:45Z", True), ("2019-11-30T08:15:27+05:30", True), ("2030-02-01T22:59:59-07:00", True), ("2021-07-04T00:00:00.123456Z", True), ("2022-12-31T23:45:12-03:00", True), ("2024-13-15T14:30:00Z", False), ("2023-02-2010:59:59Z", False), ("2021-11-05T24:00:00+05:30", False), ("2022-08-20T12:61:10-03:00", False), ] @pytest.mark.parametrize("instance, accepted", date_time_instances_accepted) def test_date_time(instance: str, accepted: bool): regex = r"^\d\d\d\d-(0[1-9]|1[0-2])-([0-2]\d|3[01])T([01]\d|2[0123]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]([01]\d|2[0123]):[0-5]\d)$" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( [0-2] [0-9] """ r"""| "3" [01] ) "T" ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ":" [0-5] [0-9] """ r"""( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) == accepted date_instances_accepted = [ ("0024-05-19", True), ("2019-11-30", True), ("2022-12-31", True), ("2024-13-15", False), ("2024-12-32", False), ] @pytest.mark.parametrize("instance, accepted", date_instances_accepted) def test_date(instance: str, accepted: bool): regex = r"^\d\d\d\d-(0[1-9]|1[0-2])-([0-2]\d|3[01])$" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= [0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" """ r"""( [0-2] [0-9] | "3" [01] ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) == accepted time_instances_accepted = [ ("14:23:45Z", True), ("08:15:27+05:30", True), ("22:59:59-07:00", True), ("00:00:00.123456Z", True), ("10:59:59ZA", False), ("24:00:00+05:30", False), ("12:15:10-03:60", False), ] @pytest.mark.parametrize("instance, accepted", time_instances_accepted) def test_time(instance: str, accepted: bool): regex = r"^([01]\d|2[0123]):[0-5]\d:[0-5]\d(\.\d+)?(Z|[+-]([01]\d|2[0123]):[0-5]\d)$" grammar_str = _regex_to_ebnf(regex) expected_grammar = ( r"""root ::= ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ":" [0-5] [0-9] """ r"""( "." [0-9]+ )? ( "Z" | [+-] ( [01] [0-9] | "2" [0123] ) ":" [0-5] [0-9] ) """ ) assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, instance) == accepted email_instances_accepted = [ ("simple@example.com", True), ("very.common@example.com", True), ("user_name+123@example.co.uk", True), ('"john.doe"@example.org', True), ("mail-host@online-shop.biz", True), ("customer/department=shipping@example.com", True), ("$A12345@example.non-profit.org", True), ('"!def!xyz%abc"@example.com', True), ("support@192.168.1.1", True), ("plainaddress", False), ("@missingusername.com", False), ("user@.com.my", False), ("user@com", False), ("user@-example.com", False), ] @pytest.mark.parametrize("instance, accepted", email_instances_accepted) def test_email(instance: str, accepted: bool): regex = ( r"""^([\w!#$%&'*+/=?^_`{|}~-]+(\.[\w!#$%&'*+/=?^_`{|}~-]+)*""" r"""|"([\w!#$%&'*+/=?^_`{|}~\-(),:;<>@[\].]|\\")+")@(([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+""" r"""[a-z0-9]([a-z0-9-]*[a-z0-9])?)$""" ) grammar_str = _regex_to_ebnf(regex) assert _is_grammar_accept_string(grammar_str, instance) == accepted def test_empty_character_class(): regex = "[]" with pytest.raises(RuntimeError, match="Empty character class is not allowed in regex."): _regex_to_ebnf(regex) def test_group_modifiers(): # Test non-capturing group regex = "(?:abc)" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" "b" "c" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "abc") # Test named capturing group regex = "(?abc)" grammar_str = _regex_to_ebnf(regex) expected_grammar = r"""root ::= ( "a" "b" "c" ) """ assert grammar_str == expected_grammar assert _is_grammar_accept_string(grammar_str, "abc") # Test unsupported group modifiers unsupported_regexes = [ "(?=abc)", # Positive lookahead "(?!abc)", # Negative lookahead "(?<=abc)", # Positive lookbehind "(?@[\].]|\\")+")@(([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+""" r"""[a-z0-9]([a-z0-9-]*[a-z0-9])?)$""" ), "customer/department=shipping@test.example.test-example.com", ), ] tokenizer_path_regex_instance = [(t, *ri) for t in tokenizer_paths for ri in regex_instances] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path, regex, instance", tokenizer_path_regex_instance) def test_mask_generation(tokenizer_path: str, regex: str, instance: str): print(f"Tokenizer: {tokenizer_path}, regex: {regex}, instance: {instance}") tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) grammar_compiler = xgr.GrammarCompiler(tokenizer_info, cache_enabled=False) time_start = time.monotonic_ns() matcher_compiled_grammar = grammar_compiler.compile_grammar(_regex_to_ebnf(regex)) time_end = time.monotonic_ns() print(f"Time for preprocessing: {(time_end - time_start) / 1e3} us") matcher = xgr.GrammarMatcher(matcher_compiled_grammar) token_bitmask = xgr.allocate_token_bitmask(1, tokenizer_info.vocab_size) for c in instance.encode("utf-8"): time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time for fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") accepted = matcher._debug_accept_string(bytes([c])) assert accepted print(f"Accepting {c}") time_start = time.monotonic_ns() matcher.fill_next_token_bitmask(token_bitmask) time_end = time.monotonic_ns() print(f"Time for fill_next_token_bitmask: {(time_end - time_start) / 1e3} us") assert matcher.accept_token(tokenizer.eos_token_id) assert matcher.is_terminated() if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_token_bitmask_operations.py000066400000000000000000000270621500705317600246240ustar00rootroot00000000000000"""This test uses the optimized JSON grammar provided by the grammar library.""" import sys import time from typing import Callable, List, Optional, Tuple import pytest import torch import xgrammar as xgr from xgrammar.testing import ( _bool_mask_to_bitmask, _get_masked_tokens_from_bitmask, _is_single_token_bitmask, ) _is_cuda_available = torch.cuda.is_available() _is_mps_available = torch.backends.mps.is_available() def test_allocate_reset_token_bitmask(): batch_size = 10 vocab_size = 128005 bitmask = xgr.allocate_token_bitmask(batch_size, vocab_size) assert bitmask.shape == (batch_size, (vocab_size + 31) // 32) assert bitmask.device.type == "cpu" assert (bitmask == 0xFFFFFFFF).all() bitmask.fill_(0) xgr.reset_token_bitmask(bitmask) assert (bitmask == 0xFFFFFFFF).all() token_mask_sizes = (1024, 32000, 32001, 32011) @pytest.mark.parametrize("token_mask_size", token_mask_sizes) @pytest.mark.parametrize("index", (0, 1)) def test_get_masked_tokens_from_bitmask(token_mask_size: int, index: int): bool_mask = torch.randint(0, 2, (2, token_mask_size), dtype=torch.bool) bitmask = _bool_mask_to_bitmask(bool_mask) expected = torch.where(~bool_mask[index])[0].tolist() assert _get_masked_tokens_from_bitmask(bitmask, token_mask_size, index) == expected def test_is_single_token_bitmask(): batch = 2 batch_index = 1 vocab_size = 1024 token_id = 100 bool_mask = torch.zeros(batch, vocab_size, dtype=torch.bool) bitmask = _bool_mask_to_bitmask(bool_mask) assert _is_single_token_bitmask(bitmask, vocab_size, batch_index) == (False, -1) bool_mask[batch_index, token_id] = True bitmask = _bool_mask_to_bitmask(bool_mask) assert _is_single_token_bitmask(bitmask, vocab_size, batch_index) == (True, token_id) bool_mask[batch_index, token_id + 1] = True bitmask = _bool_mask_to_bitmask(bool_mask) assert _is_single_token_bitmask(bitmask, vocab_size, batch_index) == (False, -1) @pytest.mark.parametrize("device", ("cpu", "cuda")) def test_apply_token_bitmask_inplace(device: str): if device == "cuda" and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") neginf = float("-inf") bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool, device=device) bitmask = torch.tensor([0b1010101010], dtype=torch.int32, device=device) logits = torch.tensor( [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.float32, device=device ) expected = torch.where(bool_mask, logits, neginf) xgr.apply_token_bitmask_inplace(logits, bitmask) torch.testing.assert_close(logits, expected) def get_apply_token_bitmask_kernel(impl: str) -> Callable: if impl == "cpu": from xgrammar.kernels.apply_token_bitmask_inplace_cpu import apply_token_bitmask_inplace_cpu return apply_token_bitmask_inplace_cpu elif impl == "cuda": from xgrammar.kernels.apply_token_bitmask_inplace_cuda import ( apply_token_bitmask_inplace_cuda, ) return apply_token_bitmask_inplace_cuda elif impl == "triton": from xgrammar.kernels.apply_token_bitmask_inplace_triton import ( apply_token_bitmask_inplace_triton, ) return apply_token_bitmask_inplace_triton elif impl == "metal": from xgrammar.kernels.apply_token_bitmask_mlx import apply_token_bitmask_mlx return apply_token_bitmask_mlx elif impl == "torch_compile": from xgrammar.kernels.apply_token_bitmask_inplace_torch_compile import ( apply_token_bitmask_inplace_torch_compile, ) return apply_token_bitmask_inplace_torch_compile else: raise ValueError(f"Invalid implementation: {impl}") @pytest.mark.parametrize("impl", ("cpu", "cuda", "triton", "metal", "torch_compile")) def test_apply_token_bitmask_inplace_kernel(impl: str): if impl in ["cuda", "triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") elif impl == "metal" and not _is_mps_available: pytest.skip(reason="MLX is not installed") kernel = get_apply_token_bitmask_kernel(impl) neginf = float("-inf") bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool) logits = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.float32) expected = torch.where(bool_mask, logits, neginf) if impl in ["cuda", "triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask = torch.tensor([0b1010101010], dtype=torch.int32).to("cuda") kernel(logits_gpu, bitmask) torch.cuda.synchronize() torch.testing.assert_close(logits_gpu, expected.to("cuda")) elif impl == "metal": # Import MLX only when needed for the Metal test import mlx.core as mx bitmask = mx.array([0b1010101010], dtype=mx.int32) logits = mx.array(logits.numpy()) result = kernel(bitmask, logits, vocab_size=10) expected = mx.array(expected.numpy()) assert mx.allclose(result, expected) else: assert impl == "cpu" bitmask = torch.tensor([0b1010101010], dtype=torch.int32) kernel(logits, bitmask) torch.testing.assert_close(logits, expected) batch_size__vocab_size__masked_cnt__stride__logits_dtype = [ (1, 128000, 1024, 1, "float32"), (1, 128000, 120000, 1, "float32"), (1, 128001, 120000, 1, "float32"), (1, 128010, 120000, 1, "float32"), (64, 128000, 1024, 1, "float32"), (64, 128000, 120000, 1, "float32"), (64, 128000, 1024, 4, "float32"), (64, 128000, 120000, 4, "float32"), (64, 128001, 120000, 1, "float32"), (64, 128010, 120000, 1, "float32"), (64, 128000, 1024, 1, "float16"), (64, 128000, 1024, 1, "bfloat16"), ] @pytest.mark.parametrize( "batch_size, vocab_size, masked_cnt, stride, logits_dtype", batch_size__vocab_size__masked_cnt__stride__logits_dtype, ) @pytest.mark.parametrize("impl", ("cpu", "cuda", "triton", "torch_compile")) def test_apply_token_bitmask_inplace_kernel_large( batch_size: int, vocab_size: int, masked_cnt: int, stride: int, logits_dtype: str, impl: str ): if impl == "cpu" and logits_dtype != "float32": pytest.skip(reason="CPU implementation supports float32 only") elif impl in ["cuda", "triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") kernel = get_apply_token_bitmask_kernel(impl) logits_dtype = getattr(torch, logits_dtype) logits = torch.randn(batch_size, vocab_size, dtype=logits_dtype) if masked_cnt >= vocab_size: bool_mask = torch.zeros(batch_size, vocab_size, dtype=torch.bool) else: bool_mask = torch.ones(batch_size, vocab_size, dtype=torch.bool) if masked_cnt > 0: masked_positions = torch.stack( [torch.randperm(vocab_size)[:masked_cnt] for _ in range(batch_size)] ) bool_mask.scatter_(1, masked_positions, False) assert (bool_mask.sum(dim=-1) + masked_cnt == vocab_size).all().item() bitmask = _bool_mask_to_bitmask(bool_mask) batch_indices = torch.arange(0, batch_size, stride, dtype=torch.int32) logits_expected = logits.clone() logits_expected[batch_indices] = torch.masked_fill( logits_expected[batch_indices], ~bool_mask[batch_indices], float("-inf") ) bitmask = _bool_mask_to_bitmask(bool_mask) if impl in ["cuda", "triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask_gpu = bitmask.to("cuda") indices = batch_indices.to("cuda") if stride != 1 else None f = lambda: kernel(logits_gpu, bitmask_gpu, indices=indices) torch.cuda.synchronize() f() torch.cuda.synchronize() torch.testing.assert_close(logits_gpu, logits_expected.to("cuda")) try: from triton.testing import do_bench exec_time = do_bench(f, warmup=100, rep=1000) exec_time *= 1e3 except ImportError: pytest.skip(reason="Triton is not installed") else: assert impl == "cpu" indices = batch_indices.tolist() if stride != 1 else None time_start = time.monotonic_ns() kernel(logits, bitmask, indices=indices) time_end = time.monotonic_ns() exec_time = (time_end - time_start) / 1e3 torch.testing.assert_close(logits, logits_expected) print( f"Batch: {batch_size:2} | Vocab: {vocab_size:6} | Masked: {masked_cnt:6} | " f"Stride: {stride:1} | DType: {str(logits_dtype):15} | Impl: {impl:6} | " f"Execution time (μs): {exec_time:.4f}" ) logits_shape__bitmask_shape__vocab_size = [ # logits is larger ((2, 130), (2, 4), None), # bitmask is larger ((2, 120), (2, 4), None), # vocab size is specified ((2, 130), (2, 4), 120), ] @pytest.mark.parametrize( "logits_shape, bitmask_shape, vocab_size", logits_shape__bitmask_shape__vocab_size ) @pytest.mark.parametrize("impl", ("cpu", "triton", "torch_compile")) def test_apply_token_bitmask_inplace_vocab_size( logits_shape: Tuple[int, int], bitmask_shape: Tuple[int, int], vocab_size: Optional[int], impl: str, ): if impl in ["triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") kernel = get_apply_token_bitmask_kernel(impl) logits_dtype = torch.float32 logits = torch.ones(logits_shape, dtype=logits_dtype) bitmask = torch.zeros(bitmask_shape, dtype=torch.int32) vocab_size = min(logits_shape[1], bitmask_shape[1] * 32) if vocab_size is None else vocab_size logits_expected = logits.clone() logits_expected[..., :vocab_size] = float("-inf") if impl in ["triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask_gpu = bitmask.to("cuda") kernel(logits_gpu, bitmask_gpu, vocab_size=vocab_size) torch.testing.assert_close(logits_gpu, logits_expected.to("cuda")) else: assert impl == "cpu" kernel(logits, bitmask, vocab_size=vocab_size) torch.testing.assert_close(logits, logits_expected) logits_batch_size__bitmask_batch_size__vocab_size__indices = [ (3, 3, 128, [0, 1]), (2, 3, 128, [0]), (3, 2, 130, [0]), ] @pytest.mark.parametrize( "logits_batch_size, bitmask_batch_size, vocab_size, indices", logits_batch_size__bitmask_batch_size__vocab_size__indices, ) @pytest.mark.parametrize("impl", ("cpu", "cuda", "triton", "torch_compile")) def test_apply_token_bitmask_inplace_indices( logits_batch_size: int, bitmask_batch_size: int, vocab_size: int, indices: List[int], impl: str ): if impl in ["cuda", "triton", "torch_compile"] and not _is_cuda_available: pytest.skip(reason="CUDA is not installed") kernel = get_apply_token_bitmask_kernel(impl) logits = torch.ones(logits_batch_size, vocab_size, dtype=torch.float32) bool_mask = torch.zeros(bitmask_batch_size, vocab_size, dtype=torch.bool) bitmask = _bool_mask_to_bitmask(bool_mask) logits_expected = logits.clone() logits_expected[indices] = torch.masked_fill( logits_expected[indices], ~bool_mask[indices], float("-inf") ) if impl in ["cuda", "triton", "torch_compile"]: logits_gpu = logits.to("cuda") bitmask_gpu = bitmask.to("cuda") kernel(logits_gpu, bitmask_gpu, indices=indices) torch.testing.assert_close(logits_gpu, logits_expected.to("cuda")) else: assert impl == "cpu" kernel(logits, bitmask, indices=indices) torch.testing.assert_close(logits, logits_expected) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/tests/python/test_tokenizer_info.py000066400000000000000000000255461500705317600225610ustar00rootroot00000000000000import logging import sys from typing import Dict, List, Tuple import pytest from transformers import AutoTokenizer, PreTrainedTokenizerBase import xgrammar as xgr @pytest.fixture(scope="module") def tokenizer_info_storage() -> Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]]: """Mapping from the tokenizer path to the huggingface tokenizer and XGrammar tokenizer info.""" return {} tokenizer_path__vocab_type__prepend_space = [ ("luodian/llama-7b-hf", xgr.VocabType.BYTE_FALLBACK, True), ("meta-llama/Llama-2-7b-chat-hf", xgr.VocabType.BYTE_FALLBACK, True), ("meta-llama/Meta-Llama-3-8B-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("meta-llama/Meta-Llama-3.1-8B-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("lmsys/vicuna-7b-v1.5", xgr.VocabType.BYTE_FALLBACK, True), ("NousResearch/Hermes-2-Theta-Llama-3-70B", xgr.VocabType.BYTE_LEVEL, False), ("NousResearch/Hermes-3-Llama-3.1-8B", xgr.VocabType.BYTE_LEVEL, False), ("google/gemma-2b-it", xgr.VocabType.BYTE_FALLBACK, False), ("CohereForAI/aya-23-8B", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-Coder-V2-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-V2-Chat-0628", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/deepseek-coder-7b-instruct-v1.5", xgr.VocabType.BYTE_LEVEL, False), ("microsoft/phi-2", xgr.VocabType.BYTE_LEVEL, False), ("microsoft/Phi-3-mini-4k-instruct", xgr.VocabType.BYTE_FALLBACK, True), ("microsoft/Phi-3.5-mini-instruct", xgr.VocabType.BYTE_FALLBACK, True), ("Qwen/Qwen1.5-4B-Chat", xgr.VocabType.BYTE_LEVEL, False), ("Qwen/Qwen2-7B-Instruct", xgr.VocabType.BYTE_LEVEL, False), ("microsoft/Phi-3-small-8k-instruct", xgr.VocabType.RAW, False), ("Qwen/Qwen-7B-Chat", xgr.VocabType.RAW, False), ("meta-llama/Llama-3.2-1B", xgr.VocabType.BYTE_LEVEL, False), ("google/gemma-2-2b-it", xgr.VocabType.BYTE_FALLBACK, False), ("deepseek-ai/DeepSeek-V2.5", xgr.VocabType.BYTE_LEVEL, False), ("Qwen/Qwen2.5-1.5B", xgr.VocabType.BYTE_LEVEL, False), ("internlm/internlm2_5-7b-chat", xgr.VocabType.BYTE_FALLBACK, False), ("mistralai/Mixtral-8x22B-Instruct-v0.1", xgr.VocabType.BYTE_FALLBACK, True), ("THUDM/glm-4-9b-chat", xgr.VocabType.RAW, False), ("THUDM/chatglm3-6b", xgr.VocabType.BYTE_FALLBACK, True), ("deepseek-ai/DeepSeek-R1", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", xgr.VocabType.BYTE_LEVEL, False), ("deepseek-ai/DeepSeek-R1-Distill-Llama-8B", xgr.VocabType.BYTE_LEVEL, False), ] tokenizer_paths = [path for path, *_ in tokenizer_path__vocab_type__prepend_space] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_build_tokenizer_info( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) tokenizer_info_storage[tokenizer_path] = (tokenizer, tokenizer_info) @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, vocab_type, add_prefix_space", tokenizer_path__vocab_type__prepend_space ) def test_properties( tokenizer_path: str, vocab_type: xgr.VocabType, add_prefix_space: bool, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] vocab_dict = tokenizer.get_vocab() max_id = max(vocab_dict.values()) if vocab_dict else -1 assert tokenizer_info.vocab_size == max(len(vocab_dict), max_id + 1) assert tokenizer_info.vocab_type == vocab_type assert tokenizer_info.add_prefix_space == add_prefix_space @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_decoded_vocab( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] decoded_vocab = tokenizer_info.decoded_vocab vocab_dict = tokenizer.get_vocab() max_id = max(vocab_dict.values()) if vocab_dict else -1 assert isinstance(decoded_vocab, list) assert all(isinstance(token, bytes) for token in decoded_vocab) assert len(decoded_vocab) == max(len(vocab_dict), max_id + 1) assert len(decoded_vocab) == tokenizer_info.vocab_size @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_stop_token_ids( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None: assert tokenizer_info.stop_token_ids == [tokenizer.eos_token_id] else: logging.warning(f"EOS token id is not defined for tokenizer {tokenizer_path}") @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path", tokenizer_paths) def test_decode_text( tokenizer_path: str, tokenizer_info_storage: Dict[str, Tuple[PreTrainedTokenizerBase, xgr.TokenizerInfo]], ): text = ( "Hello 你好 こんにちは 안녕하세요! 🌎🌍🌏 \u0300\u0301\u0302 \U0001f600\U0001f601\U0001f602 " + "αβγδ АБВГД عربي עברית" + "\n\t\r Special chars: &*()_+-=[]{}|;:'\",.<>?/\\~`!@#$%^haha" ) tokenizer, tokenizer_info = tokenizer_info_storage[tokenizer_path] decoded_vocab = tokenizer_info.decoded_vocab tokenized_text = tokenizer.encode(text) recovered_text = b"".join(decoded_vocab[token_id] for token_id in tokenized_text).decode( "utf-8" ) trial_text = "a" trial_text_roundtrip = b"".join( decoded_vocab[token_id] for token_id in tokenizer.encode(trial_text) ).decode("utf-8") assert trial_text_roundtrip[-1] == "a" detected_prefix = trial_text_roundtrip[:-1] assert tokenizer_info.add_prefix_space == ( len(detected_prefix) > 0 and detected_prefix[-1] == " " ) assert detected_prefix + text == recovered_text tokenizer_path__token_ids__raw_tokens = [ # raw ("microsoft/Phi-3-small-8k-instruct", [10, 94, 37046], [b"+", b"\xa1", b"\xe6\x88\x91"]), # byte_fallback ( "meta-llama/Llama-2-7b-chat-hf", [4, 259, 261, 20565], [b"\x01", b" ", b"er", " исследова".encode("utf-8")], ), # byte_level ( "meta-llama/Meta-Llama-3-8B-Instruct", [1, 37046, 40508], [b'"', "我".encode("utf-8"), b" automotive"], ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path, token_ids, raw_tokens", tokenizer_path__token_ids__raw_tokens ) def test_vocab_conversion(tokenizer_path: str, token_ids: List[int], raw_tokens: List[bytes]): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) vocab = tokenizer_info.decoded_vocab for token_id, raw_token in zip(token_ids, raw_tokens): assert vocab[token_id] == raw_token tokenizer_path__metadata_str = [ ( "microsoft/Phi-3-small-8k-instruct", '{"vocab_type":0,"vocab_size":100352,"add_prefix_space":false,"stop_token_ids":[100257]}', ), ( "meta-llama/Llama-2-7b-chat-hf", '{"vocab_type":1,"vocab_size":32000,"add_prefix_space":true,"stop_token_ids":[2]}', ), ( "meta-llama/Meta-Llama-3-8B-Instruct", '{"vocab_type":2,"vocab_size":128256,"add_prefix_space":false,"stop_token_ids":[128009]}', ), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path, metadata_str", tokenizer_path__metadata_str) def test_dump_metadata_load(tokenizer_path: str, metadata_str: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True, trust_remote_code=True) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer) assert tokenizer_info.dump_metadata() == metadata_str encoded_vocab = tokenizer.get_vocab() encoded_vocab = [token for token, _ in sorted(encoded_vocab.items(), key=lambda x: x[1])] loaded = xgr.TokenizerInfo.from_vocab_and_metadata(encoded_vocab, metadata_str) assert loaded.decoded_vocab == tokenizer_info.decoded_vocab loaded_new = xgr.TokenizerInfo(tokenizer_info.decoded_vocab) assert loaded_new.decoded_vocab == tokenizer_info.decoded_vocab def test_special_token_detection(): # Now only empty string "" is treated as a special token. vocab_dict = ["", "", "", "[@BOS@]", "regular", "<>", "", ""] tokenizer_info = xgr.TokenizerInfo.from_vocab_and_metadata( vocab_dict, '{"vocab_type":1,"vocab_size":8,"add_prefix_space":true,"stop_token_ids":[2]}' ) expected_special_tokens = {0} assert set(tokenizer_info.special_token_ids) == expected_special_tokens @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path", ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Meta-Llama-3-8B-Instruct"] ) def test_customize_stop_token_ids(tokenizer_path: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, stop_token_ids=[1, 2, 3]) assert tokenizer_info.stop_token_ids == [1, 2, 3] @pytest.mark.hf_token_required @pytest.mark.parametrize( "tokenizer_path", ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Meta-Llama-3-8B-Instruct"] ) def test_padding_vocab_size(tokenizer_path: str): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) original_vocab_size = len(tokenizer.get_vocab()) tokenizer_info = xgr.TokenizerInfo.from_huggingface( tokenizer, vocab_size=original_vocab_size + 5 ) assert tokenizer_info.vocab_size == original_vocab_size + 5 assert tokenizer_info.special_token_ids[-5:] == [original_vocab_size + i for i in range(5)] tokenizer_path__model_vocab_size = [ ("meta-llama/Llama-3.2-11B-Vision-Instruct", 128256), ("meta-llama/Llama-Guard-3-11B-Vision", 128256), ("allenai/Molmo-72B-0924", 152064), ] @pytest.mark.hf_token_required @pytest.mark.parametrize("tokenizer_path, model_vocab_size", tokenizer_path__model_vocab_size) def test_model_vocab_size_smaller_than_tokenizer(tokenizer_path: str, model_vocab_size: int): tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) original_vocab_size = len(tokenizer.get_vocab()) assert original_vocab_size > model_vocab_size tokenizer_info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=model_vocab_size) assert tokenizer_info.vocab_size == model_vocab_size assert len(tokenizer_info.decoded_vocab) == model_vocab_size print(tokenizer_info.special_token_ids) print(len(tokenizer_info.decoded_vocab)) if __name__ == "__main__": pytest.main(sys.argv) xgrammar-0.1.19/web/000077500000000000000000000000001500705317600142015ustar00rootroot00000000000000xgrammar-0.1.19/web/.eslintignore000066400000000000000000000001041500705317600166770ustar00rootroot00000000000000dist debug lib build node_modules xgrammar_binding.js .eslintrc.cjs xgrammar-0.1.19/web/.eslintrc.cjs000066400000000000000000000004111500705317600165770ustar00rootroot00000000000000module.exports = { extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'], parser: '@typescript-eslint/parser', plugins: ['@typescript-eslint'], root: true, rules: { "@typescript-eslint/no-explicit-any": "off" } }; xgrammar-0.1.19/web/.gitignore000066400000000000000000000001531500705317600161700ustar00rootroot00000000000000src/xgrammar_binding.js build node_modules dist lib .cache .vscode .parcel-cache example/package-lock.json xgrammar-0.1.19/web/README.md000066400000000000000000000020651500705317600154630ustar00rootroot00000000000000# web-xgrammar This folder contains the source code and emcc bindings for compiling XGrammar to Javascript/Typescript via [emscripten](https://emscripten.org/). ### Build from source 1. Install [emscripten](https://emscripten.org). It is an LLVM-based compiler that compiles C/C++ source code to WebAssembly. - Follow the [installation instruction](https://emscripten.org/docs/getting_started/downloads.html#installation-instructions-using-the-emsdk-recommended) to install the latest emsdk. - Source `emsdk_env.sh` by `source /path/to/emsdk_env.sh`, so that `emcc` is reachable from PATH and the command `emcc` works. - We can verify the successful installation by trying out `emcc` in the terminal. 2. Modify the content of `cmake/config.cmake` to be `web/config.cmake`. 3. Run the following ```bash source /path/to/emsdk_env.sh npm install npm run build ``` ### Example To try out the test webpage, run the following ```bash cd example npm install npm start ``` ### Testing For testing in `node` environment, run: ```bash npm test ``` xgrammar-0.1.19/web/build.sh000077500000000000000000000010611500705317600156350ustar00rootroot00000000000000#!/bin/bash set -euxo pipefail mkdir -p build cd build emcmake cmake ../.. -DBUILD_PYTHON_BINDINGS=OFF\ -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-O3 -DCOMPILE_WASM_RUNTIME -DXGRAMMAR_LOG_CUSTOMIZE=1" emmake make xgrammar -j8 cd .. emcc --bind -o src/xgrammar_binding.js src/xgrammar_binding.cc\ build/libxgrammar.a\ -O3 -s EXPORT_ES6=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s NO_DYNAMIC_EXECUTION=1 -s MODULARIZE=1 -s SINGLE_FILE=1 -s EXPORTED_RUNTIME_METHODS=FS -s ALLOW_MEMORY_GROWTH=1\ -I../include -I../3rdparty/picojson -I../3rdparty/dlpack/include xgrammar-0.1.19/web/config.cmake000066400000000000000000000001571500705317600164530ustar00rootroot00000000000000set(CMAKE_BUILD_TYPE RelWithDebInfo) set(XGRAMMAR_BUILD_PYTHON_BINDINGS OFF) set(XGRAMMAR_BUILD_CXX_TESTS OFF) xgrammar-0.1.19/web/example/000077500000000000000000000000001500705317600156345ustar00rootroot00000000000000xgrammar-0.1.19/web/example/package.json000066400000000000000000000010571500705317600201250ustar00rootroot00000000000000{ "name": "web-xgrammar-example", "version": "0.1.0", "private": true, "type": "module", "scripts": { "start": "parcel src/example.html --port 8888" }, "browser": {}, "devDependencies": { "@mlc-ai/web-xgrammar": "0.1.0", "@mlc-ai/web-tokenizers": "^0.1.5", "buffer": "^5.7.1", "parcel": "^2.8.3", "process": "^0.11.10", "punycode": "^1.4.1", "querystring-es3": "^0.2.1", "tslib": "^2.3.1", "typescript": "^4.9.5", "url": "^0.11.0" } } xgrammar-0.1.19/web/example/src/000077500000000000000000000000001500705317600164235ustar00rootroot00000000000000xgrammar-0.1.19/web/example/src/example.html000066400000000000000000000002501500705317600207410ustar00rootroot00000000000000

Tokenizer Test Page

Open console to see output xgrammar-0.1.19/web/example/src/example.ts000066400000000000000000000143661500705317600204400ustar00rootroot00000000000000import { Grammar, GrammarMatcher, TokenizerInfo, GrammarCompiler, CompiledGrammar, Testings } from "@mlc-ai/web-xgrammar" import { Tokenizer } from "@mlc-ai/web-tokenizers"; import { Type, Static } from "@sinclair/typebox"; async function getTokenizerInfoAndTokenizerFromUrl( tokenizerUrl: string, vocabType: string, prependSpaceInTokenization: boolean, ): Promise<[TokenizerInfo, Tokenizer]> { // 1. Get tokenizer, we use "@mlc-ai/web-tokenizers" here, but any should work const jsonBuffer = await (await fetch(tokenizerUrl)).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get encoded vocab const tstartGetToken = performance.now(); const rawTokenTable: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { rawTokenTable.push(tokenizer.idToToken(tokenId)); } console.log("Get raw token table (ms): ", (performance.now() - tstartGetToken)); // 3. Post process vocab const tstartGetTokenizerInfo = performance.now(); const tokenizerInfo = await TokenizerInfo.createTokenizerInfo(rawTokenTable, vocabType, prependSpaceInTokenization); console.log("createTokenizerInfo (ms): ", (performance.now() - tstartGetTokenizerInfo)); return [tokenizerInfo, tokenizer]; } async function jsonExample() { console.log("json example"); const result = await getTokenizerInfoAndTokenizerFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const tokenizerInfo = result[0]; const tokenizer = result[1]; const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); // 1. Initialize grammar state matcher with JSON grammar const grammar: CompiledGrammar = await compiler.compileBuiltinJSONGrammar(); const grammarMatcher = await GrammarMatcher.createGrammarMatcher(grammar); console.log(grammarMatcher); // 2. Simulated generation of an LLM const input = String.raw`{"hi": 1}<|end_of_text|>`; const encodedTokens = tokenizer.encode(input); // 3. We expect the matcher to accept all tokens generated since it is a valid JSON for (let i = 0; i < encodedTokens.length; i++) { // 3.1 Generate token bitmask that will modify logits of the LLM if (!grammarMatcher.isTerminated()) { const bitmask = await grammarMatcher.getNextTokenBitmask(); // For debugging, we can check the rejected token IDs from the mask const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); } // 3.2 Say the LLM generated `curToken`, which is simulated here, we use `acceptToken()` // to update the state of the matcher, so it will generate a new bitmask for the next // auto-regressive generation const curToken = encodedTokens[i]; const accepted = grammarMatcher.acceptToken(curToken); if (!accepted) { throw Error("Expect token to be accepted"); } } // 4. The last token is and stop token, so the matcher has terminated. console.log("grammarMatcher.isTerminated(): ", grammarMatcher.isTerminated()); grammarMatcher.dispose(); } async function jsonSchemaExample() { console.log("json schema example"); // 0. Prepare a schema const T = Type.Object({ name: Type.String(), house: Type.Enum({ Gryffindor: "Gryffindor", Hufflepuff: "Hufflepuff", Ravenclaw: "Ravenclaw", Slytherin: "Slytherin", }), blood_status: Type.Enum({ "Pure-blood": "Pure-blood", "Half-blood": "Half-blood", "Muggle-born": "Muggle-born", }), occupation: Type.Enum({ Student: "Student", Professor: "Professor", "Ministry of Magic": "Ministry of Magic", Other: "Other", }), wand: Type.Object({ wood: Type.String(), core: Type.String(), length: Type.Number(), }), alive: Type.Boolean(), patronus: Type.String(), }); type T = Static; const schema = JSON.stringify(T); console.log("schema: ", schema); const result = await getTokenizerInfoAndTokenizerFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const tokenizerInfo = result[0]; const tokenizer = result[1]; // 1. Instantiate matcher with a grammar defined by the above schema const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const tstartInitMatcher = performance.now(); const grammar: CompiledGrammar = await compiler.compileJSONSchema(schema); const grammarMatcher = await GrammarMatcher.createGrammarMatcher(grammar); console.log("createGrammarMatcher (ms): ", (performance.now() - tstartInitMatcher)); console.log(grammarMatcher); // 2. Simulated generation of an LLM const input = String.raw`{ "name": "Hermione Granger", "house": "Ravenclaw", "blood_status": "Muggle-born", "occupation": "Student", "wand": { "wood": "Vine", "core": "Phoenix Feather", "length": 10 }, "alive": true, "patronus": "Otter" }<|end_of_text|>`; const encodedTokens = tokenizer.encode(input); // 3. We expect the matcher to accept all tokens generated since it is a valid JSON for (let i = 0; i < encodedTokens.length; i++) { // 3.1 Generate token bitmask that will modify logits of the LLM if (!grammarMatcher.isTerminated()) { const bitmask = await grammarMatcher.getNextTokenBitmask(); // For debugging, we can check the rejected token IDs from the mask const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); } // 3.2 Say the LLM generated `curToken`, which is simulated here, we use `acceptToken()` // to update the state of the matcher, so it will generate a new bitmask for the next // auto-regressive generation const curToken = encodedTokens[i]; const accepted = grammarMatcher.acceptToken(curToken); if (!accepted) { throw Error("Expect token to be accepted"); } } // 4. The last token is and stop token, so the matcher has terminated. console.log("grammarMatcher.isTerminated(): ", grammarMatcher.isTerminated()); grammarMatcher.dispose(); } async function testAll() { await jsonExample(); await jsonSchemaExample(); } testAll(); xgrammar-0.1.19/web/example/tsconfig.json000066400000000000000000000001711500705317600203420ustar00rootroot00000000000000{ "compilerOptions": { }, "include": ["src"], "exclude": ["node_modules", "build", "dist", "rollup.config.js"] } xgrammar-0.1.19/web/jest.config.cjs000066400000000000000000000001121500705317600171050ustar00rootroot00000000000000module.exports = { preset: "ts-jest", testEnvironment: "node", }; xgrammar-0.1.19/web/package-lock.json000066400000000000000000005515571500705317600174370ustar00rootroot00000000000000{ "name": "@mlc-ai/web-xgrammar", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mlc-ai/web-xgrammar", "version": "0.1.0", "license": "Apache-2.0", "devDependencies": { "@jest/globals": "^29.7.0", "@mlc-ai/web-tokenizers": "^0.1.5", "@rollup/plugin-commonjs": "^20.0.0", "@rollup/plugin-node-resolve": "^13.0.4", "@rollup/plugin-wasm": "^5.1.2", "@types/jest": "^29.5.12", "@typescript-eslint/eslint-plugin": "^5.59.6", "@typescript-eslint/parser": "^5.59.6", "eslint": "^8.41.0", "jest": "^29.7.0", "rollup": "^2.56.2", "rollup-plugin-typescript2": "^0.34.1", "ts-jest": "^29.2.5", "tslib": "^2.3.1", "typescript": "^4.9.5" } }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", "dev": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" }, "engines": { "node": ">=6.0.0" } }, "node_modules/@babel/code-frame": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", "dev": true, "dependencies": { "@babel/helper-validator-identifier": "^7.25.9", "js-tokens": "^4.0.0", "picocolors": "^1.0.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/compat-data": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.2.tgz", "integrity": "sha512-Z0WgzSEa+aUcdiJuCIqgujCshpMWgUpgOxXotrYPSA53hA3qopNaqcJpyr0hVb1FeWdnqFA35/fUtXgBK8srQg==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/core": { "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", "integrity": "sha512-i1SLeK+DzNnQ3LL/CswPCa/E5u4lh1k6IAEphON8F+cXt0t9euTshDru0q7/IqMa1PMPz5RnHuHscF8/ZJsStg==", "dev": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.26.0", "@babel/generator": "^7.26.0", "@babel/helper-compilation-targets": "^7.25.9", "@babel/helper-module-transforms": "^7.26.0", "@babel/helpers": "^7.26.0", "@babel/parser": "^7.26.0", "@babel/template": "^7.25.9", "@babel/traverse": "^7.25.9", "@babel/types": "^7.26.0", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" }, "engines": { "node": ">=6.9.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/babel" } }, "node_modules/@babel/core/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" } }, "node_modules/@babel/generator": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", "dev": true, "dependencies": { "@babel/parser": "^7.26.2", "@babel/types": "^7.26.0", "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25", "jsesc": "^3.0.2" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-compilation-targets": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.25.9.tgz", "integrity": "sha512-j9Db8Suy6yV/VHa4qzrj9yZfZxhLWQdVnRlXxmKLYlhWUVB1sB2G5sxuWYXk/whHD9iW76PmNzxZ4UCnTQTVEQ==", "dev": true, "dependencies": { "@babel/compat-data": "^7.25.9", "@babel/helper-validator-option": "^7.25.9", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-compilation-targets/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" } }, "node_modules/@babel/helper-module-imports": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", "dev": true, "dependencies": { "@babel/traverse": "^7.25.9", "@babel/types": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", "dev": true, "dependencies": { "@babel/helper-module-imports": "^7.25.9", "@babel/helper-validator-identifier": "^7.25.9", "@babel/traverse": "^7.25.9" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "node_modules/@babel/helper-plugin-utils": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz", "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-string-parser": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-option": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helpers": { "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.0.tgz", "integrity": "sha512-tbhNuIxNcVb21pInl3ZSjksLCvgdZy9KwJ8brv993QtIVKJBBkYXz4q4ZbAv31GdnC+R90np23L5FbEBlthAEw==", "dev": true, "dependencies": { "@babel/template": "^7.25.9", "@babel/types": "^7.26.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { "version": "7.26.2", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz", "integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==", "dev": true, "dependencies": { "@babel/types": "^7.26.0" }, "bin": { "parser": "bin/babel-parser.js" }, "engines": { "node": ">=6.0.0" } }, "node_modules/@babel/plugin-syntax-async-generators": { "version": "7.8.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-bigint": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-class-properties": { "version": "7.12.13", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.12.13" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-class-static-block": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-import-attributes": { "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.25.9" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-import-meta": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-json-strings": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-jsx": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.25.9" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-logical-assignment-operators": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-numeric-separator": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-object-rest-spread": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-optional-catch-binding": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-optional-chaining": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-private-property-in-object": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-top-level-await": { "version": "7.14.5", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/plugin-syntax-typescript": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.25.9" }, "engines": { "node": ">=6.9.0" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "node_modules/@babel/template": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", "dev": true, "dependencies": { "@babel/code-frame": "^7.25.9", "@babel/parser": "^7.25.9", "@babel/types": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", "dev": true, "dependencies": { "@babel/code-frame": "^7.25.9", "@babel/generator": "^7.25.9", "@babel/parser": "^7.25.9", "@babel/template": "^7.25.9", "@babel/types": "^7.25.9", "debug": "^4.3.1", "globals": "^11.1.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse/node_modules/globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true, "engines": { "node": ">=4" } }, "node_modules/@babel/types": { "version": "7.26.0", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", "dev": true, "dependencies": { "@babel/helper-string-parser": "^7.25.9", "@babel/helper-validator-identifier": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.1.tgz", "integrity": "sha512-s3O3waFUrMV8P/XaF/+ZTp1X9XBZW1a4B97ZnjQF2KYWaFD2A8KyFBsrsfSjEmjn3RGWAIuvlneuZm3CUK3jbA==", "dev": true, "dependencies": { "eslint-visitor-keys": "^3.4.3" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" } }, "node_modules/@eslint-community/regexpp": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.1.tgz", "integrity": "sha512-CCZCDJuduB9OUkFkY2IgppNZMi2lBQgD2qzwXkEia16cge2pijY/aXi96CJMquDMn3nJdlPV1A5KrJEXwfLNzQ==", "dev": true, "engines": { "node": "^12.0.0 || ^14.0.0 || >=16.0.0" } }, "node_modules/@eslint/eslintrc": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", "espree": "^9.6.0", "globals": "^13.19.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.0", "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/@eslint/js": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "deprecated": "Use @eslint/config-array instead", "dev": true, "dependencies": { "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" }, "engines": { "node": ">=10.10.0" } }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", "dev": true, "engines": { "node": ">=12.22" }, "funding": { "type": "github", "url": "https://github.com/sponsors/nzakas" } }, "node_modules/@humanwhocodes/object-schema": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", "deprecated": "Use @eslint/object-schema instead", "dev": true }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", "dev": true, "dependencies": { "camelcase": "^5.3.1", "find-up": "^4.1.0", "get-package-type": "^0.1.0", "js-yaml": "^3.13.1", "resolve-from": "^5.0.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, "dependencies": { "sprintf-js": "~1.0.2" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "dev": true, "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, "dependencies": { "p-locate": "^4.1.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "dev": true, "dependencies": { "p-try": "^2.0.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, "dependencies": { "p-limit": "^2.2.0" }, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/@jest/console": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "slash": "^3.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/core": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", "dev": true, "dependencies": { "@jest/console": "^29.7.0", "@jest/reporters": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "ansi-escapes": "^4.2.1", "chalk": "^4.0.0", "ci-info": "^3.2.0", "exit": "^0.1.2", "graceful-fs": "^4.2.9", "jest-changed-files": "^29.7.0", "jest-config": "^29.7.0", "jest-haste-map": "^29.7.0", "jest-message-util": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-resolve-dependencies": "^29.7.0", "jest-runner": "^29.7.0", "jest-runtime": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "jest-watcher": "^29.7.0", "micromatch": "^4.0.4", "pretty-format": "^29.7.0", "slash": "^3.0.0", "strip-ansi": "^6.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/@jest/environment": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", "dev": true, "dependencies": { "@jest/fake-timers": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "jest-mock": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/expect": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", "dev": true, "dependencies": { "expect": "^29.7.0", "jest-snapshot": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/expect-utils": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", "dev": true, "dependencies": { "jest-get-type": "^29.6.3" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/fake-timers": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "@sinonjs/fake-timers": "^10.0.2", "@types/node": "*", "jest-message-util": "^29.7.0", "jest-mock": "^29.7.0", "jest-util": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/globals": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", "dev": true, "dependencies": { "@jest/environment": "^29.7.0", "@jest/expect": "^29.7.0", "@jest/types": "^29.6.3", "jest-mock": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/reporters": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", "dev": true, "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@jridgewell/trace-mapping": "^0.3.18", "@types/node": "*", "chalk": "^4.0.0", "collect-v8-coverage": "^1.0.0", "exit": "^0.1.2", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^4.0.0", "istanbul-reports": "^3.1.3", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "jest-worker": "^29.7.0", "slash": "^3.0.0", "string-length": "^4.0.1", "strip-ansi": "^6.0.0", "v8-to-istanbul": "^9.0.1" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/@jest/schemas": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", "dev": true, "dependencies": { "@sinclair/typebox": "^0.27.8" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/source-map": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", "dev": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.18", "callsites": "^3.0.0", "graceful-fs": "^4.2.9" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/test-result": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", "dev": true, "dependencies": { "@jest/console": "^29.7.0", "@jest/types": "^29.6.3", "@types/istanbul-lib-coverage": "^2.0.0", "collect-v8-coverage": "^1.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/test-sequencer": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", "dev": true, "dependencies": { "@jest/test-result": "^29.7.0", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "slash": "^3.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/transform": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", "dev": true, "dependencies": { "@babel/core": "^7.11.6", "@jest/types": "^29.6.3", "@jridgewell/trace-mapping": "^0.3.18", "babel-plugin-istanbul": "^6.1.1", "chalk": "^4.0.0", "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-util": "^29.7.0", "micromatch": "^4.0.4", "pirates": "^4.0.4", "slash": "^3.0.0", "write-file-atomic": "^4.0.2" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jest/types": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", "dev": true, "dependencies": { "@jest/schemas": "^29.6.3", "@types/istanbul-lib-coverage": "^2.0.0", "@types/istanbul-reports": "^3.0.0", "@types/node": "*", "@types/yargs": "^17.0.8", "chalk": "^4.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", "dev": true, "dependencies": { "@jridgewell/set-array": "^1.2.1", "@jridgewell/sourcemap-codec": "^1.4.10", "@jridgewell/trace-mapping": "^0.3.24" }, "engines": { "node": ">=6.0.0" } }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "engines": { "node": ">=6.0.0" } }, "node_modules/@jridgewell/set-array": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", "dev": true, "engines": { "node": ">=6.0.0" } }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", "dev": true }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.25", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "node_modules/@mlc-ai/web-tokenizers": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/@mlc-ai/web-tokenizers/-/web-tokenizers-0.1.5.tgz", "integrity": "sha512-G7vjJzZyOFJvAfx42kPEU7Z2hkAAGWvKJHfMTLdvY8QDLFvvvVOwmEk89Mh+7PBVPpcfh3PW0npTTupFnLMwHw==", "dev": true }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" }, "engines": { "node": ">= 8" } }, "node_modules/@nodelib/fs.stat": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true, "engines": { "node": ">= 8" } }, "node_modules/@nodelib/fs.walk": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" }, "engines": { "node": ">= 8" } }, "node_modules/@rollup/plugin-commonjs": { "version": "20.0.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-20.0.0.tgz", "integrity": "sha512-5K0g5W2Ol8hAcTHqcTBHiA7M58tfmYi1o9KxeJuuRNpGaTa5iLjcyemBitCBcKXaHamOBBEH2dGom6v6Unmqjg==", "dev": true, "dependencies": { "@rollup/pluginutils": "^3.1.0", "commondir": "^1.0.1", "estree-walker": "^2.0.1", "glob": "^7.1.6", "is-reference": "^1.2.1", "magic-string": "^0.25.7", "resolve": "^1.17.0" }, "engines": { "node": ">= 8.0.0" }, "peerDependencies": { "rollup": "^2.38.3" } }, "node_modules/@rollup/plugin-node-resolve": { "version": "13.3.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-13.3.0.tgz", "integrity": "sha512-Lus8rbUo1eEcnS4yTFKLZrVumLPY+YayBdWXgFSHYhTT2iJbMhoaaBL3xl5NCdeRytErGr8tZ0L71BMRmnlwSw==", "dev": true, "dependencies": { "@rollup/pluginutils": "^3.1.0", "@types/resolve": "1.17.1", "deepmerge": "^4.2.2", "is-builtin-module": "^3.1.0", "is-module": "^1.0.0", "resolve": "^1.19.0" }, "engines": { "node": ">= 10.0.0" }, "peerDependencies": { "rollup": "^2.42.0" } }, "node_modules/@rollup/plugin-wasm": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@rollup/plugin-wasm/-/plugin-wasm-5.2.0.tgz", "integrity": "sha512-PR3ff67ls2Kr9H04pZ24wJYPZq0YV+UHySpk7OuAJxyc7o5Q8NHFdwi4pfMtJkJkqfN1/QY/nq46SoRDoDvK2w==", "dev": true, "engines": { "node": ">=10.0.0" }, "peerDependencies": { "rollup": "^1.20.0 || ^2.0.0" } }, "node_modules/@rollup/pluginutils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.1.0.tgz", "integrity": "sha512-GksZ6pr6TpIjHm8h9lSQ8pi8BE9VeubNT0OMJ3B5uZJ8pz73NPiqOtCog/x2/QzM1ENChPKxMDhiQuRHsqc+lg==", "dev": true, "dependencies": { "@types/estree": "0.0.39", "estree-walker": "^1.0.1", "picomatch": "^2.2.2" }, "engines": { "node": ">= 8.0.0" }, "peerDependencies": { "rollup": "^1.20.0||^2.0.0" } }, "node_modules/@rollup/pluginutils/node_modules/estree-walker": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-1.0.1.tgz", "integrity": "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg==", "dev": true }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, "node_modules/@sinonjs/commons": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", "dev": true, "dependencies": { "type-detect": "4.0.8" } }, "node_modules/@sinonjs/fake-timers": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", "dev": true, "dependencies": { "@sinonjs/commons": "^3.0.0" } }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", "dev": true, "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "node_modules/@types/babel__generator": { "version": "7.6.8", "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", "dev": true, "dependencies": { "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__template": { "version": "7.4.4", "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", "dev": true, "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__traverse": { "version": "7.20.6", "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", "dev": true, "dependencies": { "@babel/types": "^7.20.7" } }, "node_modules/@types/estree": { "version": "0.0.39", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", "integrity": "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==", "dev": true }, "node_modules/@types/graceful-fs": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", "dev": true, "dependencies": { "@types/node": "*" } }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", "dev": true }, "node_modules/@types/istanbul-lib-report": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", "dev": true, "dependencies": { "@types/istanbul-lib-coverage": "*" } }, "node_modules/@types/istanbul-reports": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", "dev": true, "dependencies": { "@types/istanbul-lib-report": "*" } }, "node_modules/@types/jest": { "version": "29.5.14", "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", "dev": true, "dependencies": { "expect": "^29.0.0", "pretty-format": "^29.0.0" } }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, "node_modules/@types/node": { "version": "22.9.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.9.1.tgz", "integrity": "sha512-p8Yy/8sw1caA8CdRIQBG5tiLHmxtQKObCijiAa9Ez+d4+PRffM4054xbju0msf+cvhJpnFEeNjxmVT/0ipktrg==", "dev": true, "dependencies": { "undici-types": "~6.19.8" } }, "node_modules/@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", "integrity": "sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw==", "dev": true, "dependencies": { "@types/node": "*" } }, "node_modules/@types/semver": { "version": "7.5.8", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", "dev": true }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", "dev": true, "dependencies": { "@types/yargs-parser": "*" } }, "node_modules/@types/yargs-parser": { "version": "21.0.3", "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", "integrity": "sha512-TiZzBSJja/LbhNPvk6yc0JrX9XqhQ0hdh6M2svYfsHGejaKFIAGd9MQ+ERIMzLGlN/kZoYIgdxFV0PuljTKXag==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.4.0", "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/type-utils": "5.62.0", "@typescript-eslint/utils": "5.62.0", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.0", "natural-compare-lite": "^1.4.0", "semver": "^7.3.7", "tsutils": "^3.21.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "@typescript-eslint/parser": "^5.0.0", "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "typescript": { "optional": true } } }, "node_modules/@typescript-eslint/parser": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.62.0.tgz", "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", "@typescript-eslint/typescript-estree": "5.62.0", "debug": "^4.3.4" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "typescript": { "optional": true } } }, "node_modules/@typescript-eslint/scope-manager": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-5.62.0.tgz", "integrity": "sha512-VXuvVvZeQCQb5Zgf4HAxc04q5j+WrNAtNh9OwCsCgpKqESMTu3tF/jhZ3xG6T4NZwWl65Bg8KuS2uEvhSfLl0w==", "dev": true, "dependencies": { "@typescript-eslint/types": "5.62.0", "@typescript-eslint/visitor-keys": "5.62.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@typescript-eslint/type-utils": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-5.62.0.tgz", "integrity": "sha512-xsSQreu+VnfbqQpW5vnCJdq1Z3Q0U31qiWmRhr98ONQmcp/yhiPJFPq8MXiJVLiksmOKSjIldZzkebzHuCGzew==", "dev": true, "dependencies": { "@typescript-eslint/typescript-estree": "5.62.0", "@typescript-eslint/utils": "5.62.0", "debug": "^4.3.4", "tsutils": "^3.21.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "*" }, "peerDependenciesMeta": { "typescript": { "optional": true } } }, "node_modules/@typescript-eslint/types": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.62.0.tgz", "integrity": "sha512-87NVngcbVXUahrRTqIK27gD2t5Cu1yuCXxbLcFtCzZGlfyVWWh8mLHkoxzjsB6DDNnvdL+fW8MiwPEJyGJQDgQ==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@typescript-eslint/typescript-estree": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.62.0.tgz", "integrity": "sha512-CmcQ6uY7b9y694lKdRB8FEel7JbU/40iSAPomu++SjLMntB+2Leay2LO6i8VnJk58MtE9/nQSFIH6jpyRWyYzA==", "dev": true, "dependencies": { "@typescript-eslint/types": "5.62.0", "@typescript-eslint/visitor-keys": "5.62.0", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", "semver": "^7.3.7", "tsutils": "^3.21.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependenciesMeta": { "typescript": { "optional": true } } }, "node_modules/@typescript-eslint/utils": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-5.62.0.tgz", "integrity": "sha512-n8oxjeb5aIbPFEtmQxQYOLI0i9n5ySBEY/ZEHHZqKQSFnxio1rv6dthascc9dLuwrL0RC5mPCxB7vnAVGAYWAQ==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@types/json-schema": "^7.0.9", "@types/semver": "^7.3.12", "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", "@typescript-eslint/typescript-estree": "5.62.0", "eslint-scope": "^5.1.1", "semver": "^7.3.7" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { "eslint": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/@typescript-eslint/visitor-keys": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.62.0.tgz", "integrity": "sha512-07ny+LHRzQXepkGg6w0mFY41fVUNBrL2Roj/++7V1txKugfjm/Ci/qSND03r2RhlJhJYMcTn9AhhSSqQp0Ysyw==", "dev": true, "dependencies": { "@typescript-eslint/types": "5.62.0", "eslint-visitor-keys": "^3.3.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@ungap/structured-clone": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "dev": true }, "node_modules/acorn": { "version": "8.14.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", "dev": true, "bin": { "acorn": "bin/acorn" }, "engines": { "node": ">=0.4.0" } }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true, "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", "dev": true, "dependencies": { "type-fest": "^0.21.3" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/ansi-escapes/node_modules/type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, "dependencies": { "color-convert": "^2.0.1" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", "dev": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" }, "engines": { "node": ">= 8" } }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "dev": true }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", "dev": true, "dependencies": { "@jest/transform": "^29.7.0", "@types/babel__core": "^7.1.14", "babel-plugin-istanbul": "^6.1.1", "babel-preset-jest": "^29.6.3", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "slash": "^3.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "@babel/core": "^7.8.0" } }, "node_modules/babel-plugin-istanbul": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", "dev": true, "dependencies": { "@babel/helper-plugin-utils": "^7.0.0", "@istanbuljs/load-nyc-config": "^1.0.0", "@istanbuljs/schema": "^0.1.2", "istanbul-lib-instrument": "^5.0.4", "test-exclude": "^6.0.0" }, "engines": { "node": ">=8" } }, "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", "dev": true, "dependencies": { "@babel/core": "^7.12.3", "@babel/parser": "^7.14.7", "@istanbuljs/schema": "^0.1.2", "istanbul-lib-coverage": "^3.2.0", "semver": "^6.3.0" }, "engines": { "node": ">=8" } }, "node_modules/babel-plugin-istanbul/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" } }, "node_modules/babel-plugin-jest-hoist": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", "dev": true, "dependencies": { "@babel/template": "^7.3.3", "@babel/types": "^7.3.3", "@types/babel__core": "^7.1.14", "@types/babel__traverse": "^7.0.6" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/babel-preset-current-node-syntax": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", "dev": true, "dependencies": { "@babel/plugin-syntax-async-generators": "^7.8.4", "@babel/plugin-syntax-bigint": "^7.8.3", "@babel/plugin-syntax-class-properties": "^7.12.13", "@babel/plugin-syntax-class-static-block": "^7.14.5", "@babel/plugin-syntax-import-attributes": "^7.24.7", "@babel/plugin-syntax-import-meta": "^7.10.4", "@babel/plugin-syntax-json-strings": "^7.8.3", "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", "@babel/plugin-syntax-numeric-separator": "^7.10.4", "@babel/plugin-syntax-object-rest-spread": "^7.8.3", "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", "@babel/plugin-syntax-optional-chaining": "^7.8.3", "@babel/plugin-syntax-private-property-in-object": "^7.14.5", "@babel/plugin-syntax-top-level-await": "^7.14.5" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "node_modules/babel-preset-jest": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", "dev": true, "dependencies": { "babel-plugin-jest-hoist": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { "fill-range": "^7.1.1" }, "engines": { "node": ">=8" } }, "node_modules/browserslist": { "version": "4.24.2", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz", "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==", "dev": true, "funding": [ { "type": "opencollective", "url": "https://opencollective.com/browserslist" }, { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" }, { "type": "github", "url": "https://github.com/sponsors/ai" } ], "dependencies": { "caniuse-lite": "^1.0.30001669", "electron-to-chromium": "^1.5.41", "node-releases": "^2.0.18", "update-browserslist-db": "^1.1.1" }, "bin": { "browserslist": "cli.js" }, "engines": { "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, "node_modules/bs-logger": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", "dev": true, "dependencies": { "fast-json-stable-stringify": "2.x" }, "engines": { "node": ">= 6" } }, "node_modules/bser": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", "dev": true, "dependencies": { "node-int64": "^0.4.0" } }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, "node_modules/builtin-modules": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", "dev": true, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/caniuse-lite": { "version": "1.0.30001683", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001683.tgz", "integrity": "sha512-iqmNnThZ0n70mNwvxpEC2nBJ037ZHZUoBI5Gorh1Mw6IlEAZujEoU1tXA628iZfzm7R9FvFzxbfdgml82a3k8Q==", "dev": true, "funding": [ { "type": "opencollective", "url": "https://opencollective.com/browserslist" }, { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" }, { "type": "github", "url": "https://github.com/sponsors/ai" } ] }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/chalk?sponsor=1" } }, "node_modules/char-regex": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", "dev": true, "engines": { "node": ">=10" } }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", "dev": true, "funding": [ { "type": "github", "url": "https://github.com/sponsors/sibiraj-s" } ], "engines": { "node": ">=8" } }, "node_modules/cjs-module-lexer": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz", "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==", "dev": true }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" }, "engines": { "node": ">=12" } }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", "dev": true, "engines": { "iojs": ">= 1.0.0", "node": ">= 0.12.0" } }, "node_modules/collect-v8-coverage": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz", "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", "dev": true }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dev": true, "dependencies": { "color-name": "~1.1.4" }, "engines": { "node": ">=7.0.0" } }, "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", "dev": true }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true }, "node_modules/create-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "chalk": "^4.0.0", "exit": "^0.1.2", "graceful-fs": "^4.2.9", "jest-config": "^29.7.0", "jest-util": "^29.7.0", "prompts": "^2.0.1" }, "bin": { "create-jest": "bin/create-jest.js" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" }, "engines": { "node": ">= 8" } }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", "dev": true, "dependencies": { "ms": "^2.1.3" }, "engines": { "node": ">=6.0" }, "peerDependenciesMeta": { "supports-color": { "optional": true } } }, "node_modules/dedent": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", "dev": true, "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "peerDependenciesMeta": { "babel-plugin-macros": { "optional": true } } }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, "node_modules/deepmerge": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", "dev": true, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", "dev": true, "dependencies": { "path-type": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", "dev": true, "dependencies": { "esutils": "^2.0.2" }, "engines": { "node": ">=6.0.0" } }, "node_modules/ejs": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "dev": true, "dependencies": { "jake": "^10.8.5" }, "bin": { "ejs": "bin/cli.js" }, "engines": { "node": ">=0.10.0" } }, "node_modules/electron-to-chromium": { "version": "1.5.64", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.64.tgz", "integrity": "sha512-IXEuxU+5ClW2IGEYFC2T7szbyVgehupCWQe5GNh+H065CD6U6IFN0s4KeAMFGNmQolRU4IV7zGBWSYMmZ8uuqQ==", "dev": true }, "node_modules/emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", "dev": true, "engines": { "node": ">=12" }, "funding": { "url": "https://github.com/sindresorhus/emittery?sponsor=1" } }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", "dev": true, "dependencies": { "is-arrayish": "^0.2.1" } }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/eslint": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", "@eslint/js": "8.57.1", "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", "ajv": "^6.12.4", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", "debug": "^4.3.2", "doctrine": "^3.0.0", "escape-string-regexp": "^4.0.0", "eslint-scope": "^7.2.2", "eslint-visitor-keys": "^3.4.3", "espree": "^9.6.1", "esquery": "^1.4.2", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "find-up": "^5.0.0", "glob-parent": "^6.0.2", "globals": "^13.19.0", "graphemer": "^1.4.0", "ignore": "^5.2.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "is-path-inside": "^3.0.3", "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", "lodash.merge": "^4.6.2", "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.3", "strip-ansi": "^6.0.1", "text-table": "^0.2.0" }, "bin": { "eslint": "bin/eslint.js" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-scope": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^4.1.1" }, "engines": { "node": ">=8.0.0" } }, "node_modules/eslint-visitor-keys": { "version": "3.4.3", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint/node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint/node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, "engines": { "node": ">=4.0" } }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "dev": true, "dependencies": { "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", "eslint-visitor-keys": "^3.4.1" }, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" }, "engines": { "node": ">=4" } }, "node_modules/esquery": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz", "integrity": "sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg==", "dev": true, "dependencies": { "estraverse": "^5.1.0" }, "engines": { "node": ">=0.10" } }, "node_modules/esquery/node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, "engines": { "node": ">=4.0" } }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, "dependencies": { "estraverse": "^5.2.0" }, "engines": { "node": ">=4.0" } }, "node_modules/esrecurse/node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true, "engines": { "node": ">=4.0" } }, "node_modules/estraverse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", "dev": true, "engines": { "node": ">=4.0" } }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", "dev": true }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", "dev": true, "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", "human-signals": "^2.1.0", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.1", "onetime": "^5.1.2", "signal-exit": "^3.0.3", "strip-final-newline": "^2.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, "node_modules/exit": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", "dev": true, "engines": { "node": ">= 0.8.0" } }, "node_modules/expect": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", "dev": true, "dependencies": { "@jest/expect-utils": "^29.7.0", "jest-get-type": "^29.6.3", "jest-matcher-utils": "^29.7.0", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, "node_modules/fast-glob": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.4" }, "engines": { "node": ">=8.6.0" } }, "node_modules/fast-glob/node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "dependencies": { "is-glob": "^4.0.1" }, "engines": { "node": ">= 6" } }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "dev": true }, "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, "node_modules/fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", "dev": true, "dependencies": { "reusify": "^1.0.4" } }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", "dev": true, "dependencies": { "bser": "2.1.1" } }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, "dependencies": { "flat-cache": "^3.0.4" }, "engines": { "node": "^10.12.0 || >=12.0.0" } }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", "dev": true, "dependencies": { "minimatch": "^5.0.1" } }, "node_modules/filelist/node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/filelist/node_modules/minimatch": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { "node": ">=10" } }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/find-cache-dir": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", "integrity": "sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==", "dev": true, "dependencies": { "commondir": "^1.0.1", "make-dir": "^3.0.2", "pkg-dir": "^4.1.0" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/avajs/find-cache-dir?sponsor=1" } }, "node_modules/find-cache-dir/node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", "dev": true, "dependencies": { "semver": "^6.0.0" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/find-cache-dir/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" } }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/flat-cache": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", "dev": true, "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.3", "rimraf": "^3.0.2" }, "engines": { "node": "^10.12.0 || >=12.0.0" } }, "node_modules/flatted": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz", "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", "dev": true }, "node_modules/fs-extra": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", "dev": true, "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { "node": ">=12" } }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, "optional": true, "os": [ "darwin" ], "engines": { "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true, "engines": { "node": ">=8.0.0" } }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", "dev": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" }, "engines": { "node": "*" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, "dependencies": { "is-glob": "^4.0.3" }, "engines": { "node": ">=10.13.0" } }, "node_modules/globals": { "version": "13.24.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/globby": { "version": "11.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", "dev": true, "dependencies": { "array-union": "^2.1.0", "dir-glob": "^3.0.1", "fast-glob": "^3.2.9", "ignore": "^5.2.0", "merge2": "^1.4.1", "slash": "^3.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", "dev": true, "dependencies": { "function-bind": "^1.1.2" }, "engines": { "node": ">= 0.4" } }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, "node_modules/human-signals": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true, "engines": { "node": ">=10.17.0" } }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, "engines": { "node": ">= 4" } }, "node_modules/import-fresh": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", "dev": true, "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/import-local": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", "dev": true, "dependencies": { "pkg-dir": "^4.2.0", "resolve-cwd": "^3.0.0" }, "bin": { "import-local-fixture": "fixtures/cli.js" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true, "engines": { "node": ">=0.8.19" } }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", "dev": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true }, "node_modules/is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==", "dev": true, "dependencies": { "builtin-modules": "^3.3.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/is-core-module": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", "dev": true, "dependencies": { "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/is-generator-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, "dependencies": { "is-extglob": "^2.1.1" }, "engines": { "node": ">=0.10.0" } }, "node_modules/is-module": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", "dev": true }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, "engines": { "node": ">=0.12.0" } }, "node_modules/is-path-inside": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/is-reference": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz", "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==", "dev": true, "dependencies": { "@types/estree": "*" } }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "dev": true, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, "node_modules/istanbul-lib-coverage": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/istanbul-lib-instrument": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", "dev": true, "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", "@istanbuljs/schema": "^0.1.3", "istanbul-lib-coverage": "^3.2.0", "semver": "^7.5.4" }, "engines": { "node": ">=10" } }, "node_modules/istanbul-lib-report": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", "dev": true, "dependencies": { "istanbul-lib-coverage": "^3.0.0", "make-dir": "^4.0.0", "supports-color": "^7.1.0" }, "engines": { "node": ">=10" } }, "node_modules/istanbul-lib-source-maps": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", "dev": true, "dependencies": { "debug": "^4.1.1", "istanbul-lib-coverage": "^3.0.0", "source-map": "^0.6.1" }, "engines": { "node": ">=10" } }, "node_modules/istanbul-reports": { "version": "3.1.7", "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", "dev": true, "dependencies": { "html-escaper": "^2.0.0", "istanbul-lib-report": "^3.0.0" }, "engines": { "node": ">=8" } }, "node_modules/jake": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz", "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==", "dev": true, "dependencies": { "async": "^3.2.3", "chalk": "^4.0.2", "filelist": "^1.0.4", "minimatch": "^3.1.2" }, "bin": { "jake": "bin/cli.js" }, "engines": { "node": ">=10" } }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", "import-local": "^3.0.2", "jest-cli": "^29.7.0" }, "bin": { "jest": "bin/jest.js" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/jest-changed-files": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", "dev": true, "dependencies": { "execa": "^5.0.0", "jest-util": "^29.7.0", "p-limit": "^3.1.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-circus": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", "dev": true, "dependencies": { "@jest/environment": "^29.7.0", "@jest/expect": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "co": "^4.6.0", "dedent": "^1.0.0", "is-generator-fn": "^2.0.0", "jest-each": "^29.7.0", "jest-matcher-utils": "^29.7.0", "jest-message-util": "^29.7.0", "jest-runtime": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "p-limit": "^3.1.0", "pretty-format": "^29.7.0", "pure-rand": "^6.0.0", "slash": "^3.0.0", "stack-utils": "^2.0.3" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-cli": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", "dev": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/types": "^29.6.3", "chalk": "^4.0.0", "create-jest": "^29.7.0", "exit": "^0.1.2", "import-local": "^3.0.2", "jest-config": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "yargs": "^17.3.1" }, "bin": { "jest": "bin/jest.js" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "peerDependenciesMeta": { "node-notifier": { "optional": true } } }, "node_modules/jest-config": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", "dev": true, "dependencies": { "@babel/core": "^7.11.6", "@jest/test-sequencer": "^29.7.0", "@jest/types": "^29.6.3", "babel-jest": "^29.7.0", "chalk": "^4.0.0", "ci-info": "^3.2.0", "deepmerge": "^4.2.2", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "jest-circus": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-get-type": "^29.6.3", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-runner": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "micromatch": "^4.0.4", "parse-json": "^5.2.0", "pretty-format": "^29.7.0", "slash": "^3.0.0", "strip-json-comments": "^3.1.1" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "peerDependencies": { "@types/node": "*", "ts-node": ">=9.0.0" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, "ts-node": { "optional": true } } }, "node_modules/jest-diff": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", "dev": true, "dependencies": { "chalk": "^4.0.0", "diff-sequences": "^29.6.3", "jest-get-type": "^29.6.3", "pretty-format": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-docblock": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", "dev": true, "dependencies": { "detect-newline": "^3.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-each": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "chalk": "^4.0.0", "jest-get-type": "^29.6.3", "jest-util": "^29.7.0", "pretty-format": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-environment-node": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", "dev": true, "dependencies": { "@jest/environment": "^29.7.0", "@jest/fake-timers": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "jest-mock": "^29.7.0", "jest-util": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-get-type": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", "dev": true, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-haste-map": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "@types/graceful-fs": "^4.1.3", "@types/node": "*", "anymatch": "^3.0.3", "fb-watchman": "^2.0.0", "graceful-fs": "^4.2.9", "jest-regex-util": "^29.6.3", "jest-util": "^29.7.0", "jest-worker": "^29.7.0", "micromatch": "^4.0.4", "walker": "^1.0.8" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" }, "optionalDependencies": { "fsevents": "^2.3.2" } }, "node_modules/jest-leak-detector": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", "dev": true, "dependencies": { "jest-get-type": "^29.6.3", "pretty-format": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-matcher-utils": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", "dev": true, "dependencies": { "chalk": "^4.0.0", "jest-diff": "^29.7.0", "jest-get-type": "^29.6.3", "pretty-format": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-message-util": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", "dev": true, "dependencies": { "@babel/code-frame": "^7.12.13", "@jest/types": "^29.6.3", "@types/stack-utils": "^2.0.0", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "micromatch": "^4.0.4", "pretty-format": "^29.7.0", "slash": "^3.0.0", "stack-utils": "^2.0.3" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-mock": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", "jest-util": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-pnp-resolver": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", "dev": true, "engines": { "node": ">=6" }, "peerDependencies": { "jest-resolve": "*" }, "peerDependenciesMeta": { "jest-resolve": { "optional": true } } }, "node_modules/jest-regex-util": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", "dev": true, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-resolve": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", "dev": true, "dependencies": { "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "jest-pnp-resolver": "^1.2.2", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "resolve": "^1.20.0", "resolve.exports": "^2.0.0", "slash": "^3.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-resolve-dependencies": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", "dev": true, "dependencies": { "jest-regex-util": "^29.6.3", "jest-snapshot": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-runner": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", "dev": true, "dependencies": { "@jest/console": "^29.7.0", "@jest/environment": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "emittery": "^0.13.1", "graceful-fs": "^4.2.9", "jest-docblock": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-haste-map": "^29.7.0", "jest-leak-detector": "^29.7.0", "jest-message-util": "^29.7.0", "jest-resolve": "^29.7.0", "jest-runtime": "^29.7.0", "jest-util": "^29.7.0", "jest-watcher": "^29.7.0", "jest-worker": "^29.7.0", "p-limit": "^3.1.0", "source-map-support": "0.5.13" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-runtime": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", "dev": true, "dependencies": { "@jest/environment": "^29.7.0", "@jest/fake-timers": "^29.7.0", "@jest/globals": "^29.7.0", "@jest/source-map": "^29.6.3", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "cjs-module-lexer": "^1.0.0", "collect-v8-coverage": "^1.0.0", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "jest-haste-map": "^29.7.0", "jest-message-util": "^29.7.0", "jest-mock": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "slash": "^3.0.0", "strip-bom": "^4.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-snapshot": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", "dev": true, "dependencies": { "@babel/core": "^7.11.6", "@babel/generator": "^7.7.2", "@babel/plugin-syntax-jsx": "^7.7.2", "@babel/plugin-syntax-typescript": "^7.7.2", "@babel/types": "^7.3.3", "@jest/expect-utils": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "babel-preset-current-node-syntax": "^1.0.0", "chalk": "^4.0.0", "expect": "^29.7.0", "graceful-fs": "^4.2.9", "jest-diff": "^29.7.0", "jest-get-type": "^29.6.3", "jest-matcher-utils": "^29.7.0", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "natural-compare": "^1.4.0", "pretty-format": "^29.7.0", "semver": "^7.5.3" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-util": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "@types/node": "*", "chalk": "^4.0.0", "ci-info": "^3.2.0", "graceful-fs": "^4.2.9", "picomatch": "^2.2.3" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-validate": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", "dev": true, "dependencies": { "@jest/types": "^29.6.3", "camelcase": "^6.2.0", "chalk": "^4.0.0", "jest-get-type": "^29.6.3", "leven": "^3.1.0", "pretty-format": "^29.7.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-validate/node_modules/camelcase": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/jest-watcher": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", "dev": true, "dependencies": { "@jest/test-result": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "ansi-escapes": "^4.2.1", "chalk": "^4.0.0", "emittery": "^0.13.1", "jest-util": "^29.7.0", "string-length": "^4.0.1" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-worker": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", "dev": true, "dependencies": { "@types/node": "*", "jest-util": "^29.7.0", "merge-stream": "^2.0.0", "supports-color": "^8.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/jest-worker/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "dependencies": { "has-flag": "^4.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/supports-color?sponsor=1" } }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "node_modules/jsesc": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", "dev": true, "bin": { "jsesc": "bin/jsesc" }, "engines": { "node": ">=6" } }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true, "bin": { "json5": "lib/cli.js" }, "engines": { "node": ">=6" } }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", "dev": true, "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, "dependencies": { "json-buffer": "3.0.1" } }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "dev": true }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "dependencies": { "p-locate": "^5.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", "dev": true }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", "dev": true, "dependencies": { "yallist": "^3.0.2" } }, "node_modules/magic-string": { "version": "0.25.9", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.9.tgz", "integrity": "sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==", "dev": true, "dependencies": { "sourcemap-codec": "^1.4.8" } }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", "dev": true, "dependencies": { "semver": "^7.5.3" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", "dev": true }, "node_modules/makeerror": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", "dev": true, "dependencies": { "tmpl": "1.0.5" } }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true, "engines": { "node": ">= 8" } }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { "node": ">=8.6" } }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, "engines": { "node": "*" } }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, "node_modules/natural-compare-lite": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz", "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", "dev": true }, "node_modules/node-releases": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==", "dev": true }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", "dev": true, "dependencies": { "path-key": "^3.0.0" }, "engines": { "node": ">=8" } }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, "dependencies": { "wrappy": "1" } }, "node_modules/onetime": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", "dev": true, "dependencies": { "mimic-fn": "^2.1.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", "dev": true, "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, "dependencies": { "yocto-queue": "^0.1.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, "dependencies": { "p-limit": "^3.0.2" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "dev": true, "dependencies": { "callsites": "^3.0.0" }, "engines": { "node": ">=6" } }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", "json-parse-even-better-errors": "^2.3.0", "lines-and-columns": "^1.1.6" }, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", "dev": true }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "dev": true }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, "engines": { "node": ">=8.6" }, "funding": { "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/pirates": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", "dev": true, "engines": { "node": ">= 6" } }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", "dev": true, "dependencies": { "find-up": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/pkg-dir/node_modules/find-up": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/pkg-dir/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, "dependencies": { "p-locate": "^4.1.0" }, "engines": { "node": ">=8" } }, "node_modules/pkg-dir/node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", "dev": true, "dependencies": { "p-try": "^2.0.0" }, "engines": { "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/pkg-dir/node_modules/p-locate": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, "dependencies": { "p-limit": "^2.2.0" }, "engines": { "node": ">=8" } }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", "dev": true, "engines": { "node": ">= 0.8.0" } }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", "dev": true, "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" }, "engines": { "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, "node_modules/pretty-format/node_modules/ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", "dev": true, "dependencies": { "kleur": "^3.0.3", "sisteransi": "^1.0.5" }, "engines": { "node": ">= 6" } }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", "dev": true, "funding": [ { "type": "individual", "url": "https://github.com/sponsors/dubzzz" }, { "type": "opencollective", "url": "https://opencollective.com/fast-check" } ] }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true, "funding": [ { "type": "github", "url": "https://github.com/sponsors/feross" }, { "type": "patreon", "url": "https://www.patreon.com/feross" }, { "type": "consulting", "url": "https://feross.org/support" } ] }, "node_modules/react-is": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", "dev": true }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/resolve": { "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", "dev": true, "dependencies": { "is-core-module": "^2.13.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, "bin": { "resolve": "bin/resolve" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", "dev": true, "dependencies": { "resolve-from": "^5.0.0" }, "engines": { "node": ">=8" } }, "node_modules/resolve-cwd/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true, "engines": { "node": ">=4" } }, "node_modules/resolve.exports": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", "integrity": "sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==", "dev": true, "engines": { "node": ">=10" } }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true, "engines": { "iojs": ">=1.0.0", "node": ">=0.10.0" } }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "bin.js" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/rollup": { "version": "2.79.2", "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.79.2.tgz", "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", "dev": true, "bin": { "rollup": "dist/bin/rollup" }, "engines": { "node": ">=10.0.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "node_modules/rollup-plugin-typescript2": { "version": "0.34.1", "resolved": "https://registry.npmjs.org/rollup-plugin-typescript2/-/rollup-plugin-typescript2-0.34.1.tgz", "integrity": "sha512-P4cHLtGikESmqi1CA+tdMDUv8WbQV48mzPYt77TSTOPJpERyZ9TXdDgjSDix8Fkqce6soYz3+fa4lrC93IEkcw==", "dev": true, "dependencies": { "@rollup/pluginutils": "^4.1.2", "find-cache-dir": "^3.3.2", "fs-extra": "^10.0.0", "semver": "^7.3.7", "tslib": "^2.4.0" }, "peerDependencies": { "rollup": ">=1.26.3", "typescript": ">=2.4.0" } }, "node_modules/rollup-plugin-typescript2/node_modules/@rollup/pluginutils": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-4.2.1.tgz", "integrity": "sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ==", "dev": true, "dependencies": { "estree-walker": "^2.0.1", "picomatch": "^2.2.2" }, "engines": { "node": ">= 8.0.0" } }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", "dev": true, "funding": [ { "type": "github", "url": "https://github.com/sponsors/feross" }, { "type": "patreon", "url": "https://www.patreon.com/feross" }, { "type": "consulting", "url": "https://feross.org/support" } ], "dependencies": { "queue-microtask": "^1.2.2" } }, "node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "dev": true, "bin": { "semver": "bin/semver.js" }, "engines": { "node": ">=10" } }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, "engines": { "node": ">=8" } }, "node_modules/shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "dev": true }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/source-map-support": { "version": "0.5.13", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", "dev": true, "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "node_modules/sourcemap-codec": { "version": "1.4.8", "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", "deprecated": "Please use @jridgewell/sourcemap-codec instead", "dev": true }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", "dev": true, "dependencies": { "escape-string-regexp": "^2.0.0" }, "engines": { "node": ">=10" } }, "node_modules/stack-utils/node_modules/escape-string-regexp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", "dev": true, "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" }, "engines": { "node": ">=10" } }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" }, "engines": { "node": ">=8" } }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, "engines": { "node": ">=8" } }, "node_modules/strip-bom": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true, "engines": { "node": ">=6" } }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true, "engines": { "node": ">=8" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", "dev": true, "dependencies": { "has-flag": "^4.0.0" }, "engines": { "node": ">=8" } }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "dev": true, "engines": { "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", "dev": true, "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", "minimatch": "^3.0.4" }, "engines": { "node": ">=8" } }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, "dependencies": { "is-number": "^7.0.0" }, "engines": { "node": ">=8.0" } }, "node_modules/ts-jest": { "version": "29.2.5", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz", "integrity": "sha512-KD8zB2aAZrcKIdGk4OwpJggeLcH1FgrICqDSROWqlnJXGCXK4Mn6FcdK2B6670Xr73lHMG1kHw8R87A0ecZ+vA==", "dev": true, "dependencies": { "bs-logger": "^0.2.6", "ejs": "^3.1.10", "fast-json-stable-stringify": "^2.1.0", "jest-util": "^29.0.0", "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", "semver": "^7.6.3", "yargs-parser": "^21.1.1" }, "bin": { "ts-jest": "cli.js" }, "engines": { "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" }, "peerDependencies": { "@babel/core": ">=7.0.0-beta.0 <8", "@jest/transform": "^29.0.0", "@jest/types": "^29.0.0", "babel-jest": "^29.0.0", "jest": "^29.0.0", "typescript": ">=4.3 <6" }, "peerDependenciesMeta": { "@babel/core": { "optional": true }, "@jest/transform": { "optional": true }, "@jest/types": { "optional": true }, "babel-jest": { "optional": true }, "esbuild": { "optional": true } } }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "dev": true }, "node_modules/tsutils": { "version": "3.21.0", "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", "integrity": "sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==", "dev": true, "dependencies": { "tslib": "^1.8.1" }, "engines": { "node": ">= 6" }, "peerDependencies": { "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, "node_modules/tsutils/node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", "dev": true, "dependencies": { "prelude-ls": "^1.2.1" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "dev": true, "engines": { "node": ">=4" } }, "node_modules/type-fest": { "version": "0.20.2", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/typescript": { "version": "4.9.5", "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { "node": ">=4.2.0" } }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", "dev": true }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "dev": true, "engines": { "node": ">= 10.0.0" } }, "node_modules/update-browserslist-db": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz", "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==", "dev": true, "funding": [ { "type": "opencollective", "url": "https://opencollective.com/browserslist" }, { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" }, { "type": "github", "url": "https://github.com/sponsors/ai" } ], "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.0" }, "bin": { "update-browserslist-db": "cli.js" }, "peerDependencies": { "browserslist": ">= 4.21.0" } }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, "dependencies": { "punycode": "^2.1.0" } }, "node_modules/v8-to-istanbul": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", "dev": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", "convert-source-map": "^2.0.0" }, "engines": { "node": ">=10.12.0" } }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", "dev": true, "dependencies": { "makeerror": "1.0.12" } }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "bin/node-which" }, "engines": { "node": ">= 8" } }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", "dev": true, "engines": { "node": ">=0.10.0" } }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" }, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, "node_modules/write-file-atomic": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", "dev": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^3.0.7" }, "engines": { "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "dev": true, "engines": { "node": ">=10" } }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" }, "engines": { "node": ">=12" } }, "node_modules/yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", "dev": true, "engines": { "node": ">=12" } }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true, "engines": { "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } } } } xgrammar-0.1.19/web/package.json000066400000000000000000000021061500705317600164660ustar00rootroot00000000000000{ "name": "@mlc-ai/web-xgrammar", "version": "0.1.0", "description": "", "main": "lib/index.js", "types": "lib/index.d.ts", "type": "module", "scripts": { "build": "./build.sh; rollup -c", "lint": "npx eslint .", "test": "./run_test.sh" }, "files": [ "lib" ], "repository": { "type": "git", "url": "git+https://github.com/mlc-ai/xgrammar" }, "keywords": [ "machine_learning", "llm", "nlp" ], "license": "Apache-2.0", "homepage": "https://github.com/mlc-ai/xgrammar/tree/main/web", "devDependencies": { "@jest/globals": "^29.7.0", "@rollup/plugin-commonjs": "^20.0.0", "@rollup/plugin-node-resolve": "^13.0.4", "@rollup/plugin-wasm": "^5.1.2", "@types/jest": "^29.5.12", "@typescript-eslint/eslint-plugin": "^5.59.6", "@typescript-eslint/parser": "^5.59.6", "eslint": "^8.41.0", "jest": "^29.7.0", "rollup": "^2.56.2", "rollup-plugin-typescript2": "^0.34.1", "ts-jest": "^29.2.5", "tslib": "^2.3.1", "typescript": "^4.9.5", "@mlc-ai/web-tokenizers": "^0.1.5" } } xgrammar-0.1.19/web/rollup.config.js000066400000000000000000000011771500705317600173260ustar00rootroot00000000000000import { nodeResolve } from '@rollup/plugin-node-resolve'; import commonjs from '@rollup/plugin-commonjs'; import { wasm } from '@rollup/plugin-wasm'; import typescript from 'rollup-plugin-typescript2' export default { input: 'src/index.ts', output: [ { file: 'lib/index.js', exports: 'named', name: 'xgrammar', format: 'umd', sourcemap: true } ], plugins: [ nodeResolve({ browser: true }), commonjs(), wasm(), typescript({ rollupCommonJSResolveHack: false, clean: true }) ] }; xgrammar-0.1.19/web/run_test.sh000077500000000000000000000016061500705317600164060ustar00rootroot00000000000000# We cannot naively run `yarn test` due to error: # `TypeError: A dynamic import callback was invoked without --experimental-vm-modules` # Thus, we need to run `node --experimental-vm-modules node_modules/jest/bin/jest` # We also need to change to `commonjs` to avoid error `Must use import to load ES Module`. # Thus we make this edit, run test, and edit back. # We also need to make this change for web-tokenizers sed -e s/'"type": "module",'/'"type": "commonjs",'/g -i .backup package.json sed -e s/'"type": "module",'/'"type": "commonjs",'/g -i .backup node_modules/@mlc-ai/web-tokenizers/package.json node --experimental-vm-modules node_modules/jest/bin/jest sed -e s/'"type": "commonjs",'/'"type": "module",'/g -i .backup package.json sed -e s/'"type": "commonjs",'/'"type": "module",'/g -i .backup node_modules/@mlc-ai/web-tokenizers/package.json # Cleanup backup files rm package.json.backup xgrammar-0.1.19/web/src/000077500000000000000000000000001500705317600147705ustar00rootroot00000000000000xgrammar-0.1.19/web/src/index.ts000066400000000000000000000004651500705317600164540ustar00rootroot00000000000000import { Grammar, GrammarCompiler, CompiledGrammar, GrammarMatcher, TokenizerInfo, Testings } from "./xgrammar" export { Grammar, GrammarCompiler, CompiledGrammar, GrammarMatcher, TokenizerInfo, Testings } export default { Grammar, GrammarCompiler, CompiledGrammar, GrammarMatcher, TokenizerInfo, Testings } xgrammar-0.1.19/web/src/xgrammar.ts000066400000000000000000000474421500705317600171710ustar00rootroot00000000000000import Module from "./xgrammar_binding"; let binding: any = null; async function asyncInitBinding() { if (binding == null) { binding = await Module(); } } /** * Various testing methods that are not optimized for performance. */ export class Testings { /** * Convert JSON schema string to EBNF grammar string. For test purposes. * * @param {string} schema The schema string. * @param {number} [indent=2] The number of spaces for indentation. If -1, the grammar will * enforce the output to be in one line. * @param {[string, string]} [separators] Two separators that will be enforced by the grammar: * comma and colon. Examples: (",", ":"), (", ", ": "). If undefined, the default separators will * be used: (",", ": ") when the indent is not undefined, and (", ", ": ") otherwise. This follows * the convention in Python's json.dumps(). Currently unsupported and will use the default value. * @param {boolean} [strictMode=true] Whether to use strict mode. In strict mode, the generated * grammar will not allow properties and items that is not specified in the schema. This is * equivalent to setting unevaluatedProperties and unevaluatedItems to false. * @returns {string} The EBNF grammar string. */ static async _jsonSchemaToEBNF( schema: string, indent = 2, separators?: [string, string], strictMode = true ): Promise { // TODO(Charlie): Add support for separators, which requires binding std::pair // in emscripten if (separators !== undefined) { throw new Error( `Argument separators is not supported yet, please leave it as undefined, and the ` + `default value (",", ": ") will be used.` ); } await asyncInitBinding(); // indent being -1 is equivalent to not having a value for the std::optional arg in C++. // This is a workaround to Typescript not being able to express Optional value like Python; if // user specifies indent to be undefined, it still becomes 2. let optionalIndent: number | undefined = indent == -1 ? undefined : indent; return binding._JSONSchemaToEBNF(schema, optionalIndent, separators, strictMode); } /** * * @param {Int32Array} bitmask Bitmask returned by getNextTokenBitmask(). * @param {number} vocabSize Vocab size returned by getVocabSize(). * @param {number} index The batch index of the bitmask. For batch inference, bitmask[index] will * be used. Defaults to 0. * @returns An array of vocab ID that will be rejected as a result of the bitmask. */ static async debugGetMaskedTokensFromBitmask( bitmask: Int32Array, vocabSize: number, index: number = 0, ): Promise { await asyncInitBinding(); const bitmaskIntVector = binding.vecIntFromJSArray(bitmask); const rejectedIDsIntVector = binding.DebugGetMaskedTokensFromBitmask( bitmaskIntVector, vocabSize, index ); bitmaskIntVector.delete(); const rejectedIDsInt32Array = binding.vecIntToView(rejectedIDsIntVector).slice(); rejectedIDsIntVector.delete(); return rejectedIDsInt32Array; } } /** * This class stores the abstract syntax tree (AST) of the Backus-Naur Form (BNF) grammar and * provides utilities to parse and print the AST. User should provide a BNF/EBNF (Extended * Backus-Naur Form) grammar, and use from_ebnf_string to parse and simplify the grammar into an * AST of BNF grammar. */ export class Grammar { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of Grammar created by binding. */ constructor(handle: any) { this.handle = handle; } /** * Dispose this Grammar. */ dispose() { this.handle.delete(); } /** * Construct a BNF grammar with a EBNF-formatted string. The grammar will be normalized * (simplified) by default. * EBNF grammar: see https://www.w3.org/TR/xml/#sec-notation. Note: * 1. Use # as the comment mark * 2. Use C-style unicode escape sequence \u01AB, \U000001AB, \xAB * 3. A-B (match A and not match B) is not supported yet * 4. Lookahead assertion can be added at the end of a rule to speed up matching. E.g. * ``` * root ::= "ab" a [a-z] * a ::= "cd" (=[a-z]) * ``` * The assertion (=[a-z]) means a must be followed by [a-z]. * @param {string} ebnfString The grammar string * @param {string} [rootRule="root"] The name of the root rule. Default: "root". * @returns {Grammar} The parsed BNF grammar. */ static async fromEBNF(ebnfString: string, rootRule = "root"): Promise { await asyncInitBinding(); return new Grammar(new binding.Grammar.FromEBNF(ebnfString, rootRule)); } /** * Get the grammar of standard JSON. * @returns {Grammar} The JSON grammar. */ static async builtinJSONGrammar(): Promise { await asyncInitBinding(); return new Grammar(new binding.Grammar.BuiltinJSONGrammar()); } /** * Construct a BNF grammar from the json schema string. The schema string should be in the * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. * * @param {string} schema The schema string. * @param {number} [indent=2] The number of spaces for indentation. If -1, the grammar will * enforce the output to be in one line. * @param {[string, string]} [separators] Two separators that will be enforced by the grammar: * comma and colon. Examples: (",", ":"), (", ", ": "). If undefined, the default separators will * be used: (",", ": ") when the indent is not undefined, and (", ", ": ") otherwise. This follows * the convention in Python's json.dumps(). Currently unsupported and will use the default value. * @param {boolean} [strictMode=true] Whether to use strict mode. In strict mode, the generated * grammar will not allow properties and items that is not specified in the schema. This is * equivalent to setting unevaluatedProperties and unevaluatedItems to false. * @returns {Grammar} The generated BNF grammar. */ static async fromJSONSchema( schema: string, indent = 2, separators?: [string, string], strictMode = true ): Promise { // TODO(Charlie): Add support for separators, which requires binding std::pair // in emscripten if (separators !== undefined) { throw new Error( `Argument separators is not supported yet, please leave it as undefined, and the ` + `default value (",", ": ") will be used.` ); } await asyncInitBinding(); // indent being -1 is equivalent to not having a value for the std::optional arg in C++. // This is a workaround to Typescript not being able to express Optional value like Python; if // user specifies indent to be undefined, it still becomes 2. let optionalIndent: number | undefined = indent == -1 ? undefined : indent; return new Grammar( new binding.Grammar.FromJSONSchema(schema, optionalIndent, separators, strictMode)); } /** * Print the BNF grammar to a string, in standard BNF format. * @returns The BNF grammar string. */ toString(): string { return this.handle.ToString(); } } /** * A class that wraps a preprocessed vocab, needed to instantiate GrammarCompiler. */ export class TokenizerInfo { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of TokenizerInfo created by binding. */ constructor(handle: any) { this.handle = handle; }; /** * Dispose this tokenizer info object. */ dispose() { this.handle.delete(); } /** * Get the vocab size. */ getVocabSize(): number { return this.handle.GetVocabSize(); } /** * Get the post-processed vocab. Returned as a handle of type binding.VectorString */ getDecodedVocabHandle(): any { return this.handle.GetDecodedVocab(); } /** * Instantiate with raw vocab and the vocab type by internally post-processing * the raw vocab by decoding each token with the provided vocab type. * @param {string[]} encodedVocab: the vocab in the form of a string list of tokens, * ordered by their token id. It should include all the special tokens. * @param {string} vocabType: either "byte_fallback", "byte_level", or `raw`. See `tokenizer.cc` * for its semantic. * @param {boolean} prependSpaceInTokenization: whether the tokenizer will prepend a space before * the text in the tokenization process. * @param {number} vocabSize: the full vocab size read from `config.json`. If not provided, will * use length of `encodedVocab`. Note some model has a vocab size larger in `config.json` due * to padding. Essentially the size of the logits. * @param {number[] | number} [stopTokenIds=undefined] Stop tokens to override the default ones. */ static async createTokenizerInfo( encodedVocab: string[], vocabType: string, prependSpaceInTokenization: boolean, vocabSize?: number, stopTokenIds?: number[] | number, ): Promise { await asyncInitBinding(); // Convert string[] to std::vector const encodedVocabVec = binding.vecStringFromJSArray(encodedVocab); // Convert stopTokenIds to std::vector if not undefined if (stopTokenIds !== undefined) { if (!Array.isArray(stopTokenIds)) { stopTokenIds = [stopTokenIds]; } stopTokenIds = binding.vecIntFromJSArray(stopTokenIds); } // Instantiate TokenizerInfo return new TokenizerInfo(new binding.TokenizerInfo( encodedVocabVec, vocabType.toUpperCase(), vocabSize, stopTokenIds, prependSpaceInTokenization, )); } } export class CompiledGrammar { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of CompiledGrammar created by binding. */ constructor(handle: any) { this.handle = handle; }; /** * Dispose this compiled grammar object. */ dispose() { this.handle.delete(); } /** * @returns {Grammar} The grammar used to compile this CompiledGrammar. */ grammar(): Grammar { return new Grammar(this.handle.GetGrammar()); } /** * @returns {TokenizerInfo} The tokenizer info used to compile this CompiledGrammar. */ tokenizerInfo(): TokenizerInfo { return new TokenizerInfo(this.handle.GetTokenizerInfo()); } } export class GrammarCompiler { handle: any; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of GrammarCompiler created by binding. */ private constructor(handle: any) { this.handle = handle; }; /** * Dispose this grammar compiler object. */ dispose() { this.handle.delete(); }; /** * * @param tokenizerInfo {TokenizerInfo} The tokenizer info that contains preprocessed vocab. * @param cacheEnabled {boolean} Whether to enable caching. Default is true. */ static async createGrammarCompiler( tokenizerInfo: TokenizerInfo, cacheEnabled: boolean = true, ): Promise { await asyncInitBinding(); // NOTE(Charlie): Have not figured out how to do multithreading in WASM, so always set to 1. return new GrammarCompiler(new binding.GrammarCompiler( tokenizerInfo.handle, /**max_threads=*/1, cacheEnabled )); } /** * Get CompiledGrammar from the json schema string. The schema string should be in the * format of the schema of a JSON file. We will parse the schema and generate a BNF grammar. * * @param {string} schema The schema string. * @param {number} [indent=2] The number of spaces for indentation. If -1, the grammar will * enforce the output to be in one line. * @param {[string, string]} [separators] Two separators that will be enforced by the grammar: * comma and colon. Examples: (",", ":"), (", ", ": "). If undefined, the default separators will * be used: (",", ": ") when the indent is not undefined, and (", ", ": ") otherwise. This follows * the convention in Python's json.dumps(). Currently unsupported and will use the default value. * @param {boolean} [strictMode=true] Whether to use strict mode. In strict mode, the generated * grammar will not allow properties and items that is not specified in the schema. This is * equivalent to setting unevaluatedProperties and unevaluatedItems to false. * @returns {CompiledGrammar} The compiled grammar for the specified JSON schema. */ async compileJSONSchema( schema: string, indent = 2, separators?: [string, string], strictMode = true ): Promise { // TODO(Charlie): Add support for separators, which requires binding std::pair // in emscripten if (separators !== undefined) { throw new Error( `Argument separators is not supported yet, please leave it as undefined, and the ` + `default value (",", ": ") will be used.` ); } await asyncInitBinding(); // indent being -1 is equivalent to not having a value for the std::optional arg in C++. // This is a workaround to Typescript not being able to express Optional value like Python; if // user specifies indent to be undefined, it still becomes 2. let optionalIndent: number | undefined = indent == -1 ? undefined : indent; return new CompiledGrammar( this.handle.CompileJSONSchema(schema, optionalIndent, separators, strictMode)); } /** * @returns {CompiledGrammar} The compiled grammar for JSON. */ async compileBuiltinJSONGrammar(): Promise { await asyncInitBinding(); return new CompiledGrammar(this.handle.CompileBuiltinJSONGrammar()); } /** * Get CompiledGrammar from the EBNF-formatted string. The grammar will be normalized * (simplified) by default. * EBNF grammar: see https://www.w3.org/TR/xml/#sec-notation. Note: * 1. Use # as the comment mark * 2. Use C-style unicode escape sequence \u01AB, \U000001AB, \xAB * 3. A-B (match A and not match B) is not supported yet * 4. Lookahead assertion can be added at the end of a rule to speed up matching. E.g. * ``` * root ::= "ab" a [a-z] * a ::= "cd" (=[a-z]) * ``` * The assertion (=[a-z]) means a must be followed by [a-z]. * @param {string} ebnfString The grammar string * @param {string} [rootRule="root"] The name of the root rule. Default: "root". * @returns {CompiledGrammar} The compiled grammar for the specified EBNF string. */ async compileGrammar(grammar: Grammar): Promise; async compileGrammar(grammar: string, rootRule?: string): Promise; async compileGrammar(grammar: string | Grammar, rootRule: string="root"): Promise { await asyncInitBinding(); if (typeof grammar === "string") { const grammarObj = await Grammar.fromEBNF(grammar, rootRule); return new CompiledGrammar(this.handle.CompileGrammar(grammarObj.handle)); } else { return new CompiledGrammar(this.handle.CompileGrammar(grammar.handle)); } } } /** * A stateful matcher to match tokens to the specified BNF grammar. This class is the core logic * of the grammar-guided generation. * * This class implements the non-deterministic pushdown automaton (NPDA) matching algorithm to * match characters to a BNF grammar. It keep track of the current state of the matching process by * maintaining several stacks internally as possible paths in the NPDA. It also supports * backtracking. * * It is particularly capable of finding the set of tokens that are acceptable for the next step * and storing them in a bitmask. This aids in grammar-guided generation. */ export class GrammarMatcher { private handle: any; private vocab_size: number; /** * @internal * Private constructor. Factory methods are used since binding initialization is asynchronous. * @param {any} handle handle of GrammarMatcher created by binding. */ private constructor(handle: any, vocab_size: number) { this.handle = handle; this.vocab_size = vocab_size; } /** * Dispose this grammar state matcher. */ dispose() { this.handle.delete(); } /** * Construct a GrammarMatcher. * @param {CompiledGrammar} compiledGrammar A compiled grammar from GrammarCompiler. * @param {number[] | number} [overrideStopTokens=undefined] Stop tokens to override the default ones. * @param {boolean} [terminateWithoutStopToken=false] Whether to terminate without stop token. * @param {number} [maxRollbackTokens=0] Max rollback tokens. * @returns {GrammarMatcher} The constructed GrammarMatcher. */ static async createGrammarMatcher( compiledGrammar: CompiledGrammar, overrideStopTokens?: number[] | number, terminateWithoutStopToken: boolean = false, maxRollbackTokens: number = 0, ): Promise { await asyncInitBinding(); // Convert overrideStopTokens to std::vector if not undefined if (overrideStopTokens !== undefined) { if (!Array.isArray(overrideStopTokens)) { overrideStopTokens = [overrideStopTokens]; } overrideStopTokens = binding.vecIntFromJSArray(overrideStopTokens); } return new GrammarMatcher(new binding.GrammarMatcher( compiledGrammar.handle, overrideStopTokens, terminateWithoutStopToken, maxRollbackTokens, ), compiledGrammar.tokenizerInfo().getVocabSize()); } /** * Get the maximum number of rollback tokens allowed. */ getMaxRollbackTokens(): number { return this.handle.GetMaxRollbackTokens(); } /** * Accept one token and update the state of the matcher. * @param {number} tokenID The id of the token to accept. * @param {boolean} [verbose=false] To print debugging info * @returns {boolean} Whether the token is accepted. */ acceptToken(tokenID: number, verbose: boolean = false): boolean { return this.handle.AcceptToken(tokenID, verbose); } /** * Accept one unicode codepoint to the current state. For test purposes. * @param {string} inputStr The unicode codepoint of the character to be accepted. * @param {boolean} [verbose=false] To print debugging info * @returns {boolean} Whether the input string is accepted. */ _debugAcceptString(inputStr: string, verbose: boolean = false): boolean { return this.handle._DebugAcceptString(inputStr, verbose); } /** * Returns a bitmask in the form of an Int32Array of length ceildiv(vocab_size, 32) * based on what tokens can/cannot be accepted by the current state of the grammar state matcher. * * @returns {Int32Array} An array representing the bitmask that masks the rejected token IDs */ async getNextTokenBitmask(): Promise { await asyncInitBinding(); // a handle of std::vector const maskIntVector = this.handle.GetNextTokenBitmask(this.vocab_size) const maskInt32Array = binding.vecIntToView(maskIntVector).slice(); maskIntVector.delete(); return maskInt32Array; } /** * Check if the matcher has accepted the stop token and terminated. See also * GrammarMatcher.acceptToken. */ isTerminated(): boolean { return this.handle.IsTerminated(); } /** * Reset the matcher to the initial state. */ reset(): void { this.handle.Reset(); } /** * Find the jump-forward string for jump-forward decoding. This is the longest string that * will be valid according to the current syntax. * @returns {string} The jump-forward string. */ findJumpForwardString(): string { return this.handle.FindJumpForwardString(); } /** * Rollback the matcher to a previous state. * @param {number} numTokens The number of tokens to rollback. It cannot exceed the current * number of steps, nor can it exceed the specified maximum number of rollback tokens. */ rollBack(numTokens: number): void { this.handle.Rollback(numTokens); } } xgrammar-0.1.19/web/src/xgrammar_binding.cc000066400000000000000000000145221500705317600206130ustar00rootroot00000000000000/* * \file xgrammar_binding.cc * \brief XGrammar wasm runtime library pack. */ // Configuration for XGRAMMAR_LOG() #define XGRAMMAR_LOG_CUSTOMIZE 1 #include #include #include #include #include #include "../../cpp/testing.h" // #include "../../cpp/support/logging.h" namespace xgrammar { // Override logging mechanism [[noreturn]] void LogFatalImpl(const std::string& file, int lineno, const std::string& message) { std::cerr << "[FATAL] " << file << ":" << lineno << ": " << message << std::endl; abort(); } void LogMessageImpl(const std::string& file, int lineno, int level, const std::string& message) { static const char* level_strings_[] = { "[INFO] ", "[DEBUG] ", "[WARNING] ", }; std::cout << level_strings_[level] << file << ":" << lineno << ": " << message << std::endl; } } // namespace xgrammar using namespace emscripten; using namespace xgrammar; TokenizerInfo TokenizerInfo_Init( const std::vector& encoded_vocab, std::string vocab_type, std::optional vocab_size, std::optional> stop_token_ids, bool add_prefix_space ) { static const std::unordered_map VOCAB_TYPE_MAP = { {"RAW", VocabType::RAW}, {"BYTE_FALLBACK", VocabType::BYTE_FALLBACK}, {"BYTE_LEVEL", VocabType::BYTE_LEVEL}, }; return TokenizerInfo( encoded_vocab, VOCAB_TYPE_MAP.at(vocab_type), vocab_size, stop_token_ids, add_prefix_space ); } GrammarMatcher GrammarMatcher_Init( const CompiledGrammar& grammar, std::optional> override_stop_tokens, bool terminate_without_stop_token, int max_rollback_tokens ) { return GrammarMatcher( grammar, override_stop_tokens, terminate_without_stop_token, max_rollback_tokens ); } /*! * \brief Finds the next token bitmask of the matcher. */ std::vector GrammarMatcher_GetNextTokenBitmask(GrammarMatcher& matcher, int vocab_size) { // 1. Initialize std::vector result auto buffer_size = GetBitmaskSize(vocab_size); std::vector result(buffer_size); // 2. Initialize DLTensor with the data pointer of the std vector. DLTensor tensor; tensor.data = result.data(); tensor.device = DLDevice{kDLCPU, 0}; tensor.ndim = 1; tensor.dtype = DLDataType{kDLInt, 32, 1}; // int32 std::vector shape = {buffer_size}; tensor.shape = &shape[0]; std::vector strides = {1}; tensor.strides = &strides[0]; tensor.byte_offset = 0; // 3. Populate tensor, hence result matcher.FillNextTokenBitmask(&tensor); return result; } /*! * \brief Return the list of rejected token IDs based on the bit mask. * \note This method is mainly used in testing, so performance is not as important. */ std::vector Testing_DebugGetMaskedTokensFromBitmask( std::vector token_bitmask, size_t vocab_size, int index ) { // 1. Convert token_bitmask into DLTensor DLTensor tensor; tensor.data = token_bitmask.data(); tensor.device = DLDevice{kDLCPU, 0}; tensor.ndim = 1; tensor.dtype = DLDataType{kDLInt, 32, 1}; // int32 std::vector shape = {token_bitmask.size()}; tensor.shape = &shape[0]; std::vector strides = {1}; tensor.strides = &strides[0]; tensor.byte_offset = 0; // 2. Get rejected token IDs std::vector result; _DebugGetMaskedTokensFromBitmask(&result, tensor, vocab_size, index); return result; } /*! * \brief Helps view an std::vector handle as Int32Array in JS without copying. */ emscripten::val vecIntToView(const std::vector& vec) { return emscripten::val(typed_memory_view(vec.size(), vec.data())); } EMSCRIPTEN_BINDINGS(xgrammar) { // Register std::optional used in Grammar::FromJSONSchema register_optional(); register_optional>(); // Register std::vector for TokenizerInfo.GetDecodedVocab() register_vector("VectorString"); function( "vecStringFromJSArray", select_overload(const emscripten::val&)>(&vecFromJSArray) ); // Register std::optional> for GrammarMatcher_Init register_vector("VectorInt"); register_optional>(); function( "vecIntFromJSArray", select_overload(const emscripten::val&)>(&vecFromJSArray) ); // Register view so we can read std::vector as Int32Array in JS without copying function("vecIntToView", &vecIntToView); // Testing methods function("_JSONSchemaToEBNF", &_JSONSchemaToEBNF); function("DebugGetMaskedTokensFromBitmask", &Testing_DebugGetMaskedTokensFromBitmask); class_("Grammar") .function("ToString", &Grammar::ToString) .class_function("FromEBNF", &Grammar::FromEBNF) .class_function("FromJSONSchema", &Grammar::FromJSONSchema) .class_function("BuiltinJSONGrammar", &Grammar::BuiltinJSONGrammar); class_("TokenizerInfo") .constructor(&TokenizerInfo_Init) .function("GetVocabSize", &TokenizerInfo::GetVocabSize) .function("GetDecodedVocab", &TokenizerInfo::GetDecodedVocab); class_("CompiledGrammar") .function("GetGrammar", &CompiledGrammar::GetGrammar) .function("GetTokenizerInfo", &CompiledGrammar::GetTokenizerInfo); class_("GrammarCompiler") .constructor() .function("CompileJSONSchema", &GrammarCompiler::CompileJSONSchema) .function("CompileBuiltinJSONGrammar", &GrammarCompiler::CompileBuiltinJSONGrammar) .function("CompileGrammar", &GrammarCompiler::CompileGrammar) .function("ClearCache", &GrammarCompiler::ClearCache); class_("GrammarMatcher") .constructor(&GrammarMatcher_Init) .smart_ptr>("GrammarMatcher") .function("GetMaxRollbackTokens", &GrammarMatcher::GetMaxRollbackTokens) .function("AcceptToken", &GrammarMatcher::AcceptToken) .function("GetNextTokenBitmask", &GrammarMatcher_GetNextTokenBitmask) .function("IsTerminated", &GrammarMatcher::IsTerminated) .function("Reset", &GrammarMatcher::Reset) .function("FindJumpForwardString", &GrammarMatcher::FindJumpForwardString) .function("Rollback", &GrammarMatcher::Rollback) .function("_DebugAcceptString", &GrammarMatcher::_DebugAcceptString); } xgrammar-0.1.19/web/tests/000077500000000000000000000000001500705317600153435ustar00rootroot00000000000000xgrammar-0.1.19/web/tests/grammar.test.ts000066400000000000000000000622151500705317600203250ustar00rootroot00000000000000/** * Test all APIs exposed in the web-xgrammar package. The goal of these unit tests * are to test each API works as expected. It does not test behavior correctness * thoroughly since that is done in `tests/python`. */ import { describe, expect, test } from "@jest/globals"; import { Grammar, GrammarCompiler, CompiledGrammar, TokenizerInfo, GrammarMatcher, Testings } from ".."; import { Tokenizer } from "@mlc-ai/web-tokenizers"; async function getTokenizerInfoFromUrl(tokenizerUrl: string, vocabType: string, prependSpace: boolean): Promise { // 1. Get tokenizer const jsonBuffer = await (await fetch(tokenizerUrl)).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get raw vocab const encodedVocab: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 3. Decode const decodedVocab = await TokenizerInfo.createTokenizerInfo(encodedVocab, vocabType, prependSpace); return decodedVocab; } describe("Test all Grammar APIs", () => { const ebnf_grammar = String.raw`basic_escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] basic_string_sub ::= ("\"" | [^"\\\r\n] basic_string_sub | "\\" basic_escape basic_string_sub) (= [ \n\t]* [,}\]:]) basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? basic_string ::= ["] basic_string_sub basic_boolean ::= "true" | "false" basic_null ::= "null" basic_array ::= ("[" "" basic_any (", " basic_any)* "" "]") | "[]" basic_object ::= ("{" "" basic_string ": " basic_any (", " basic_string ": " basic_any)* "" "}") | "{}" root_prop_1 ::= basic_boolean | basic_null root_prop_2 ::= basic_number | basic_null root ::= "{" "" ("\"num\"" ": " basic_integer ", ")? ("\"opt_bool\"" ": " root_prop_1 ", ")? "\"size\"" ": " root_prop_2 (", " "\"name\"" ": " basic_string)? "" "}" `; /** * Equivalent to class MainModel(BaseModel): num: int = 0 opt_bool: Optional[bool] = None size: Optional[float] name: str = "" */ const schema = String.raw`{"properties": {"num": {"default": 0, "title": "Num", "type": "integer"}, "opt_bool": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "default": null, "title": "Opt Bool"}, "size": {"anyOf": [{"type": "number"}, {"type": "null"}], "title": "Size"}, "name": {"default": "", "title": "Name", "type": "string"}}, "required": ["size"], "title": "MainModel", "type": "object"}`; test("Test Grammar.fromEBNF and Grammar.toString", async () => { const before = `root ::= ((b c) | (b root)) b ::= ((b_1 d [a]*)) c ::= ((c_1)) d ::= ((d_1)) b_1 ::= ("" | ("b" b_1)) c_1 ::= ((c_2 c_1) | (c_2)) c_2 ::= (([acep-z])) d_1 ::= ("" | ("d")) `; const grammar1 = await Grammar.fromEBNF(before) const outputStr = grammar1.toString(); expect(outputStr).toEqual(before); }); test("Test Grammar.fromJSONSchema()", async () => { const grammar = await Grammar.fromJSONSchema(schema); const outputStr = grammar.toString(); expect(outputStr == "").toEqual(false); }); test("Test _jsonSchemaToEBNF", async () => { // Equivalent to test_optional() in test_json_schema_converter.py const grammar = await Testings._jsonSchemaToEBNF(schema, -1); expect(grammar).toEqual(ebnf_grammar); }); test("Test indent _jsonSchemaToEBNF", async () => { const grammar0 = await Testings._jsonSchemaToEBNF(schema, -1); const grammar1 = await Testings._jsonSchemaToEBNF(schema); const grammar2 = await Testings._jsonSchemaToEBNF(schema, 2); expect(grammar1).toEqual(grammar2); expect(grammar0).not.toEqual(grammar2); }); test("Test indent Grammar.fromJSONSchema()", async () => { const grammar0 = (await Grammar.fromJSONSchema(schema, -1)).toString(); const grammar1 = (await Grammar.fromJSONSchema(schema)).toString(); const grammar2 = (await Grammar.fromJSONSchema(schema, 2)).toString(); expect(grammar1).toEqual(grammar2); expect(grammar0).not.toEqual(grammar2); }); test("Test jsonSchema() argument separators not supported yet", async () => { expect(async () => { const grammar = await Grammar.fromJSONSchema(schema, 2, [",", ":"]); }).rejects.toThrow("Argument separators is not supported yet"); }); test("Test Grammar.builtinJSONGrammar()", async () => { const grammar = await Grammar.builtinJSONGrammar(); const outputStr = grammar.toString(); expect(outputStr == "").toEqual(false); }); }); describe("Test TokenizerInfo", () => { test("Test basic tokenizer info", async () => { const dummyVocab = ["!", "éĶ¦"]; const dummyVocabType = "byte_level"; const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( dummyVocab, dummyVocabType, false ); expect(tokenizerInfo.getDecodedVocabHandle().get(0)).toEqual("!"); expect(tokenizerInfo.getDecodedVocabHandle().get(1)).toEqual("锦"); tokenizerInfo.dispose(); }); test("Test with Llama3.2, byte_level", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); expect(tokenizerInfo.getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); }) test("Test with Phi3.5, byte_fallback", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC/raw/main/tokenizer.json", "byte_fallback", true, ); // phi-3.5 though vocab size is 32064 in config.json, has 32011 actual vocab. The size of the // table (i.e. tokenizer.getVocabSize()) may be smaller than the `vocab_size` in config.json // (length of logits), see https://github.com/QwenLM/Qwen2/issues/147 and // https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/discussions/47. expect(tokenizerInfo.getDecodedVocabHandle().size()).toEqual(32011); tokenizerInfo.dispose(); }) }); describe("Test CompiledGrammar and GrammarCompiler", () => { /** * Equivalent to class MainModel(BaseModel): num: int = 0 opt_bool: Optional[bool] = None size: Optional[float] name: str = "" */ const schema = String.raw`{"properties": {"num": {"default": 0, "title": "Num", "type": "integer"}, "opt_bool": {"anyOf": [{"type": "boolean"}, {"type": "null"}], "default": null, "title": "Opt Bool"}, "size": {"anyOf": [{"type": "number"}, {"type": "null"}], "title": "Size"}, "name": {"default": "", "title": "Name", "type": "string"}}, "required": ["size"], "title": "MainModel", "type": "object"}`; test("Test compileBuiltinJSONGrammar", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const compiledGrammar = await compiler.compileBuiltinJSONGrammar(); const outputStr = compiledGrammar.grammar().toString(); expect(outputStr == "").toEqual(false); expect(compiledGrammar.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); compiledGrammar.dispose(); compiler.dispose(); }); test("Test compileJSONSchema", async () => { const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const compiledGrammar0 = await compiler.compileJSONSchema(schema, -1); const compiledGrammar1 = await compiler.compileJSONSchema(schema); const compiledGrammar2 = await compiler.compileJSONSchema(schema, 2); const outputStr0 = compiledGrammar0.grammar().toString(); const outputStr1 = compiledGrammar1.grammar().toString(); const outputStr2 = compiledGrammar2.grammar().toString(); expect(outputStr1).toEqual(outputStr2); expect(outputStr0).not.toEqual(outputStr2); expect(compiledGrammar1.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); compiledGrammar0.dispose(); compiledGrammar1.dispose(); compiledGrammar2.dispose(); compiler.dispose(); }); test("Test compileGrammar with a EBNF string and a Grammar", async () => { const before = `root ::= ((b c) | (b root)) b ::= ((b_1 d [a]*)) c ::= ((c_1)) d ::= ((d_1)) b_1 ::= ("" | ("b" b_1)) c_1 ::= ((c_2 c_1) | (c_2)) c_2 ::= (([acep-z])) d_1 ::= ("" | ("d")) `; const tokenizerInfo = await getTokenizerInfoFromUrl( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json", "byte_level", false, ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const grammar = await Grammar.fromEBNF(before); const compiledGrammar1 = await compiler.compileGrammar(grammar); const compiledGrammar2 = await compiler.compileGrammar(before); const outputStr1 = compiledGrammar1.grammar().toString(); const outputStr2 = compiledGrammar2.grammar().toString(); expect(outputStr1).toEqual(before); expect(outputStr2).toEqual(before); expect(compiledGrammar1.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); expect(compiledGrammar2.tokenizerInfo().getDecodedVocabHandle().size()).toEqual(128256); tokenizerInfo.dispose(); compiledGrammar1.dispose(); compiledGrammar2.dispose(); compiler.dispose(); }); }); // Identical to tests in `test_grammar_matcher.py` describe("Test GrammarMatcher E2E", () => { const vocab = [ "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', ]; const input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}"]; const input_ids: number[] = []; input_splitted.forEach((input) => { input_ids.push(vocab.indexOf(input)); }); test("test_token_operations", async () => { // 1. Instantiate matcher const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher(jsonGrammar); // 2. Test const expected = [ ["{"], ['"', "}", "\n", " ", '"a":true'], ["a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], ["a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], [':"', ":", "\n", " "], ['"', "{", "6", "\n", " "], ["}", ", ", "6", "\n", " "], ['"', "\n", " ", '"a":true'], ['"', "\n", " ", '"a":true'], ["}", ", ", "\n", " "], [""], ] const result: Array> = [] for (let i = 0; i <= input_ids.length; i++) { const input_id = input_ids[i]; // Find rejected IDs const bitmask = await matcher.getNextTokenBitmask(); const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); // Find accepted tokens const vocabIDSet = new Set([...Array(vocab.length).keys()]); const rejectedIDSet = new Set(rejectedIDs); const acceptedIDSet = new Set([...vocabIDSet].filter(x => !rejectedIDSet.has(x))); const acceptedIDList = Array.from(acceptedIDSet.values()).sort(((a, b) => a - b)); const acceptedTokens: string[] = []; acceptedIDList.forEach((acceptedID) => { acceptedTokens.push(vocab[acceptedID]); }); result.push(acceptedTokens); // Note the <= in the loop bound. We do an extra checking for the last input. if (i < input_ids.length) { // Check input_id is accepted, and update matcher expect(acceptedIDSet.has(input_id)).toEqual(true); const accepted = matcher.acceptToken(input_id); expect(accepted).toEqual(true); } } expect(result).toEqual(expected); matcher.dispose(); tokenizerInfo.dispose(); }); // Identical to the test above, except we specify stop token to be both 0 and 1 test("test_token_operations with customized stop token id", async () => { // 1. Instantiate matcher // TODO(Charlie): Specifying only 0 still makes 1 a valid stop token -- is this what we want? const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false, undefined, [0, 1] ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher(jsonGrammar); // 2. Test const expected = [ ["{"], ['"', "}", "\n", " ", '"a":true'], ["a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], ["a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", " "], [':"', ":", "\n", " "], ['"', "{", "6", "\n", " "], ["}", ", ", "6", "\n", " "], ['"', "\n", " ", '"a":true'], ['"', "\n", " ", '"a":true'], ["}", ", ", "\n", " "], ["", ""], ] const result: Array> = [] for (let i = 0; i <= input_ids.length; i++) { const input_id = input_ids[i]; // Find rejected IDs const bitmask = await matcher.getNextTokenBitmask(); const rejectedIDs = await Testings.debugGetMaskedTokensFromBitmask( bitmask, tokenizerInfo.getVocabSize() ); // Find accepted tokens const vocabIDSet = new Set([...Array(vocab.length).keys()]); const rejectedIDSet = new Set(rejectedIDs); const acceptedIDSet = new Set([...vocabIDSet].filter(x => !rejectedIDSet.has(x))); const acceptedIDList = Array.from(acceptedIDSet.values()).sort(((a, b) => a - b)); const acceptedTokens: string[] = []; acceptedIDList.forEach((acceptedID) => { acceptedTokens.push(vocab[acceptedID]); }); result.push(acceptedTokens); // Note the <= in the loop bound. We do an extra checking for the last input. if (i < input_ids.length) { // Check input_id is accepted, and update matcher expect(acceptedIDSet.has(input_id)).toEqual(true); const accepted = matcher.acceptToken(input_id); expect(accepted).toEqual(true); } } expect(result).toEqual(expected); tokenizerInfo.dispose(); matcher.dispose(); }); test("test_roll_back", async () => { // 1. Instantiate matcher const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher( jsonGrammar, undefined, undefined, 5, ); tokenizerInfo.dispose(); expect(matcher.getMaxRollbackTokens()).toEqual(5); // 2. Test const input_ids_splitted: number[][] = []; for (let i = 0; i < input_ids.length; i += 2) { input_ids_splitted.push(input_ids.slice(i, i + 2)); } for (let i = 0; i < input_ids_splitted.length; i++) { const i_1 = input_ids_splitted[i][0]; const i_2 = input_ids_splitted[i][1]; const orig_result: Int32Array[] = []; // Accept firt round orig_result.push(await matcher.getNextTokenBitmask()); const accept_i1 = matcher.acceptToken(i_1); orig_result.push(await matcher.getNextTokenBitmask()); const accept_i2 = matcher.acceptToken(i_2); expect(accept_i1).toEqual(true); expect(accept_i2).toEqual(true); // Rollback, then accept again matcher.rollBack(2); const result_after_rollback: Int32Array[] = []; result_after_rollback.push(await matcher.getNextTokenBitmask()); const accept_i1_r = matcher.acceptToken(i_1); result_after_rollback.push(await matcher.getNextTokenBitmask()); const accept_i2_r = matcher.acceptToken(i_2); expect(accept_i1_r).toEqual(true); expect(accept_i2_r).toEqual(true); // Expect same token bitmask expect(orig_result).toEqual(result_after_rollback); } matcher.dispose(); }); test("test reset and termination", async () => { // This one has ``, different from the ones used before const vocab = [ "", "", "a", "abc", 'b"', '"', ':"', "{", "}", ", ", "6", ":", "\n", " ", '"a":true', ]; const input_splitted = ["{", '"', "abc", 'b"', ":", "6", ", ", " ", '"a":true', "}", ""]; const input_ids: number[] = []; input_splitted.forEach((input) => { input_ids.push(vocab.indexOf(input)); }); // 1. Instantiate matcher const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const jsonGrammar = await compiler.compileBuiltinJSONGrammar(); const matcher = await GrammarMatcher.createGrammarMatcher( jsonGrammar, undefined, undefined, 5 ); tokenizerInfo.dispose(); // 2. Accept all one time const orig_result: Int32Array[] = []; for (let i = 0; i < input_ids.length; i++) { orig_result.push(await matcher.getNextTokenBitmask()); const accepted = matcher.acceptToken(input_ids[i]); expect(accepted).toEqual(true); } // 3. Check termination expect(matcher.isTerminated()).toEqual(true); const acceptedAfterTerm0 = matcher.acceptToken(0); expect(acceptedAfterTerm0).toEqual(false); // this will throw error, but cannot be caught by jest // await matcher.getNextTokenBitmask() // 4. Reset, accept again matcher.reset(); const result_after_reset: Int32Array[] = []; for (let i = 0; i < input_ids.length; i++) { result_after_reset.push(await matcher.getNextTokenBitmask()); const accepted = matcher.acceptToken(input_ids[i]); expect(accepted).toEqual(true); } // 5. Check same bitmask result, check termination again expect(orig_result).toEqual(result_after_reset); expect(matcher.isTerminated()).toEqual(true); const acceptedAfterTerm1 = matcher.acceptToken(0); expect(acceptedAfterTerm1).toEqual(false); // this will throw error, but cannot be caught by jest // await matcher.getNextTokenBitmask() // 6. Rollback 2, and should not be terminated and should accept "}" matcher.rollBack(2); expect(matcher.isTerminated()).toEqual(false); const acceptedAfterTerm2 = matcher.acceptToken(input_ids.slice(-2, -1)[0]); expect(acceptedAfterTerm2).toEqual(true); matcher.dispose(); }); test("test_get_jump_forward_string", async () => { const grammar_ebnf = String.raw`root ::= "abb" | "abbd" | other_rule other_rule ::= "a" sub_rule "b" sub_rule ::= "b" `; const vocab = ["a", "bb"]; const tokenizerInfo = await TokenizerInfo.createTokenizerInfo( vocab, "byte_level", false ); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); const grammar = await compiler.compileGrammar(grammar_ebnf); const matcher = await GrammarMatcher.createGrammarMatcher(grammar); tokenizerInfo.dispose(); expect(matcher.acceptToken(0)).toEqual(true); expect(matcher.findJumpForwardString()).toEqual("bb"); }); }); // Identical to `test_builtin_grammar_json_schema.py` describe("Test json schema E2E", () => { // Equivalent to MainModel used in the python test, except removed minItem and maxItem from tuple_field to avoid long log const schemaStr = String.raw`{"properties": {"integer_field": {"title": "Integer Field", "type": "integer"}, "number_field": {"title": "Number Field", "type": "number"}, "boolean_field": {"title": "Boolean Field", "type": "boolean"}, "any_array_field": {"items": {}, "title": "Any Array Field", "type": "array"}, "array_field": {"items": {"type": "string"}, "title": "Array Field", "type": "array"}, "tuple_field": {"prefixItems": [{"type": "string"}, {"type": "integer"}, {"items": {"type": "string"}, "type": "array"}], "title": "Tuple Field", "type": "array"}, "object_field": {"additionalProperties": {"type": "integer"}, "title": "Object Field", "type": "object"}, "nested_object_field": {"additionalProperties": {"additionalProperties": {"type": "integer"}, "type": "object"}, "title": "Nested Object Field", "type": "object"}}, "required": ["integer_field", "number_field", "boolean_field", "any_array_field", "array_field", "tuple_field", "object_field", "nested_object_field"], "title": "MainModel", "type": "object"}`; // Equivalent to instance in the python test, a valid json following the above schema const instanceStr = String.raw`{ "integer_field": 42, "number_field": 314000.0, "boolean_field": true, "any_array_field": [ 3.14, "foo", null, true ], "array_field": [ "foo", "bar" ], "tuple_field": [ "foo", 42, [ "bar", "baz" ] ], "object_field": { "foo": 42, "bar": 43 }, "nested_object_field": { "foo": { "bar": 42 } } }`; // Note: This test much slower than others test("Test with Llama3.2, byte_level", async () => { // 1. Get tokenizer const jsonBuffer = await (await fetch( "https://huggingface.co/mlc-ai/Llama-3.2-1B-Instruct-q4f16_0-MLC/raw/main/tokenizer.json" )).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get encoded vocab const encodedVocab: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 3. Decode const tokenizerInfo = await TokenizerInfo.createTokenizerInfo(encodedVocab, "byte_level", false); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); // 4. Instantiate matcher const grammar = await compiler.compileJSONSchema(schemaStr, 2); const matcher = await GrammarMatcher.createGrammarMatcher(grammar); const inputIds = tokenizer.encode(instanceStr); // 5. Expect to accept all inputIds for (let i = 0; i < inputIds.length; i++) { const inputId = inputIds[i]; await matcher.getNextTokenBitmask(); const accepted = matcher.acceptToken(inputId); expect(accepted).toEqual(true); } // 6. Check finalization const final_bitmask = await matcher.getNextTokenBitmask(); expect(final_bitmask.length).toEqual(Math.ceil(128256 / 32)); const final_rejected_tokens = (await Testings.debugGetMaskedTokensFromBitmask( final_bitmask, tokenizerInfo.getVocabSize() )); expect(final_rejected_tokens.indexOf(128001)).toEqual(-1); // stop token not rejected const acceptStop = matcher.acceptToken(128001); expect(acceptStop).toEqual(true); expect(matcher.isTerminated()).toEqual(true); matcher.dispose(); grammar.dispose(); tokenizerInfo.dispose(); }); // Note: This test much slower than others test("Test with Phi-3.5, byte_fallback, _debugAcceptString", async () => { // 1. Get tokenizer const jsonBuffer = await (await fetch( "https://huggingface.co/mlc-ai/Phi-3.5-mini-instruct-q4f16_1-MLC/raw/main/tokenizer.json", )).arrayBuffer(); const tokenizer = await Tokenizer.fromJSON(jsonBuffer); // 2. Get encoded vocab const encodedVocab: string[] = []; const vocabSize = tokenizer.getVocabSize(); for (let tokenId = 0; tokenId < vocabSize; tokenId++) { encodedVocab.push(tokenizer.idToToken(tokenId)); } // 3. Decode; note that phi-3.5 has 32064 as vocab size in `config.json` const tokenizerInfo = await TokenizerInfo.createTokenizerInfo(encodedVocab, "byte_fallback", false, 32064); const compiler = await GrammarCompiler.createGrammarCompiler(tokenizerInfo); // 4. Instantiate matcher const grammar = await compiler.compileJSONSchema(schemaStr, 2); const matcher = await GrammarMatcher.createGrammarMatcher(grammar); // 5. Expect to accept all inputIds for (let i = 0; i < instanceStr.length; i++) { const inputStr = instanceStr[i]; await matcher.getNextTokenBitmask(); // if use acceptToken, the first token of instanceStr will be encoded as 426, `_{`, // while the matcher only accepts 29912, `{`, and another token of `<0x??>` const accepted = matcher._debugAcceptString(inputStr); expect(accepted).toEqual(true); } // 6. Check finalization const final_bitmask = await matcher.getNextTokenBitmask(); // Tests how phi3.5 has dummy padded tokens. See https://github.com/mlc-ai/mlc-llm/pull/2651 expect(final_bitmask.length).toEqual(Math.ceil(32064 / 32)); const final_rejected_tokens = (await Testings.debugGetMaskedTokensFromBitmask( final_bitmask, tokenizerInfo.getVocabSize() )); expect(final_rejected_tokens.indexOf(2)).toEqual(-1); // stop token not rejected expect(final_rejected_tokens.indexOf(32000)).toEqual(-1); // stop token not rejected const acceptStop = matcher.acceptToken(2); expect(acceptStop).toEqual(true); expect(matcher.isTerminated()).toEqual(true); tokenizerInfo.dispose(); }) }); xgrammar-0.1.19/web/tsconfig.json000066400000000000000000000005261500705317600167130ustar00rootroot00000000000000{ "compilerOptions": { "declaration": true, "outDir": "lib", "declarationMap": true, "esModuleInterop": true, "strict": true, "noImplicitAny": false, // to allow importing xgrammar_binding.js }, "include": [ "src" ], "exclude": [ "node_modules", "build", "dist", "rollup.config.js" ] }