pax_global_header00006660000000000000000000000064145355513470014526gustar00rootroot0000000000000052 comment=5e50b8821d2623f68ece2b9d5dd65bdd472c1e12 yte-1.5.4/000077500000000000000000000000001453555134700123365ustar00rootroot00000000000000yte-1.5.4/.flake8000066400000000000000000000003041453555134700135060ustar00rootroot00000000000000[flake8] max-line-length = 88 max-complexity = 10 select = C,E,F,W,B,B950 ignore = E203,E501,W503 exclude = .git, __pycache__, *.egg-info, .nox, .pytest_cache, .mypy_cache yte-1.5.4/.github/000077500000000000000000000000001453555134700136765ustar00rootroot00000000000000yte-1.5.4/.github/workflows/000077500000000000000000000000001453555134700157335ustar00rootroot00000000000000yte-1.5.4/.github/workflows/conventional-prs.yml000066400000000000000000000005331453555134700217600ustar00rootroot00000000000000name: PR on: pull_request_target: types: - opened - reopened - edited - synchronize jobs: title-format: runs-on: ubuntu-latest steps: - uses: amannn/action-semantic-pull-request@v3.4.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: validateSingleCommit: trueyte-1.5.4/.github/workflows/release-please.yml000066400000000000000000000017111453555134700213450ustar00rootroot00000000000000on: push: branches: - main name: release-please jobs: release-please: runs-on: ubuntu-latest outputs: release_created: ${{ steps.release.outputs.release_created }} steps: - uses: GoogleCloudPlatform/release-please-action@v3 id: release with: release-type: python package-name: yte publish: runs-on: ubuntu-latest needs: release-please if: ${{ needs.release-please.outputs.release_created }} steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: "3.9" - name: Install poetry run: pip install poetry - name: Install Dependencies using Poetry run: poetry install - name: Publish to PyPi env: PYPI_USERNAME: __token__ PYPI_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: poetry publish --build --username $PYPI_USERNAME --password $PYPI_PASSWORDyte-1.5.4/.github/workflows/testing.yml000066400000000000000000000027671453555134700201470ustar00rootroot00000000000000name: CI on: [push] jobs: formatting: runs-on: ubuntu-latest steps: - name: Check out the code uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: "3.9" - name: Setup poetry run: pip install poetry - name: Install Dependencies using Poetry run: poetry install - name: Check formatting run: poetry run black --check . linting: runs-on: ubuntu-latest steps: - name: Check out the code uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: "3.9" - name: Setup poetry run: pip install poetry - name: Install Dependencies using Poetry run: poetry install - name: Check code run: poetry run flake8 testing: needs: - formatting - linting strategy: fail-fast: false matrix: python-version: ["3.7", "3.9", "3.10"] poetry-version: ["1.1"] runs-on: ubuntu-latest steps: - name: Check out the code uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Setup poetry run: pip install poetry - name: Install Dependencies using Poetry run: poetry install - name: Run pytest run: poetry run coverage run -m pytest tests.py - name: Run Coverage run: poetry run coverage report -myte-1.5.4/.gitignore000066400000000000000000000000501453555134700143210ustar00rootroot00000000000000__pycache__/ poetry.lock .coverage dist/yte-1.5.4/CHANGELOG.md000066400000000000000000000140261453555134700141520ustar00rootroot00000000000000# Changelog ## [1.5.4](https://github.com/yte-template-engine/yte/compare/v1.5.3...v1.5.4) (2023-12-11) ### Miscellaneous Chores * fix release process ([7ed63f9](https://github.com/yte-template-engine/yte/commit/7ed63f9fde43899b4b1b746003d66d869a0ed415)) ## [1.5.3](https://github.com/yte-template-engine/yte/compare/v1.5.2...v1.5.3) (2023-12-11) ### Miscellaneous Chores * release 1.5.3 ([d3d9e56](https://github.com/yte-template-engine/yte/commit/d3d9e56cf40a53999b0e46a67c766e6a1d229b8e)) ## [1.5.2](https://github.com/yte-template-engine/yte/compare/v1.5.1...v1.5.2) (2023-12-11) ### Bug Fixes * fix errors occuring in document context building when having dicts as list items ([#37](https://github.com/yte-template-engine/yte/issues/37)) ([f347d32](https://github.com/yte-template-engine/yte/commit/f347d32845f4e0bd109adf3fde9e5e25d956c852)) ## [1.5.1](https://github.com/yte-template-engine/yte/compare/v1.5.0...v1.5.1) (2022-06-03) ### Bug Fixes * revert to Python 3.9 for release uploading ([a35a10c](https://github.com/yte-template-engine/yte/commit/a35a10c77cae661d9696e672e382cd8c1b20bc31)) ## [1.5.0](https://github.com/yte-template-engine/yte/compare/v1.4.0...v1.5.0) (2022-06-03) ### Features * allow to require a __use_yte__ statement when processing ([#27](https://github.com/yte-template-engine/yte/issues/27)) ([abf3d95](https://github.com/yte-template-engine/yte/commit/abf3d95c1a241088f24825606034e35c0600be7b)) ## [1.4.0](https://www.github.com/yte-template-engine/yte/compare/v1.3.0...v1.4.0) (2022-05-10) ### Features * rename the global doc object into `this` (`doc` remains available as an alias) ([#25](https://www.github.com/yte-template-engine/yte/issues/25)) ([e3adc67](https://www.github.com/yte-template-engine/yte/commit/e3adc67094188e5af8000580d4732e7d8fa68a09)) ### Documentation * CLI and formatting ([e21512d](https://www.github.com/yte-template-engine/yte/commit/e21512d1a71761f9078a6abf8ea2b4708fe5caf0)) ## [1.3.0](https://www.github.com/yte-template-engine/yte/compare/v1.2.3...v1.3.0) (2022-05-09) ### Features * enable access to already rendered parts of the document via a global variable `doc` ([#23](https://www.github.com/yte-template-engine/yte/issues/23)) ([a2ac30a](https://www.github.com/yte-template-engine/yte/commit/a2ac30a6c97124bc4a57405877832b48b1a8bb4f)) * variable definitions as an alternative to full Python definitions ([#19](https://www.github.com/yte-template-engine/yte/issues/19)) ([c1f4b1c](https://www.github.com/yte-template-engine/yte/commit/c1f4b1ceacd662db33c2e55968c9f402724adbe1)) ### Documentation * Fix README.md example ([#22](https://www.github.com/yte-template-engine/yte/issues/22)) ([0247fe2](https://www.github.com/yte-template-engine/yte/commit/0247fe229a6c38940f485bcce18f51a6dea72551)) ### [1.2.3](https://www.github.com/yte-template-engine/yte/compare/v1.2.2...v1.2.3) (2022-05-06) ### Bug Fixes * display error context ([#17](https://www.github.com/yte-template-engine/yte/issues/17)) ([cbe74a3](https://www.github.com/yte-template-engine/yte/commit/cbe74a357be3449bbb8e0325f1e87ec6469a4b3b)) ### Documentation * add example and testcase for variable definition ([#15](https://www.github.com/yte-template-engine/yte/issues/15)) ([818f886](https://www.github.com/yte-template-engine/yte/commit/818f886b9c44f2bd15fe5e0f32119c0c3ace3ca1)) ### [1.2.2](https://www.github.com/yte-template-engine/yte/compare/v1.2.1...v1.2.2) (2022-04-12) ### Documentation * fix readme example and better error message in case of mixing list and dict returns ([#13](https://www.github.com/yte-template-engine/yte/issues/13)) ([afc0e69](https://www.github.com/yte-template-engine/yte/commit/afc0e69b0ab5a9c2087558886336f34227fd248b)) ### [1.2.1](https://www.github.com/yte-template-engine/yte/compare/v1.2.0...v1.2.1) (2022-03-18) ### Bug Fixes * improved error message in case of YAML syntax errors ([#11](https://www.github.com/yte-template-engine/yte/issues/11)) ([4377e22](https://www.github.com/yte-template-engine/yte/commit/4377e22566edbff34083687256fb269b95ee788b)) ### Documentation * Fix example for __definitions__ ([#9](https://www.github.com/yte-template-engine/yte/issues/9)) ([4fff096](https://www.github.com/yte-template-engine/yte/commit/4fff096109b5e3ed5141e4294232c20aaf2bdd1f)) ## [1.2.0](https://www.github.com/yte-template-engine/yte/compare/v1.1.0...v1.2.0) (2022-02-28) ### Features * rename __imports__ into __definitions__ reflecting that it actually allows arbitrary Python statements ([35dde8f](https://www.github.com/yte-template-engine/yte/commit/35dde8f7cb9c8a71d9006f116972ed89d3795535)) ## [1.1.0](https://www.github.com/yte-template-engine/yte/compare/v1.0.0...v1.1.0) (2022-02-28) ### Features * allow import statements via special keyword __imports__ ([be4e497](https://www.github.com/yte-template-engine/yte/commit/be4e497d952747169db1418f288f2025a1654153)) ### Documentation * import keyword ([4582037](https://www.github.com/yte-template-engine/yte/commit/45820379337d5b98e3a70290e9488d11cd3022af)) ## [1.0.0](https://www.github.com/yte-template-engine/yte/compare/v0.2.0...v1.0.0) (2022-02-21) first stable language release ## [0.2.0](https://www.github.com/yte-template-engine/yte/compare/v0.1.1...v0.2.0) (2022-02-18) ### Features * python 3.7 support ([#4](https://www.github.com/yte-template-engine/yte/issues/4)) ([fcd69e2](https://www.github.com/yte-template-engine/yte/commit/fcd69e28e8af53789f04015e89e64fab03bf1701)) ### [0.1.1](https://www.github.com/yte-template-engine/yte/compare/v0.1.0...v0.1.1) (2022-02-15) ### Bug Fixes * more relaxed Python version dependency, more metadata ([#2](https://www.github.com/yte-template-engine/yte/issues/2)) ([545275f](https://www.github.com/yte-template-engine/yte/commit/545275ff90071c400b06ae7512db530dafb197a9)) ## 0.1.0 (2022-02-15) ### Features * add command line interface ([688ab12](https://www.github.com/yte-template-engine/yte/commit/688ab124268b3a9f9191f66d5486d5196493c2c0)) ### Documentation * API documentation ([d0931d5](https://www.github.com/yte-template-engine/yte/commit/d0931d54804ff9527cd2b663d40585586961fd5b)) yte-1.5.4/LICENSE000066400000000000000000000020761453555134700133500ustar00rootroot00000000000000Copyright 2022 Johannes Köster Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. yte-1.5.4/README.md000066400000000000000000000047221453555134700136220ustar00rootroot00000000000000# YTE - A YAML template engine with Python expressions [![Docs](https://img.shields.io/badge/user-documentation-green)](https://yte-template-engine.github.io) [![test coverage: 100%](https://img.shields.io/badge/test%20coverage-100%25-green)](https://github.com/yte-template-engine/yte/blob/main/pyproject.toml#L30) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/yte-template-engine/yte/testing.yml?branch=main) ![PyPI](https://img.shields.io/pypi/v/yte) [![Conda Recipe](https://img.shields.io/badge/recipe-yte-green.svg)](https://anaconda.org/conda-forge/yte) [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/yte.svg)](https://anaconda.org/conda-forge/yte) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/yte.svg)](https://github.com/conda-forge/yte-feedstock) YTE is a template engine for YAML format that utilizes the YAML structure in combination with Python expressions for enabling to dynamically build YAML documents. The key idea of YTE is to rely on the YAML structure to enable conditionals, loops and other arbitrary Python expressions to dynamically render YAML files. Python expressions are thereby declared by prepending them with a `?` anywhere in the YAML. Any such value will be automatically evaluated by YTE, yielding plain YAML as a result. Importantly, YTE templates are still valid YAML files (for YAML, the `?` expressions are just strings). Documentation of YTE can be found at https://yte-template-engine.github.io. ## Comparison with other engines Lots of template engines are available, for example the famous generic [jinja2](https://jinja.palletsprojects.com). The reasons to generate a YAML specific engine are 1. The YAML syntax can be exploited to simplify template expression syntax, and make it feel less foreign (i.e. fewer special characters for control flow needed) while increasing human readability. 2. Whitespace handling (which is important with YAML since it has a semantic there) becomes unnecessary (e.g. with jinja2, some [tuning](https://radeksprta.eu/posts/control-whitespace-in-ansible-templates) is required to obtain proper YAML rendering). Of course, YTE is not the first YAML specific template engine. Others include * [Yglu](https://yglu.io) * [Emrichen](https://github.com/con2/emrichen) The main difference between YTE and these two is that YTE extends YAML with plain Python syntax instead of introducing another specialized language. Of course, the choice is also a matter of taste. yte-1.5.4/docs/000077500000000000000000000000001453555134700132665ustar00rootroot00000000000000yte-1.5.4/docs/main.md000066400000000000000000000170521453555134700145410ustar00rootroot00000000000000# YTE - A YAML template engine with Python expressions [![Docs](https://img.shields.io/badge/user-documentation-green)](https://yte-template-engine.github.io) [![test coverage: 100%](https://img.shields.io/badge/test%20coverage-100%25-green)](https://github.com/yte-template-engine/yte/blob/main/pyproject.toml#L30) ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/yte-template-engine/yte/CI) ![PyPI](https://img.shields.io/pypi/v/yte) [![Conda Recipe](https://img.shields.io/badge/recipe-yte-green.svg)](https://anaconda.org/conda-forge/yte) [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/yte.svg)](https://anaconda.org/conda-forge/yte) [![Conda Version](https://img.shields.io/conda/vn/conda-forge/yte.svg)](https://github.com/conda-forge/yte-feedstock) YTE is a template engine for YAML format that utilizes the YAML structure in combination with Python expressions for enabling to dynamically build YAML documents. ## Syntax The key idea of YTE is to rely on the YAML structure to enable conditionals, loops and other arbitrary Python expressions to dynamically render YAML files. Python expressions are thereby declared by prepending them with a `?` anywhere in the YAML. Any such value will be automatically evaluated by YTE, yielding plain YAML as a result. Importantly, YTE templates are still valid YAML files (for YAML, the `?` expressions are just strings). ### Conditionals ```yaml ?if True: foo: 1 ?elif False: bar: 2 ?else: bar: 1 ``` ```yaml foo: 1 ``` ```yaml ?if True: - a - b ``` ```yaml - a - b ``` ```yaml - foo - bar - ?if True: baz ?else: bar ``` ```yaml - foo - bar - baz ``` ### Loops ```yaml ?for i in range(2): '?f"key:{i}"': 1 # When expressions in keys or values contain colons, they need to be additionally quoted. ?if i == 1: foo: true ``` ```yaml "key:0": 1 "key:1": 1 foo: true ``` ### Accessing already rendered document parts A globally available object `this` (a wrapper around a Python dict) enables to access parts of the document that have already been rendered above. This way, one can often avoid variable definitions (see below). In addition to normal dict access, the object allows to search (`this.dpath_search`) and access (`this.dpath_get`) its contents via [dpath](https://github.com/dpath-maintainers/dpath-python) queries (see the [dpath docs](https://github.com/dpath-maintainers/dpath-python) for allowed expressions). Simple dpath get queries can also be performed by putting the dpath query directly into the square bracket operator of the `doc` object (the logic in that case is as follows: first, the given value is tried as plain key, if that fails, `this.dpath_get` is tried as a fallback); see example below. ```yaml foo: 1 bar: a: 2 # dict access b: ?this["foo"] + this["bar"]["a"] # implicit simple dpath get query c: ?this["bar/a"] # explicit dpath queries d: ?this.dpath_get("foo") + this.dpath_get("bar/a") ``` ```yaml foo: 1 bar: a: 2 b: 3 c: 2 c: 3 ``` ### Variable definitions The special keyword `__variables__` allows to define variables that can be reused below. It can be used anywhere in the YAML, also repeatedly and inside of ifs or loops with the restriction of not having duplicate `__variables__` keys on the same level. The usage of `__variables__` can be disabled via the API. ```yaml __variables__: first: foo second: 1.5 # apart from constant values as defined above, also Python expressions are allowed: third: ?2 * 3 a: ?first b: ?second c: ?for x in range(3): __variables__: y: ?x * 2 ?if True: - ?y ``` ```yaml a: foo b: 1.5 c: - 0 - 2 - 4 ``` ### Arbitrary definitions The special keyword `__definitions__` allows to define custom statements. It can be used anywhere in the YAML, also repeatedly and inside of ifs or loops with the restriction of not having duplicate `__definitions__` keys on the same level. The usage of `__definitions__` can be disabled via the API. ```yaml __definitions__: - from itertools import product - someval = 2 - | def squared(value): return value ** 2 ?for item in product([1, 2], ["a", "b"]): - ?f"{item}" ?if True: - ?squared(2) * someval - someval: ?someval ``` ```yaml - 1-a - 1-b - 2-a - 2-b - 4 - someval: 2 ``` ## Usage ### Installation YTE can be installed as a Python package via [PyPi](https://pypi.org/project/yte) or [Conda/Mamba](https://anaconda.org/conda-forge/yte). ### Python API Alternatively, you can invoke YTE via its Python API: ```python from yte import process_yaml # Set some variables as a Python dictionary. variables = ... # Render a string and obtain the result as a Python dict. result = process_yaml(""" ?for i in range(10): - ?f"item-{i}" """, variables=variables) # Render a file and obtain the result as a Python dict. with open("the-template.yaml", "r") as template: result = process_yaml(template, variables=variables) # Render a file and write the result as valid YAML. with open("the-template.yaml", "r") as template, open("the-rendered-version.yaml", "w") as outfile: result = process_yaml(template, outfile=outfile, variables=variables) # Render a file while disabling the __definitions__ feature. with open("the-template.yaml", "r") as template: result = process_yaml(template, variables=variables, disable_features=["definitions"]) # Render a file while disabling the __variables__ feature. with open("the-template.yaml", "r") as template: result = process_yaml(template, variables=variables, disable_features=["variables"]) # Render a file while disabling the __variables__ and __definitions__ feature. with open("the-template.yaml", "r") as template: result = process_yaml(template, variables=variables, disable_features=["variables", "definitions"]) # Require that the document contains a `__use_yte__: true` statement at the top level. # If the statement is not present or false, return document unprocessed (except removing the `__use_yte__: false` statement if present). with open("the-template.yaml", "r") as template: result = process_yaml(template, variables=variables, require_use_yte=True) ``` ### Command line interface YTE also provides a command line interface: ```bash yte --help ``` It can be used to process a YTE template from STDIN and prints the rendered version to STDOUT: ```bash yte < template.yaml > rendered.yaml ``` ## Comparison with other engines Lots of template engines are available, for example the famous generic [jinja2](https://jinja.palletsprojects.com). The reasons to generate a YAML specific engine are 1. The YAML syntax can be exploited to simplify template expression syntax, and make it feel less foreign (i.e. fewer special characters for control flow needed) while increasing human readability. 2. Whitespace handling (which is important with YAML since it has a semantic there) becomes unnecessary (e.g. with jinja2, some [tuning](https://radeksprta.eu/posts/control-whitespace-in-ansible-templates) is required to obtain proper YAML rendering). Of course, YTE is not the first YAML specific template engine. Others include * [Yglu](https://yglu.io) * [Emrichen](https://github.com/con2/emrichen) The main difference between YTE and these two is that YTE extends YAML with plain Python syntax instead of introducing another specialized language. Of course, the choice is also a matter of taste. yte-1.5.4/pyproject.toml000066400000000000000000000020361453555134700152530ustar00rootroot00000000000000[tool.poetry] authors = ["Johannes Köster "] description = "A YAML template engine with Python expressions" homepage = "https://github.com/yte-template-engine/yte" license = "MIT" name = "yte" readme = "README.md" repository = "https://github.com/yte-template-engine/yte" version = "1.5.4" [tool.poetry.dependencies] dpath = "^2.1" plac = "^1.3.4" python = ">=3.7" pyyaml = "^6.0" [tool.poetry.dev-dependencies] black = "^22.1.0" coverage = {extras = ["toml"], version = "^6.3.1"} flake8 = "^4.0.1" flake8-bugbear = "^22.1.11" pytest = "^7.0" [tool.coverage.run] omit = [".*", "*/site-packages/*"] [tool.coverage.report] # exclude CLI handling lines. They cannot be captured properly by coverage, but we have a testcase for them. exclude_lines = [ "yaml\\.dump\\(result, outfile\\)", "process_yaml\\(sys.stdin, outfile=sys\\.stdout\\)", "plac.call\\(cli\\)", ] fail_under = 100 [tool.poetry.scripts] yte = "yte:main" [build-system] build-backend = "poetry.core.masonry.api" requires = ["poetry-core>=1.0.0"] yte-1.5.4/tests.py000066400000000000000000000227661453555134700140670ustar00rootroot00000000000000import tempfile import yte import textwrap import pytest import yaml import subprocess as sp from yte.context import Context from yte.document import Document, Subdocument from yte.exceptions import YteError def _process(yaml_str, outfile=None, disable_features=None, require_use_yte=False): return yte.process_yaml( textwrap.dedent(yaml_str), outfile=outfile, require_use_yte=require_use_yte, disable_features=disable_features, ) def test_ifelse(): result = _process( """ ?if True: foo: 1 ?elif False: bar: 2 ?else: bar: 1 """ ) assert result == {"foo": 1} def test_for(): result = _process( """ ?for i in range(2): ?f"key{i}": 1 ?if i == 1: foo: True """ ) assert result == {"key0": 1, "key1": 1, "foo": True} def test_list(): result = _process( """ - foo - bar - ?if True: baz ?else: bar """ ) assert result == ["foo", "bar", "baz"] def test_if_list(): result = _process( """ ?if True: - a - b """ ) assert result == ["a", "b"] def test_fail_mixed_loop_return(): with pytest.raises(YteError): _process( """ ?for i in range(2): ?if i == 0: - foo ?else: bar: True """ ) def test_unexpected_elif(): with pytest.raises(YteError): _process( """ ?elif True: foo: True """ ) def test_unexpected_else(): with pytest.raises(YteError): _process( """ ?else: foo: True """ ) def test_custom_import(): result = _process( """ __definitions__: - from itertools import product ?for a in product([1, 2], [3]): - a """ ) assert result == ["a"] * 2 def test_variable_definition1(): result = _process( """ __definitions__: - test = "foo" ?test: 1 """ ) assert result == {"foo": 1} def test_variable_definition2(): result = _process( """ __definitions__: - foo = "bar" - test = "foo" ?f"{test}": 1 """ ) assert result == {"foo": 1} def test_variable_definition3(): result = _process( """ __definitions__: - test = "foo" bar: ?test ?for item in ["foo", "baz"]: __definitions__: - and_now = "for something completely different" ?item: ?and_now """ ) assert result == { "bar": "foo", "foo": "for something completely different", "baz": "for something completely different", } def test_custom_import_syntax_error(): with pytest.raises(YteError): _process( """ __definitions__: from itertools import product """ ) def test_variable_definition(): result = _process( """ __definitions__: - foo = 1 ?for a in range(2): - ?foo """ ) assert result == [1] * 2 def test_func_definition(): result = _process( """ __definitions__: - | def foo(): return 1 ?for a in range(2): - ?foo() """ ) assert result == [1] * 2 def test_cli(): sp.check_call("echo -e '?if True:\n foo: 1' | yte", shell=True) def test_colon(): result = _process( """ ?for sample in ["normal", "tumor"]: '?f"{sample}: observations"': 1 """ ) assert result == {"normal: observations": 1, "tumor: observations": 1} def test_colon_unquoted(): with pytest.raises(yaml.scanner.ScannerError): _process( """ ?for sample in ["normal", "tumor"]: ?f"{sample}: observations": 1 """ ) def test_outfile(): with tempfile.NamedTemporaryFile(mode="w") as tmp: _process( """ foo """, outfile=tmp, ) def test_simple_error(): with pytest.raises(YteError): _process( """ ?unknown_var """ ) def test_definitions_error(): with pytest.raises(YteError): _process( """ __definitions__: - blpasd sad """ ) def test_conditional_error(): with pytest.raises(YteError): _process( """ ?if asdkn: "foo" """ ) def test_variables(): result = _process( """ __variables__: foo: "x" bar: 3 ?foo: 1 bar: ?bar """ ) assert result == {"x": 1, "bar": 3} def test_variables_error(): with pytest.raises(YteError): _process( """ __variables__: foo: ?some error """ ) def test_disable_definitions(): with pytest.raises(YteError): _process( """ __definitions__: - foo = 1 """, disable_features=["definitions"], ) def test_disable_variables(): with pytest.raises(YteError): _process( """ __variables__: foo: 1 """, disable_features=["variables"], ) def test_disable_invalid_feature(): with pytest.raises(ValueError): _process( """ __variables__: foo: 1 """, disable_features=["bar"], ) def test_invalid_variables(): with pytest.raises(YteError): _process( """ __variables__: - foo: 1 """, ) def test_doc_object(): result = _process( """ foo: 1 other: some: 2 bar: ?this["foo"] + this["other"]["some"] ?f"yetanother-{this['foo']}": 2 other-items: ?sorted(this["other"].items()) """ ) assert result == { "foo": 1, "other": {"some": 2, "bar": 3}, "yetanother-1": 2, "other-items": [("bar", 3), ("some", 2)], } @pytest.fixture def dummy_document(): doc = Document() doc.inner.inner["foo"] = "bar" return doc @pytest.fixture def dummy_document_complex(): doc = Document() doc.inner.inner["foo"] = Subdocument() doc.inner.inner["foo"].inner = {"bar": 1} return doc def test_doc_items(dummy_document): assert list(dummy_document.items()) == [("foo", "bar")] def test_doc_keys(dummy_document): assert list(dummy_document.keys()) == ["foo"] def test_doc_len(dummy_document): assert len(dummy_document) == 1 def test_doc_repr(dummy_document): assert repr(dummy_document) == "{'foo': 'bar'}" def test_doc_insert(): dummy_document = Document() context = Context() context.rendered += ["foo", "bar"] dummy_document._insert(context, 1) assert dummy_document == {"foo": {"bar": 1}} def test_doc_eq(dummy_document): assert dummy_document == dummy_document assert dummy_document == {"foo": "bar"} assert dummy_document != 1 def test_doc_dpath_get(dummy_document_complex): assert dummy_document_complex.dpath_get("foo/bar") == 1 def test_doc_dpath_search(dummy_document_complex): assert dummy_document_complex.dpath_search("foo/bar") == {"foo": {"bar": 1}} def test_doc_dpath_fallback(dummy_document_complex): assert dummy_document_complex["foo/bar"] == 1 def test_doc_dpath_fallback_key_error(dummy_document_complex): with pytest.raises(KeyError): dummy_document_complex["some"] def test_require_use_yte(): result = _process( """ __use_yte__: true ?if True: foo: 1 ?else: bar: 1 """, require_use_yte=True, ) assert result == {"foo": 1} def test_require_use_yte_not_found(): result = _process( """ ?if True: foo: 1 """, require_use_yte=True, ) assert result == {"?if True": {"foo": 1}} def test_require_use_yte_false(): result = _process( """ __use_yte__: false ?if True: foo: 1 """, require_use_yte=True, ) assert result == {"?if True": {"foo": 1}} def test_complex_1(): _process( """ __use_yte__: true html: head: title: Test landing page body: div: class: foo content: - p: Hello, this is some text. How do we get a tag into it? We could avoid tags and just render this as markdown. - p: class: bar content: - This is untagged text - span: content: This is a span class: bold - This is more untagged text - ?if True: markdown: | # This is a markdown heading This is some markdown text ?else: markdown: | # This is a different markdown heading This is some different markdown text """ # noqa: B950 ) yte-1.5.4/yte/000077500000000000000000000000001453555134700131375ustar00rootroot00000000000000yte-1.5.4/yte/__init__.py000066400000000000000000000055331453555134700152560ustar00rootroot00000000000000import sys import yaml import plac from yte.context import Context from yte.process import FEATURES, _process_yaml_value from yte.document import Document def process_yaml( file_or_str, outfile=None, variables=None, require_use_yte=False, disable_features=None, ): """Process a YAML file or string with YTE, returning the processed version. # Arguments * file_or_str - file object or string to render * outfile - output file to write to, if None output is returned as string * variables - variables to be available in the template * require_use_yte - skip templating if there is no `__use_yte__ = True` statement in the top level of the document * disable_features - list of features that should be disabled during rendering. Possible values to choose from are ["definitions", "variables"] """ if variables is None: variables = dict() variables["_process_yaml_value"] = _process_yaml_value doc = Document() variables["doc"] = doc variables["this"] = doc try: yaml_doc = yaml.load(file_or_str, Loader=yaml.FullLoader) except yaml.scanner.ScannerError as e: raise yaml.scanner.ScannerError( f"{e}\nNote that certain characters like colons have a special " "meaning in YAML and hence keys or values containing them have " "to be quoted." ) is_use_yte = yaml_doc.get("__use_yte__") if isinstance(yaml_doc, dict) else None if is_use_yte is not None: # remove __use_yte__ key yaml_doc.pop("__use_yte__") if not require_use_yte or is_use_yte: if disable_features is not None: disable_features = frozenset(disable_features) if not FEATURES.issuperset(disable_features): raise ValueError("Invalid features given to `disable_features`.") else: disable_features = frozenset([]) result = _process_yaml_value( yaml_doc, variables, context=Context(), disable_features=disable_features, ) else: # do not process document since use_yte is required but not found in document result = yaml_doc if outfile is not None: yaml.dump(result, outfile, sort_keys=False) else: return result @plac.flg( "require_use_yte", "Require that the document contains a `__use_yte__: true` statement at the top level. " "If the statement is not present or false, return document unprocessed " "(except removing the `__use_yte__: false` statement if present)", ) def cli( require_use_yte=False, ): """Process a YAML file given at STDIN with YTE, and print the result to STDOUT. Note: if nothing is provided at STDIN, this will wait forever. """ process_yaml(sys.stdin, outfile=sys.stdout) def main(): plac.call(cli) yte-1.5.4/yte/context.py000066400000000000000000000003521453555134700151750ustar00rootroot00000000000000class Context: def __init__(self, other=None): self.template = [] self.rendered = [] if other is not None: self.template = list(other.template) self.rendered = list(other.rendered) yte-1.5.4/yte/document.py000066400000000000000000000043571453555134700153400ustar00rootroot00000000000000from yte.context import Context from dpath import get as dpath_get from dpath import search as dpath_search class Subdocument: def __init__(self): self.inner = dict() def __getitem__(self, key): try: return self.inner[key] except KeyError as e: try: return self.dpath_get(key) except KeyError: raise e def __setitem__(self, key, value): self.inner[key] = value def items(self): return self.inner.items() def keys(self): return self.inner.keys() def dpath_get(self, glob, separator="/"): return dpath_get(self.inner, glob, separator=separator) def dpath_search(self, glob, yielded=False): return dpath_search(self.inner, glob, yielded=yielded) def __contains__(self, key): return key in self.inner def __eq__(self, other): if isinstance(other, Subdocument): return self.inner == other.inner elif isinstance(other, dict): return self.inner == other else: return False def __len__(self): return len(self.inner) def __repr__(self): return repr(self.inner) class Document(Subdocument): def __init__(self): self.inner = Subdocument() def _insert(self, context: Context, value): if not context.rendered: # There is no key under which we could insert anything # This can happen with list-only or value-only # yaml documents, for which the doc object cannot be used # and will remain empty. return inner = self.inner for key in context.rendered[:-1]: if isinstance(inner, list): assert key <= len(inner), "bug: cannot insert at index > len(list)" if key == len(inner): inner.append(Subdocument()) else: if key not in inner: inner[key] = Subdocument() inner = inner[key] if isinstance(inner, list): i = context.rendered[-1] assert isinstance(i, int) and i == len(inner) inner.append(value) else: inner[context.rendered[-1]] = value yte-1.5.4/yte/exceptions.py000066400000000000000000000003761453555134700157000ustar00rootroot00000000000000class YteError(Exception): def __init__(self, msg, context): section = ( "in section /" + "/".join(context.template) if context else "at top level" ) super().__init__(f"Error processing template {section}: {msg}") yte-1.5.4/yte/process.py000066400000000000000000000203321453555134700151670ustar00rootroot00000000000000import re from yte.context import Context from yte.exceptions import YteError re_for_loop = re.compile(r"^\?for .+ in .+$") re_if = re.compile(r"^\?if (?P.+)$") re_elif = re.compile(r"^\?elif (?P.+)$") re_else = re.compile(r"^\?else$") FEATURES = frozenset(["variables", "definitions"]) def _process_yaml_value( yaml_value, variables: dict, context: Context, disable_features: frozenset, ): if isinstance(yaml_value, dict): return _process_dict(yaml_value, variables, Context(context), disable_features) elif isinstance(yaml_value, list): result = _process_list(yaml_value, variables, context, disable_features) return result elif _is_expr(yaml_value): result = _process_expr(yaml_value, variables, context) return result else: return yaml_value def _is_expr(yaml_value): return isinstance(yaml_value, str) and yaml_value.startswith("?") def _process_expr(yaml_value, variables, context: Context): try: return eval(yaml_value[1:], variables) except Exception as e: raise YteError(e, context) def _process_list( yaml_value, variables, context: Context, disable_features: frozenset, ): variables["doc"]._insert(context, []) def _process(): for i, item in enumerate(yaml_value): _context = Context(context) _context.rendered.append(i) value = _process_yaml_value(item, variables, _context, disable_features) if not isinstance(item, (dict, list)): variables["doc"]._insert(_context, value) yield value return list(_process()) def _process_dict( yaml_value, variables, context: Context, disable_features: frozenset, ): items = list(_process_dict_items(yaml_value, variables, context, disable_features)) if all(isinstance(item, dict) for item in items): result = dict() for item in items: result.update(item) return result elif all(isinstance(item, list) for item in items): return [item for sublist in items for item in sublist] elif len(items) == 1: return items[0] else: raise YteError( "Conditional or for loop did not consistently return map or list. " f"Returned: {items}", context, ) def _process_dict_items( yaml_value, variables, context: Context, disable_features: frozenset, ): conditional = Conditional() for key, value in yaml_value.items(): key_context = Context(context) key_context.template.append(key) if key == "__definitions__": if "definitions" in disable_features: raise YteError("__definitions__ have been disabled", key_context) _process_definitions(value, variables, key_context) elif key == "__variables__": if "variables" in disable_features: raise YteError("__variables__ have been disabled", key_context) _process_variables(value, variables, key_context) elif re_for_loop.match(key): yield from _process_for_loop( key, value, variables, conditional, key_context, disable_features ) elif re_if.match(key): yield from _process_if( key, value, variables, conditional, key_context, disable_features ) elif re_elif.match(key): _process_elif(key, value, variables, conditional, key_context) elif re_else.match(key): yield from _process_else( value, variables, conditional, key_context, disable_features ) else: # a normal key that will end up in the result yield from conditional.process_conditional( variables, key_context, disable_features ) key_result = _process_yaml_value( key, variables, key_context, disable_features ) key_context.rendered.append(key_result) value_result = _process_yaml_value( value, variables, key_context, disable_features ) variables["doc"]._insert(key_context, value_result) yield {key_result: value_result} yield from conditional.process_conditional(variables, key_context, disable_features) def _process_definitions(value, variables, context: Context): if isinstance(value, list): for item in value: try: exec(item, variables) except Exception as e: raise YteError(e, context) else: raise YteError( "__definitions__ keyword expects a list of Python statements", context ) def _process_variables(value, variables, context: Context): if isinstance(value, dict): for name, val in value.items(): if _is_expr(val): val = _process_expr(val, variables, context) variables[name] = val else: raise YteError( "__variables__ keyword expects a map of variable names and values", context ) def _process_for_loop( key, value, variables, conditional, context: Context, disable_features: frozenset ): yield from conditional.process_conditional(variables, context, disable_features) _variables = dict(variables) _variables["_yte_value"] = value _variables["_context"] = context _variables["_disable_features"] = disable_features _variables["_yte_variables"] = _variables yield from eval( f"[_process_yaml_value(_yte_value, dict(_yte_variables, **locals()), " "_context, _disable_features) " f"{key[1:]}]", _variables, ) def _process_if( key, value, variables, conditional, context: Context, disable_features: frozenset ): yield from conditional.process_conditional(variables, context, disable_features) expr = re_if.match(key).group("expr") conditional.register_if(expr, value) def _process_elif(key, value, variables, conditional, context: Context): if conditional.is_empty(): raise YteError("Unexpected elif: no if or elif before", context) expr = re_elif.match(key).group("expr") conditional.register_if(expr, value) def _process_else( value, variables, conditional, context: Context, disable_features: frozenset ): if conditional.is_empty(): raise YteError("Unexpected else: no if or elif before", context) conditional.register_else(value) yield from conditional.process_conditional(variables, context, disable_features) class Conditional: def __init__(self): self.exprs = [] self.values = [] def process_conditional( self, variables, context: Context, disable_features: frozenset ): if not self.is_empty(): variables = dict(variables) variables.update(self.value_dict) variables["_yte_variables"] = variables variables["_context"] = context variables["_disable_features"] = disable_features try: result = eval(self.conditional_expr(), variables) except Exception as e: raise YteError(e, context) if result is not None: yield result self.exprs.clear() self.values.clear() def conditional_expr(self, index=0): if index < len(self.exprs): return ( f"_process_yaml_value({self.value_name(index)}, " "_yte_variables, _context, _disable_features) " f"if {self.exprs[index]} else {self.conditional_expr(index + 1)}" ) if index < len(self.values): return ( f"_process_yaml_value({self.value_name(index)}, " "_yte_variables, _context, _disable_features)" ) else: return "None" def register_if(self, expr, value): self.exprs.append(expr) self.values.append(value) def register_else(self, value): self.values.append(value) def is_empty(self): return not self.exprs @property def value_dict(self): return {self.value_name(i): value for i, value in enumerate(self.values)} def value_name(self, index): return f"_yte_value_{index}"