pax_global_header00006660000000000000000000000064152056142270014516gustar00rootroot0000000000000052 comment=d13ad3a59d8d56dded4ecdee5886076a0f8d6dc6 siemens-kas-41ad961/000077500000000000000000000000001520561422700142705ustar00rootroot00000000000000siemens-kas-41ad961/.coveragerc000066400000000000000000000000441520561422700164070ustar00rootroot00000000000000[run] omit = /usr/* tests/* siemens-kas-41ad961/.dockerignore000066400000000000000000000000611520561422700167410ustar00rootroot00000000000000.git .gitignore .dockerignore Dockerfile .github siemens-kas-41ad961/.flake8000066400000000000000000000000501520561422700154360ustar00rootroot00000000000000[flake8] select = C,E,F,W ignore = W503 siemens-kas-41ad961/.github/000077500000000000000000000000001520561422700156305ustar00rootroot00000000000000siemens-kas-41ad961/.github/ISSUE_TEMPLATE/000077500000000000000000000000001520561422700200135ustar00rootroot00000000000000siemens-kas-41ad961/.github/ISSUE_TEMPLATE/custom.md000066400000000000000000000005031520561422700216450ustar00rootroot00000000000000--- name: The best way to contribute is via email to kas-devel@googlegroups.com about: Check out CONTRIBUTING.md for details title: '' labels: '' assignees: '' --- This project uses a mailing list workflow for contributions. Using Issues is allowed but discouraged. Remove this line if you want to open an Issue anyways. siemens-kas-41ad961/.github/actions/000077500000000000000000000000001520561422700172705ustar00rootroot00000000000000siemens-kas-41ad961/.github/actions/docker-init/000077500000000000000000000000001520561422700215005ustar00rootroot00000000000000siemens-kas-41ad961/.github/actions/docker-init/action.yml000066400000000000000000000104321520561422700235000ustar00rootroot00000000000000name: docker-init inputs: deploy-user: required: true deploy-token: required: true image-name: required: true distro-release: required: false runs: using: composite steps: - name: Set up QEMU shell: bash env: QEMU_USER_STATIC_PACKAGE: qemu-user-static_7.2+dfsg-7+deb12u12_amd64.deb REPO_DATE: 20250130T084806Z PACKAGE_SHA256: 1a2696081c1f30d464f79fd300196822397c77f05440ea9ce6dc8e9658b595ec run: | # temporarily use Debian qemu-user-static until Ubuntu fixes theirs wget -q http://snapshot.debian.org/archive/debian/${REPO_DATE}/pool/main/q/qemu/${QEMU_USER_STATIC_PACKAGE} echo "${PACKAGE_SHA256} ${QEMU_USER_STATIC_PACKAGE}" | sha256sum -c sudo dpkg -i ${QEMU_USER_STATIC_PACKAGE} - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 with: driver-opts: image=moby/buildkit:v0.16.0 - name: Login to ghcr.io uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0. with: registry: ghcr.io username: ${{ inputs.deploy-user }} password: ${{ inputs.deploy-token }} - name: Set SOURCE_DATE_EPOCH run: | echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV shell: bash - name: Determine Debian tag run: | if [[ ! "${{ inputs.distro-release }}" == debian-* ]]; then echo "Non debian 'distro-release' values are currently not supported" exit 1 fi COMMIT_DATE=$(date -d @$(git log -1 --pretty=%ct) +%Y%m%d) DEBIAN_LATEST_RELEASE=$(grep -m 1 'ARG DEBIAN_TAG=' Dockerfile | sed 's/.*DEBIAN_TAG=\(.*\)-.*/\1/') if [ -n "${{ inputs.distro-release }}" ]; then DISTRO_RELEASE="${{ inputs.distro-release }}" DEBIAN_RELEASE="${DISTRO_RELEASE#debian-}" else DEBIAN_RELEASE=$DEBIAN_LATEST_RELEASE fi echo "DEBIAN_TAG=$(podman search --list-tags docker.io/debian --limit 1000000000 | \ grep "$DEBIAN_RELEASE-.*-slim" | sort -r | sed 's/.*[ ]\+//' | \ ./scripts/lower-bound.py $DEBIAN_RELEASE-$COMMIT_DATE-slim )" \ >> $GITHUB_ENV echo "DISTRO_LATEST_RELEASE=debian-$DEBIAN_LATEST_RELEASE" >> $GITHUB_ENV shell: bash - name: Prepare repository for COPY-in run: | git clone . /home/runner/kas-clone shell: bash - name: Define image metadata run: | case ${{ inputs.image-name }} in kas) echo "IMAGE_DESCRIPTION=kas build environment for Yocto/OpenEmbedded projects" >> $GITHUB_ENV ;; kas-isar) echo "IMAGE_DESCRIPTION=kas build environment for isar-based Debian projects" >> $GITHUB_ENV ;; esac # make image metadata reproducible (also for image re-builders) echo "IMAGE_COMMIT_DATE=$(date -d @$(git log -1 --pretty=%ct) --iso-8601=seconds)" >> $GITHUB_ENV echo "IMAGE_OFFICIAL_URL=https://github.com/siemens/kas" >> $GITHUB_ENV shell: bash - name: Extract metadata id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v5.0.0 with: annotations: | org.opencontainers.image.description=${{ env.IMAGE_DESCRIPTION }} org.opencontainers.image.licenses=MIT and others org.opencontainers.image.created=${{ env.IMAGE_COMMIT_DATE }} org.opencontainers.image.source=${{ env.IMAGE_OFFICIAL_URL }} org.opencontainers.image.url=${{ env.IMAGE_OFFICIAL_URL }} env: DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - name: Cache apt id: cache-apt uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: | var-cache-apt var-lib-apt key: cache-apt-${{ env.DEBIAN_TAG }}-${{ inputs.image-name }} - name: Inject cache into docker uses: reproducible-containers/buildkit-cache-dance@1b8ab18fbda5ad3646e3fcc9ed9dd41ce2f297b4 #v3.3.2 with: cache-map: | { "var-cache-apt": "/var/cache/apt", "var-lib-apt": "/var/lib/apt" } skip-extraction: ${{ steps.cache.outputs.cache-hit }} siemens-kas-41ad961/.github/actions/perform-tests/000077500000000000000000000000001520561422700221025ustar00rootroot00000000000000siemens-kas-41ad961/.github/actions/perform-tests/action.yml000066400000000000000000000031211520561422700240770ustar00rootroot00000000000000name: perform-tests inputs: python-version: required: true runs: using: composite steps: - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ inputs.python-version }} architecture: x64 - name: Install Python dependencies of kas and tests shell: bash run: | # install kas to have all kas dependencies: pip install .[test] - name: Install python-newt shell: bash run: | sudo apt-get update sudo apt-get install libpopt-dev libslang2-dev wget -q https://releases.pagure.org/newt/newt-$NEWT_VERSION.tar.gz echo "$NEWT_SHA256 newt-$NEWT_VERSION.tar.gz" | sha256sum -c tar -C /tmp -xzf newt-$NEWT_VERSION.tar.gz cd /tmp/newt-$NEWT_VERSION autoconf ./configure --with-python=python${{ matrix.python-version }} make -j $(nproc) sudo make install ln -s /usr/local/lib/python${{ matrix.python-version }}/site-packages/_snack.so \ $(python3 -c 'import site; print(site.getsitepackages()[0])')/ ln -s /usr/local/lib/python${{ matrix.python-version }}/site-packages/snack.py \ $(python3 -c 'import site; print(site.getsitepackages()[0])')/ - name: Run offline tests shell: bash env: TERM: xterm http_proxy: http://0.0.0.0:8118 https_proxy: http://0.0.0.0:8118 run: pytest -m "not online" - name: Run online tests shell: bash env: TERM: xterm run: pytest -m "online" siemens-kas-41ad961/.github/workflows/000077500000000000000000000000001520561422700176655ustar00rootroot00000000000000siemens-kas-41ad961/.github/workflows/master.yml000066400000000000000000000040001520561422700216750ustar00rootroot00000000000000name: master on: push: branches: - master jobs: deploy_containers: name: Build and deploy container images runs-on: ubuntu-24.04 permissions: id-token: write packages: write contents: read attestations: write artifact-metadata: write strategy: matrix: image-name: ["kas", "kas-isar"] distro-release: ["debian-bookworm", "debian-trixie"] steps: - name: Check out repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Set up docker build uses: ./.github/actions/docker-init with: deploy-user: ${{ github.actor }} deploy-token: ${{ secrets.GITHUB_TOKEN }} image-name: ${{ matrix.image-name }} distro-release: ${{ matrix.distro-release }} - name: Build ${{ matrix.image-name }} image uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 id: push with: context: /home/runner/kas-clone target: ${{ matrix.image-name }} platforms: linux/amd64,linux/arm64 build-args: | SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} DEBIAN_TAG=${{ env.DEBIAN_TAG }} provenance: false outputs: type=registry,rewrite-timestamp=true tags: | ${{ matrix.distro-release == env.DISTRO_LATEST_RELEASE && format('ghcr.io/{0}/{1}:latest', github.repository, matrix.image-name) || '' }} ghcr.io/${{ github.repository }}/${{ matrix.image-name }}:latest-${{ matrix.distro-release }} annotations: ${{ env.DOCKER_METADATA_OUTPUT_ANNOTATIONS }} - name: Attest ${{ matrix.image-name }} image uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 with: subject-name: ghcr.io/${{ github.repository }}/${{ matrix.image-name }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true siemens-kas-41ad961/.github/workflows/next.yml000066400000000000000000000133511520561422700213710ustar00rootroot00000000000000name: next on: push: branches-ignore: - 'master' # keep as path to test exclusions - 'ci/ignore-master' env: SHELLCHECK_VERSION: v0.10.0 SHELLCHECK_SHA256: 6c881ab0698e4e6ea235245f22832860544f17ba386442fe7e9d629f8cbedf87 NEWT_VERSION: 0.52.24 NEWT_SHA256: 5ded7e221f85f642521c49b1826c8de19845aa372baf5d630a51774b544fbdbb jobs: codestyle: name: Code style runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Checkcode dependencies run: | pip install flake8 pycodestyle doc8 Pygments - name: Install recent shellcheck run: | wget -q https://github.com/koalaman/shellcheck/releases/download/$SHELLCHECK_VERSION/shellcheck-$SHELLCHECK_VERSION.linux.x86_64.tar.xz echo "$SHELLCHECK_SHA256 shellcheck-$SHELLCHECK_VERSION.linux.x86_64.tar.xz" | sha256sum -c tar -xJf shellcheck-$SHELLCHECK_VERSION.linux.x86_64.tar.xz sudo cp shellcheck-$SHELLCHECK_VERSION/shellcheck /usr/local/bin/ - name: Check code run: scripts/checkcode.sh . perform_tests: name: Modern Python runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Check out repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: PyTest uses: ./.github/actions/perform-tests with: python-version: ${{ matrix.python-version }} build_containers: name: Build, test and deploy container images needs: - codestyle - perform_tests runs-on: ubuntu-24.04 permissions: id-token: write packages: write contents: read attestations: write artifact-metadata: write strategy: matrix: image-name: ["kas", "kas-isar"] distro-release: ["debian-bookworm", "debian-trixie"] steps: - name: Check out repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up docker build uses: ./.github/actions/docker-init with: deploy-user: ${{ github.actor }} deploy-token: ${{ secrets.GITHUB_TOKEN }} image-name: ${{ matrix.image-name }} distro-release: ${{ matrix.distro-release }} - name: Build ${{ matrix.image-name }} image uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 with: context: /home/runner/kas-clone target: ${{ matrix.image-name }} platforms: linux/amd64 build-args: | SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} DEBIAN_TAG=${{ env.DEBIAN_TAG }} outputs: type=docker,rewrite-timestamp=true tags: ghcr.io/${{ github.repository }}/${{ matrix.image-name }}:next-${{ matrix.distro-release }} - name: Test ${{ matrix.image-name }} image env: KAS_CONTAINER_IMAGE: ghcr.io/${{ github.repository }}/${{ matrix.image-name }}:next-${{ matrix.distro-release }} KAS_CLONE_DEPTH: 1 run: | cd image-tests/${{ matrix.image-name }} ../../kas-container build kas.yml if [ "${{ matrix.image-name }}" = "kas" ]; then ../../kas-container build kas-buildtools.yml fi [ -d build/tmp ] echo "Test kas clean" ../../kas-container clean kas.yml ! [ -d build/tmp/deploy ] [ -d build/sstate-cache ] && [ -d build/downloads ] [ -d poky ] || [ -d isar ] echo "Test kas purge" ../../kas-container purge kas.yml ! [ -d poky ] && ! [ -d isar ] && ! [ -d build ] # kas only: build oe-core nodistro example if [[ "${{ matrix.image-name }}" == "kas" ]]; then echo "Test oe-core nodistro" ../../kas-container build ../../examples/oe-core-nodistro.yml fi - name: Complete build and deploy ${{ matrix.image-name }} image if: github.ref == 'refs/heads/next' uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 id: push with: context: /home/runner/kas-clone target: ${{ matrix.image-name }} platforms: linux/amd64,linux/arm64 build-args: | SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} DEBIAN_TAG=${{ env.DEBIAN_TAG }} provenance: false outputs: type=registry,rewrite-timestamp=true tags: | ${{ matrix.distro-release == env.DISTRO_LATEST_RELEASE && format('ghcr.io/{0}/{1}:next', github.repository, matrix.image-name) || '' }} ghcr.io/${{ github.repository }}/${{ matrix.image-name }}:next-${{ matrix.distro-release }} annotations: ${{ env.DOCKER_METADATA_OUTPUT_ANNOTATIONS }} - name: Attest ${{ matrix.image-name }} image if: github.ref == 'refs/heads/next' uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 with: subject-name: ghcr.io/${{ github.repository }}/${{ matrix.image-name }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true cleanup_ghcr_containers: name: cleanup untagged ${{ matrix.image-name }} containers if: github.ref == 'refs/heads/next' runs-on: ubuntu-24.04 needs: build_containers permissions: packages: write strategy: matrix: image-name: ["kas", "kas-isar"] steps: - uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16 with: dry-run: false validate: true package: kas/${{ matrix.image-name }} token: ${{ secrets.GITHUB_TOKEN }} siemens-kas-41ad961/.github/workflows/release.yml000066400000000000000000000046341520561422700220370ustar00rootroot00000000000000name: release on: push: tags: - '*.*' jobs: deploy_containers: name: Build and deploy container images runs-on: ubuntu-24.04 permissions: id-token: write packages: write contents: read attestations: write artifact-metadata: write strategy: matrix: image-name: ["kas", "kas-isar"] distro-release: ["debian-bookworm", "debian-trixie"] steps: - name: Check out repo uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Get release run: echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - name: Set up docker build uses: ./.github/actions/docker-init with: deploy-user: ${{ github.actor }} deploy-token: ${{ secrets.GITHUB_TOKEN }} image-name: ${{ matrix.image-name }} distro-release: ${{ matrix.distro-release }} - name: Find latest tag run: echo "LATEST_TAG=$(git tag | sort --version-sort | tail -n1)" >> $GITHUB_ENV - name: Build ${{ matrix.image-name }} image uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 id: push with: context: /home/runner/kas-clone target: ${{ matrix.image-name }} platforms: linux/amd64,linux/arm64 build-args: | SOURCE_DATE_EPOCH=${{ env.SOURCE_DATE_EPOCH }} DEBIAN_TAG=${{ env.DEBIAN_TAG }} provenance: false outputs: type=registry,rewrite-timestamp=true tags: | ghcr.io/${{ github.repository }}/${{ matrix.image-name }}:${{ env.RELEASE_VERSION }}-${{ matrix.distro-release }} ${{ matrix.distro-release == env.DISTRO_LATEST_RELEASE && format('ghcr.io/{0}/{1}:{2}', github.repository, matrix.image-name, env.RELEASE_VERSION) || '' }} ${{ matrix.distro-release == env.DISTRO_LATEST_RELEASE && github.ref_name == env.LATEST_TAG && format('ghcr.io/{0}/{1}:latest-release', github.repository, matrix.image-name) || '' }} annotations: ${{ env.DOCKER_METADATA_OUTPUT_ANNOTATIONS }} - name: Attest ${{ matrix.image-name }} image uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 with: subject-name: ghcr.io/${{ github.repository }}/${{ matrix.image-name }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true siemens-kas-41ad961/.gitignore000066400000000000000000000011441520561422700162600ustar00rootroot00000000000000### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ # Translations *.mo *.pot # Sphinx documentation docs/_build/ # pyenv .python-version # dotenv .env # virtualenv .venv venv/ ENV/ # release automation release-email.txt siemens-kas-41ad961/.readthedocs.yaml000066400000000000000000000027061520561422700175240ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2021-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. version: 2 build: os: ubuntu-24.04 tools: python: "3.12" apt_packages: - python3-newt - make - podman jobs: pre_build: - cd docs && make kas-container-usage && cd .. python: install: - method: pip path: . - requirements: docs/requirements.txt sphinx: configuration: docs/conf.py siemens-kas-41ad961/CHANGELOG.md000066400000000000000000000516561520561422700161160ustar00rootroot000000000000005.3 - kas: git: Avoid checking out sha-like branches as commits (CVE-2026-47191) - kas: verify signatures prior to checkout (CVE-2026-47192) - kas: strip credentials from attestation also if token is used - kas: ensure _source_dir is only set from main config file - kas: ensure git-clone path is not processed as option - kas: drop never correctly support for absolute include path - kas: limit include path traversals to repository - kas: Warn about repos with branches but without commit or lock file - kas: create a CACHEDIR.TAG in the kas build directory - kas: add Arch Linux to supported distros for locale settings - kas: schema: switch default distro to nodistro - kas: schema: enforce signer config constraints via schema - kas: dump: Use 2 spaces as indention in generated yaml - kas: Properly convert error list to string prior to output - kas: improve printing of os version - kas-container: do not construct image name if providing KAS_CONTAINER_IMAGE - kas-container: Fix podman detection - kas-container: do not process locale aliases - kas-container: query system docker path in isar mode, not assuming it - docs: Document a simpler way to disable layers in a repo - docs: Clarify environment variable handling - release: Publish both source and wheel to PyPI 5.2 - kas: Add support for layer priorities - kas: Add support for AWS SSO login credentials - kas: Allow to specify commits via id of annotated tags as well - kas: Avoid unneeded commit ID updates in lockfiles - kas: Catch error when updating an invalid commit in a lockfile - kas: Disable terminal prompts in git - kas: Improve error logging on failed fetches - kas: Avoid redundant error logging of invoked commands - kas: Improve error logging of failed for_all_repos commands - docs: Add community support section to getting started guide - docs: Show schema defaults that evaluate to false - docs: Show allowed integer ranges in the schema - ci: Fix pushing of latest container tag on releases 5.1 - kas: use DL_DIR environment variable on cleanall - kas: Print distro name and version on startup - kas-container: Install binfmts for qemu manually - kas-container: add custom git path to safedirs - kas-container: restore debian sources list after installation - docs: Update installation instructions - docs: add missing property to buildtools examples 5.0 - kas: introduce buildtools support - kas: Allow branch override - kas: Dump output of init-build-env scripts at debug log-level - kas: fail on fetch errors, rather than warn and continue - kas: only fetch repos on update if commit is not pinned - kas: call sudo only once in ISAR mode of clean plugin - kas: avoid redundant diff output of filename - kas: print commit log on lock update - kas-container: Provide images based on both Debian 12 and 13 - kas-container: allow to select specific image distribution - kas-container: add support for KAS_BUILDTOOLS_DIR - kas-container: add partial support for run0 - docs: add oe-core nodistro example 4.8.2 - kas: clean: avoid creation of tmp data to recover from full-disk scenario - kas: avoid duplicate creations of tmpdir - kas-container: fix host timezone propagation, resolving yocto breakage - kas-container: restore git safe.directory exception outside of kas for GitLab CI - kas-container: avoid collision of internal vars with environment - kas-container: fallback to xterm if a TERM value is not supported - docs: precisely describe where SHELL env var is in effect - docs: add note on YAML support and its limitations - docs: remove GnuPG limitation of repo verification - docs: sphinx_kas_schema.py: support numbers in regex 4.8.1 - kas-container: Update new version variable on release, fixing mismatch in 4.8 4.8 - kas: Add diff plugin to compare config files and repos - kas: add support to verify signatures of git repos - kas: add support for distributed lock file updates - kas: add purge command to remove all managed data - kas: add support for NPMRC_FILE environment variable - kas: GitLab CI: add git safe dir for project root - kas: Lift minimal Python version to 3.9 - kas: drop overrides from output of dump when using --resolve-refs - kas: inject current version of kas into config dumps - kas: enforce allowed values for layer enabling/disabling - kas: deprecate various magic values to disable a layer - kas: consistently check for the validity of path provided via env vars - kas: skip shallow cloning for reference repos - kas: improve reporting of yaml format errors - kas: truly make kconfiglib a weak dependency - kas: fix output on patching errors - kas: perform packaging via pyproject.toml - kas-container: add limited support for docker rootless - kas-container: propagate timezone information into container - kas-container: warn if script version does not match container - kas-container: prepare for looser coupling of script and container versions - kas-container: install python3-cairo for pybootchartgui support - kas-container: add bubblewrap package to kas-isar - kas-container: do not ship kas-container script within the container - kas-container: do not ship unneeded artifacts of kas repository - docs: precisely describe how config files are merged - docs: make documentation of project layout more precise - docs: specify envs that are processed prior to reading the config 4.7 - kas: preserve history across shell plugin invocations - kas: adjust/improve dirty repo detection - kas: only sort keys in dump plugin output if explicitly requested - kas: ensure consistent config ordering in dump output - kas: enforce maximum header version when merging configs - kas: only include config in attestation if not tracked - kas: fix repo type identifier for mercurial in attestations - kas: improve annotation of kas-generated repo commits - kas: improve schema validation error messages - kas: show both commit ids in error message on tag commit mismatch - kas-container: do not create KAS_WORK_DIR - kas-container: add support for git worktrees - kas-container: add support for Ubuntu 24.04 hosts - kas-container: account for lock plugin in container entrypoint - kas-container: make AWS_WEB_IDENTITY_TOKEN_FILE errors more precise - docs: document rules for provided directories - docs: describe steps needed to add a new plugin - docs: auto add enum values of schema node 4.6 - kas: move lock file handling to new lock plugin - kas: add support for running inside vscode devcontainer - kas: keep repos in their current state when using -k - kas: permit kas config snippets in git submodules of main repo - kas: do not apply patches to unclean repos - kas: GitLab CI: allow ssh config adjustments in more cases - kas: add override to remove default tag or branch - kas: order layers by repo and layer name in bblayers.conf - kas: deprecate non full-length digests for git repos - kas: mercurial: create valid branch names on patching - kas: attestation: make repo digest sha256-aware - kas: make version property in json schema more strict - kas: fix repo processing in the presence of identical names - kas: fix resource leaks on command execution error - kas: fix/improve event loop and termination handling - kas: Drop -d option - kas-container: Drop already deprecated -d and -v options - kas-container: gracefully handle rate-limits of snapshot.d.o - docs: document kas-container command - docs: enhance json schema with descriptions - docs: kas-container: document how to select image - docs: add note about snack test dependency 4.5 - kas: avoid bitbake parsing due to non-deterministic layer patches - kas: correctly handle upstream ff merges on fetch - kas: keep git committer identity if provided in .gitconfig - kas: add support for container registry authentication - kas: Improve GitLab CI rewrite rules for git - kas-container: Use official snapshot.debian.org - kas-container: Fix positional argument processing with for-all-repos - kas-container: allow recent Ubuntu builds via isar - kas-container: re-add deterministic metadata - docs: document difference between file and env credentials - sign pip packages on release 4.4 - kas: Auto-import runner-provided .gitconfig also inside GitLab-CI - kas: Auto-inject git credentials on gitlab ci - kas: Add --keep-config-unchanged to preserve repos and configs on actions - kas: Fix --skip'ing multiple steps - kas: List --skip'able steps in --help - kas: Add support for shallow clones - kas: Add support to create provenance build attestations - kas: Add config key to describe build artifacts (used by attestation) - kas: Add option to dump-plugin to include VCS info of local repos - kas-container: Handle missing extra argument in subcommands gracefully - kas-container: improve container reproduction using git commit date - docs: Several format improvements - docs: Add simple examples 4.3.2 - kas: don't add comments to .netrc, fixing gitlab-ci - kas: make file permissions on credentials more strict (not a security fix) - kas: align hg semantics of repo dirty checking - kas-container: fix warnings from shellcheck 0.9.0 - docs: do not build docs against installed version - docs: update to match bitbake variable changes - docs: unify spelling of kas - docs: document scope of environment variables 4.3.1 - kas: Fix regression of 4.3 when using SSH_PRIVATE_KEY[_FILE] - kas-container: Update to debian:bookworm-20240311-slim (implicitly) 4.3 - kas: fix including from transitively referenced repos - kas: Add support for .gitconfig pass-through - kas: Optimize checkout of repos in larger configurations - kas: Reduce verbosity of kas startup output - kas: check if branch contains commit if both are set - kas: Improve error reporting in several places - kas-container: Bit-identically reproducible images - kas-container: Enrich manifests with provenance information - kas-container: Add bash completion for kas - docs: Separate man pages per subcommand - docs: Various smaller improvements 4.2 - kas: Fix lock files when references repos by tags - kas: add forgotten `tag` key to repos `defaults` - kas: add support for OAuth2 worflow - kas-container: add python3-websockets - kas-container: unify error handling 4.1 - kas: Add "tag" property to repo, to replace usage of refspec - kas: generalize revision locking to all included files - kas: allow for --skip repos_checkout - kas: forward SSTATE_MIRRORS environment variable - kas: Allow PyYAML 6, fixing dependency conflicts - kas: Add Python 3.12 support - kas: Update jsonschema upper version limit - kas: menu plugin: Reorder help and exit buttons - kas: menu plugin: Add separate return button for submenus - kas: Fix Mercurial's branch resolution - kas-container: detect build system on clean commands - kas-container: report error if ssh-agent is requested but not running 4.0 - kas container: Switch to Debian bookworm - kas-container: Make kas-isar ready for mmdebstrap 3.3 - kas: Introduce commit and branch as alternative to refspec key - kas: Warn if a repo uses legacy refspec - kas: add support for lock files via dump plugin - kas: track root repo dir config files of menu plugin - kas: add support for --log-level argument - kas: add GIT_CREDENTIAL_USEHTTPPATH environment variable - kas: improve error reporting - kas: drop support for Python 3.5 - kas-container: fix invocations with --isar for some layers - kas-container: Purge tmp* on clean - kas-container: enable colored logging 3.2.3 - kas-container: mount KAS_REPO_REF_DIR rw to support auto-creation - kas-container: fix --ssh-dir (3.2.2 regression) - container: Use original UID/GID when run without kas-container (3.2.2 regression) 3.2.2 - kas-container: Start as non-root when running without kas-container - kas-container: Disable git safe.directory when running without kas-container - kas-container: Make sure privileged podman will find sbin tools - docs: Leave notice on inherit integrity weaknesses of repo fetches - docs: Add a SECURITY.md 3.2.1 - kas-container: Add unzip package to kas-base - docs: Fix description of container image generation - docs: Fix description of bblayers_conf_header and local_conf_header 3.2 - kas: add conditional, default-free environment variables - kas: add plugin to dump flattened config and resolve repo refs - kas: auto-create repo refs when KAS_REPO_REF_DIR is set - kas: print build bitbake command when running shell - kas: forward BB_NUMBER_THREADS and PARALLEL_MAKE env vars into build - kas-container: Fix engine detection when docker is an alias for podman - kas-container: forward DISTRO_APT_PREMIRRORS environment variable - kas-container: reduce log chattiness of container runtime - kas-container: write debug messages to stderr - kas-container: Refresh Yocto build dependency list - kas-container: Rework generation of kas images, shrinking kas-isar - kas-container: avoid deploying the python pip cache 3.1 - kas: Add support for authentication with gitlab CI - kas: Add NETRC_FILE to allow passing credentials into kas home - kas: for-all-repos: Add option to keep current env - kas: Avoid whitespace warnings when applying repo patches - kas: Use relative layer dirs to make build relocatable - kas: Allow "deleting" url/path of repo in override - kas: Fix repo-relative include file handling if no config file is given - kas: Fix include errors from repos defined via multiple yaml files - kas: Fix handling of -- separator in the absence of a config file - kas: Bundle kas-container script - kas-container: Add support for podman >= 4.1 - kas-container: Add '--ssh-agent' option - kas-container: Add telnet to image - kas-container: Remove obsolete schroot mntpoint - kas-container: Reduce the image size a bit 3.0.2 - kas-container: Fix the fix for chatty sbuild-adduser in kas-isar 3.0.1 - kas-container: Silence chatty sbuild-adduser in kas-isar 3.0 - kas: git fetch always with quiet flag, suppressing false error messages - kas: Add BB_ENV_PASSTHROUGH_ADDITIONS support - kas: shell: Add option to keep current environment - kas: Raise an error on missing repo refspec - kas-container: Base containers on bullseye - kas-container: Add pigz package to container to enable parallel compression - kas-container: Support for sbuild in kas-isar - kas-container: podman: Remove --pid=host - kas-container: Start init service inside container - kas-container: Add cleansstate and cleanall - kas-container: Pass http_proxy et.al through sudo - kas-container: Address shellcheck findings in container-entrypoint - docs: Add recommendation for repo-id naming - docs: Clarify local file include paths 2.6.3 - kas: Do not overwrite existing .ssh/config - kas: Properly describe package build - kas-container: create KAS_WORK_DIR if it not exists - kas-container: validate KAS_REPO_REF_DIR correctness - docs: Fix generation - docs: Extended "layers" section in the user guide. 2.6.2 - kas-container: Restore oe-git-proxy location (/usr/bin) - kas-container: Drop world-write permission from /kas folder 2.6.1 - kas: fix installation via pip 2.6 - kas: Add kconfiglib-based menu plugin - kas: Enable kas to checkout repositories using git credentials - kas: Enable gerrit/gitlab/github refspecs - kas: Write more bblayers.conf boilerplate settings - kas: Add environment variable SSH_PRIVATE_KEY_FILE - kas: Add support for relative KAS_WORK/BUILD/REPO_REF_DIR paths - kas: Move config json schema to standalone json file - kas: Avoid duplicate cloning of repos in command line includes - kas: for_all_repos: Exit on command failure - kas: for_all_repos: Fix KAS_REPO_URL or unversioned repos - kas: Declare proxy_config obsolete - kas-container: install lz4 - kas-container: install g++-multilib - kas-container: install newer git-lfs - kas-container: Enter with /repo as current dir - kas-container: Carry oe-git-proxy locally and relocate to /usr/local/bin 2.5 - kas: Apply patches before doing an environment setup - kas: repos: strip dot from layer name - kas: Introduce KAS_BUILD_DIR environment variable - kas: add GIT_CREDENTIAL_HELPER environment variable - kas-container: add `--git-credential-store` options - kas-container: mount /repo as read-write for shell command - kas-container: add an argument to get version information - kas-container: Add support for checkout and for-all-repos - kas-container: add support to set a custom container images location - kas-container: Fix mounting of custom KAS_REPO_REF_DIR - kas-container: Add skopeo and umoci to ISAR image - kas-container: add sudo to standard kas image 2.4 - kas: Silence "Exception ignored when trying to write to the signal wakeup fd" - kas: drop bitbakes "-k" from the default args - kas: fix repos path if no url, but path given - kas: Set upper version limit for dependencies - kas-container: Add support for rootless podman with userns keep-id - kas-container: Add support for multi-word --command arguments - kas-container: make sure that we pass shellcheck - kas-container/kas*: Add support for multi-arch containers - kas-container/kas: Pull all Python dependencies from Debian - kas-container/kas-isar: Drop grub package 2.3.3 - Fix binfmt setup in kas-isar container image 2.3.2 - Fix release script fix /wrt kas-container image version updates 2.3.1 - Fix release scripting 2.3 - kas: add "checkout" and for-all-repos subcommands - kas: add python 3.9 compatibility - kas: improve documentation - config: add build_system property to pre-select OE/Yocto or Isar - kas-container: rename from kas-docker - kas-container: add support for build_system property (making --isar optional) - kas-container: adjust environment variables interface - kas-container: switch to github container repository - kas-container: add support for Debian bullseye cross building - kas-container: add zstd package 2.2 - kas: allow extra bitbake arguments to be passed - kas: add --force-checkout and --update arguments to ease CI usage - kas: allow for layer-free repositories - kas: fix cloning of repos without default branch - kas: enable standard-conforming .yaml file extensions - kas-docker: enhance with podman support - kas-docker: switch to /bin/bash as SHELL per default - config: Allow a default refspec to be specified - config: Allow a default repo to be specified for patches 2.1.1 - repos: Silence pycodestyle error (that broke docker image generation) 2.1 - Add support for S3 fetcher to docker image - Lift Python minimal requirements to 3.5 - Fix reporting of of repo patch IDs - config: use 'qemux86-64' instead of 'qemu' as default for KAS_MACHINE - Ensure SSH key ends with newline - kas-docker: Make it harder to run as root - kas-docker: Make loop device passing optional - kas-docker: Various fixes 2.0 - Add support for Yocto 3.0 / latest Isar - Move docker image to Debian buster - Add git-lfs support to docker image - Add Yocto testimage dependencies to docker image 1.1 - Restore mercurial support - Add -c and --cmd as aliases for --task - Fix repo patching when using a branch name as refspec - Update repo remote URL on kas file changes - kas-docker: fix SHELL forwarding - kas-docker: use released image, rather than "latest" - kas-docker: allow to define custom image version - kasproject/kas: enable devshell and menuconfig targets - kasproject/kas image: add gnupg and quilt - kasproject/kas-isar image: fix /var/tmp handling 1.0 - isar: Take qemu-user-static from buster and adjust binfmt setup 0.20.1 - kas-docker: Restore KAS_PREMIRRORS support 0.20.0 - kas-docker: enable passing SSH configs - kas-docker: add --no-proxy-from-env option - kas-docker: Pass in NO_PROXY - Add KAS_PREMIRRORS support - Remove SSH_AGENT_PID forwarding 0.19.0 - Recursive include handler refactoring and cleanups - A lot of code cleanups, refactoring and bug fixings - Isar docker support improvements 0.18.0 - Add patch support for repos - Use git diff-index to check if repo is dirty - docker: add debootstrap and qemu-user-static 0.17.0 - Add iproute and zx-utils to the docker image - Fix relative path for repos - Write MACHINE and DISTRO as weak defaults 0.16.0 - Support Mercurial repos - Support Gentoo distro 0.15.0 - Environment variable passthrough - Support major distro variants - Add initial support for multiconfig 0.14.0 - Multi-target support - Avoid downloading same repo twice 0.13.0 - Increase config file version 0.12.0 - Remove dynamic configuration support (Python config files) - Shell command prepares complete bitbake configuration - Add to define task in config and environment - Improved error handling and reporting 0.11.0 - Allow in-tree repos not to be in a git repo - Pass through git proxy related environment variables - Write deterministic local.conf and bblayers.con - Make configuration file versioning independent of project version - Cleanups for uploading project to PyPI - Print proper error message for config file format exception 0.10.0 - Docker image creation (Debian Stretch), pushed on kasproject/kas - Restructure documentation add support for Sphinx export it to readthedocs - Add support for include feature for Yaml files - Add support for Isar build system - Handling of SIGTERM/TERM improved - Parallel download of git sources - Allow environment to overwrite proxy, target, machine and distro - Add unit testing for include/merge config file handling - Rename sublayers back to layers - pylint & pep8 cleanups - Allow to define workdir via KAS_WORK_DIR - Shell honors SHELL and TERM environment variable 0.9.0 - initial public release siemens-kas-41ad961/CONTRIBUTING.md000066400000000000000000000107211520561422700165220ustar00rootroot00000000000000Contributing to kas =================== Contributions to kas are always welcome. This document explains the general requirements on contributions and the recommended preparation steps. It also sketches the typical integration process of patches. Contribution Checklist ---------------------- - use git to manage your changes [*recommended*] - follow Python coding style outlined in pep8 [**required**] - add the required copyright header to each new file introduced, see [licensing information](LICENSE) [**required**] - structure patches logically, in small steps [**required**] - one separable functionality/fix/refactoring = one patch - do not mix those there in a single patch - after each patch, the tree still has to build and work, i.e. do not add even temporary breakages inside a patch series (helps when tracking down bugs) - use `git rebase -i` to restructure a patch series - base patches on top of latest master or - if there are dependencies - on next (note: next is an integration branch that may change non-linearly) - test patches sufficiently (obvious, but...) [**required**] - no regressions are caused in affected code - the world is still spinning - add signed-off to all patches [**required**] - to certify the "Developer's Certificate of Origin", see below - check with your employer when not working on your own! - post patches to mailing list [**required**] - use `git format-patch/send-email` if possible - send patches inline, do not append them - no HTML emails! - CC people who you think should look at the patches, e.g. - someone who wrote a change that is fixed or reverted by you now - who commented on related changes in the recent past - who otherwise has expertise and is interested in the topic - pull requests on github are only optional - post follow-up version(s) if feedback requires this - send reminder if nothing happened after about a week Developer's Certificate of Origin 1.1 ------------------------------------- When signing-off a patch for this project like this Signed-off-by: Random J Developer using your real name (no pseudonyms or anonymous contributions), you declare the following: By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. Contribution Integration Process -------------------------------- 1. patch reviews performed on mailing list * at least by maintainers, but everyone is invited * feedback has to consider design, functionality and style * simpler and clearer code preferred, even if original code works fine 2. accepted patches merged into next branch 3. further testing done by community, including CI build tests and code analyzer runs 4. if no new problems or discussions showed up, acceptance into master * grace period for master: about 3 days * urgent fixes may be applied sooner github facilities are not used for the review process so that people can follow all changes and related discussions at a single stop, the mailing list. This may change in the future if github should improve their email integration. Send patches to: kas-devel@googlegroups.com Archive: https://groups.google.com/d/forum/kas-devel Subscription: - kas-devel+subscribe@googlegroups.com - https://groups.google.com/forum/#!forum/kas-devel/join siemens-kas-41ad961/Dockerfile000066400000000000000000000156371520561422700162760ustar00rootroot00000000000000# # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ARG DEBIAN_TAG=trixie-slim FROM debian:${DEBIAN_TAG} AS kas-base ARG SOURCE_DATE_EPOCH ARG CACHE_SHARING=locked ARG DEBIAN_TAG=trixie-slim ENV DEBIAN_BASE_IMAGE_TAG=${DEBIAN_TAG} ARG TARGETPLATFORM ARG DEBIAN_FRONTEND=noninteractive ENV LANG=en_US.utf8 RUN --mount=type=cache,target=/var/cache/apt,sharing=${CACHE_SHARING} \ --mount=type=cache,target=/var/lib/apt,sharing=${CACHE_SHARING} \ rm -f /etc/apt/apt.conf.d/docker-clean && \ echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-packages.conf && \ if echo "${DEBIAN_TAG}" | grep -q "[0-9]"; then \ cp /etc/apt/sources.list.d/debian.sources /etc/apt/sources.list.d/debian.sources~; \ sed -i -e 's|^URIs:|#|' -e 's|^# http://snapshot\.|URIs: http://snapshot.|' \ /etc/apt/sources.list.d/debian.sources; \ echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/use-snapshot.conf; \ echo 'Acquire::Retries "10";' >> /etc/apt/apt.conf.d/use-snapshot.conf; \ echo 'Acquire::Retries::Delay::Maximum "600";' >> /etc/apt/apt.conf.d/use-snapshot.conf; \ fi && \ apt-get update && \ apt-get install -y locales && \ localedef -i en_US -c -f UTF-8 en_US.UTF-8 && \ apt-get install --no-install-recommends -y \ python3-pip python3-setuptools python3-wheel python3-yaml python3-distro python3-jsonschema \ python3-newt python3-colorlog python3-kconfiglib python3-websockets \ gosu lsb-release file vim less procps tree tar bzip2 zstd pigz lz4 unzip tmux libncurses-dev \ git-lfs mercurial iproute2 ssh-client telnet curl rsync gnupg awscli sudo \ socat bash-completion python3-shtab python3-git python3-gnupg python3-cairo && \ rm -rf /var/log/* /tmp/* /var/tmp/* /var/cache/ldconfig/aux-cache && \ rm -f /etc/gitconfig && \ git config --system filter.lfs.clean 'git-lfs clean -- %f' && \ git config --system filter.lfs.smudge 'git-lfs smudge -- %f' && \ git config --system filter.lfs.process 'git-lfs filter-process' && \ git config --system filter.lfs.required true RUN --mount=type=bind,target=/kas,rw \ pip3 --proxy=$https_proxy install \ --no-deps \ --no-build-isolation \ --break-system-packages \ /kas && \ install -d /usr/local/share/bash-completion/completions/ && \ shtab --shell=bash -u kas.kas.kas_get_argparser --error-unimportable --prog kas \ > /usr/local/share/bash-completion/completions/kas && \ rm -rf $(pip3 cache dir) && \ rm -f /usr/local/bin/kas-container && \ install -m 0755 /kas/contrib/oe-git-proxy /usr/bin/ && \ install -m 0755 /kas/container-entrypoint / && \ kas --version ENV GIT_PROXY_COMMAND="oe-git-proxy" \ NO_PROXY="*" RUN echo "builder ALL=NOPASSWD: ALL" > /etc/sudoers.d/builder-nopasswd && \ chmod 660 /etc/sudoers.d/builder-nopasswd && \ echo "Defaults env_keep += \"ftp_proxy http_proxy https_proxy no_proxy\"" \ > /etc/sudoers.d/env_keep && chmod 660 /etc/sudoers.d/env_keep && \ groupadd builder -g 30000 && \ useradd builder -u 30000 -g 30000 --create-home --home-dir /builder ENTRYPOINT ["/container-entrypoint"] # # kas-isar image # FROM kas-base AS kas-isar ARG SOURCE_DATE_EPOCH ARG CACHE_SHARING=locked # The install package list are actually taking 1:1 from their documentation, # so there some packages that can already installed by other downstream layers. # This will not change any image sizes on all the layers in use. ENV LC_ALL=en_US.UTF-8 RUN --mount=type=cache,target=/var/cache/apt,sharing=${CACHE_SHARING} \ --mount=type=cache,target=/var/lib/apt,sharing=${CACHE_SHARING} \ apt-get update && \ apt-get install -y -f --no-install-recommends \ bzip2 mmdebstrap arch-test apt-utils dosfstools \ dpkg-dev gettext-base git mtools parted python3 \ quilt qemu-user-static reprepro sudo unzip git-buildpackage \ pristine-tar '(^sbuild-schroot$|^sbuild$)' schroot zstd \ umoci skopeo \ python3-botocore \ bubblewrap \ debootstrap && \ rm -f /etc/apt/apt.conf.d/use-snapshot.conf /etc/apt/apt.conf.d/keep-packages.conf && \ if [ -f "/etc/apt/sources.list.d/debian.sources~" ]; then \ mv -f /etc/apt/sources.list.d/debian.sources~ /etc/apt/sources.list.d/debian.sources; \ fi && \ rm -rf /var/log/* /tmp/* /var/tmp/* /var/cache/ldconfig/aux-cache && \ sbuild-adduser builder && \ sed -i 's|# kas-isar: ||g' /container-entrypoint USER builder # # kas image # FROM kas-base AS kas ARG SOURCE_DATE_EPOCH ARG CACHE_SHARING=locked # The install package list are actually taking 1:1 from their documentation # (exception: pylint3 -> pylint), so there some packages that can already # installed by other downstream layers. This will not change any image sizes # on all the layers in use. RUN --mount=type=cache,target=/var/cache/apt,sharing=${CACHE_SHARING} \ --mount=type=cache,target=/var/lib/apt,sharing=${CACHE_SHARING} \ apt-get update && \ apt-get install --no-install-recommends -y \ gawk wget git diffstat unzip texinfo \ gcc build-essential chrpath socat cpio python3 python3-pip python3-pexpect \ xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1 libsdl1.2-dev \ pylint xterm python3-subunit mesa-common-dev zstd lz4 && \ if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ apt-get install --no-install-recommends -y gcc-multilib g++-multilib; \ fi && \ rm -f /etc/apt/apt.conf.d/use-snapshot.conf /etc/apt/apt.conf.d/keep-packages.conf && \ if [ -f "/etc/apt/sources.list.d/debian.sources~" ]; then \ mv -f /etc/apt/sources.list.d/debian.sources~ /etc/apt/sources.list.d/debian.sources; \ fi && \ rm -rf /var/log/* /tmp/* /var/tmp/* /var/cache/ldconfig/aux-cache USER builder siemens-kas-41ad961/LICENSE000066400000000000000000000020611520561422700152740ustar00rootroot00000000000000MIT License Copyright (c) Siemens AG, 2017-2024 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. siemens-kas-41ad961/MANIFEST.in000066400000000000000000000002651520561422700160310ustar00rootroot00000000000000# The following folders are excluded to prevent setuptools' package # auto-discovery from adding the *.py files in these folders to the sdist # tarball exclude scripts exclude docs siemens-kas-41ad961/README.rst000066400000000000000000000031301520561422700157540ustar00rootroot00000000000000Setup tool for bitbake based projects ===================================== +--------------------+ | Build Status | +====================+ | |workflow-master|_ | +--------------------+ | |workflow-next|_ | +--------------------+ .. |workflow-master| image:: https://github.com/siemens/kas/workflows/master/badge.svg .. _workflow-master: https://github.com/siemens/kas/actions?query=workflow%3Amaster .. |workflow-next| image:: https://github.com/siemens/kas/workflows/next/badge.svg .. _workflow-next: https://github.com/siemens/kas/actions?query=workflow%3Anext This tool provides an easy mechanism to setup bitbake based projects. The OpenEmbedded tooling support starts at step 2 with bitbake. The downloading of sources and then configuration has to be done by hand. Usually, this is explained in a README. Instead kas is using a project configuration file and does the download and configuration phase. Key features provided by the build tool: - clone and checkout bitbake layers - create default bitbake settings (machine, arch, ...) - launch minimal build environment, reducing risk of host contamination - initiate bitbake build process See the `kas documentation `_ for further details. SECURITY NOTICE --------------- At this stage, kas does not validate the integrity of fetched repositories. Make sure to only pull from trusted sources to ensure that the selected revisions are the expected ones, specifically when using mirrors. Later versions of kas may introduce integrity validation mechanisms such as cryptographic checksums to strengthen supply chain security. siemens-kas-41ad961/SECURITY.md000066400000000000000000000031441520561422700160630ustar00rootroot00000000000000# Security Policy The kas community takes the security of its code seriously. If you think you have found a security vulnerability, please read the next sections and follow the instructions to report your finding. ## Security Context Open source software can be used in various contexts that may go far beyond what it was originally designed and also secured for. Therefore, we describe here how kas is currently expected to be used in security-sensitive scenarios. In a nutshell, the purpose of kas is fetching known and previously validated content, identifying it as original, and then configuring and building artifacts. Therefore, anything that may prevent checking the integrity of fetched content prior to executing instructions it carries is security-wise in scope for kas. This affects both the kas tool itself and the containers provided by kas because they also contain tools that kas or bitbake use for fetching and validating. ## Reporting a Vulnerability Please DO NOT report any potential security vulnerability via a public channel (mailing list, github issue etc.). Instead, create a report via https://github.com/siemens/kas/security/advisories/new or contact the maintainer jan.kiszka@siemens.com via email directly. Please provide a detailed description of the issue, the steps to reproduce it, the affected versions and, if already available, a proposal for a fix. You should receive a response within 5 working days. If the issue is confirmed as a vulnerability by us, we will open a Security Advisory on github and give credits for your report if desired. This project follows a 90 day disclosure timeline. siemens-kas-41ad961/container-entrypoint000077500000000000000000000116731520561422700204210ustar00rootroot00000000000000#!/bin/sh # # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2023 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. enable_qemu_binfmts() { if ! [ -f /proc/sys/fs/binfmt_misc/status ]; then sudo mount -t binfmt_misc null /proc/sys/fs/binfmt_misc fi for CONF in /usr/lib/binfmt.d/qemu-*.conf; do if ! [ -f /proc/sys/fs/binfmt_misc/"$(basename "${CONF%%.conf}")" ]; then sudo sh -c "cat '$CONF' > /proc/sys/fs/binfmt_misc/register" fi done } # kas-isar: enable_qemu_binfmts chown_managed_dirs() { for DIR in /build /work /sstate /downloads /repo-ref; do if [ -d "$DIR" ]; then chown "$1":"$2" "$DIR" fi done # SSH Directory Permissions if [ -d "/builder/.ssh" ]; then chown -R "$1":"$2" "/builder/.ssh" fi } restore_managed_dirs_owner() { chown_managed_dirs 0 0 } if mount | grep -q "on / type aufs"; then cat <&2 WARNING: Generation of wic images will fail! Your docker host setup uses broken aufs as storage driver. Adjust the docker configuration to use a different driver (overlay, overlay2, devicemapper). You may also need to update the host distribution (e.g. Debian Jessie -> Stretch). EOF fi # if the container is started from a not supported terminal, fallback to xterm if [ -n "$TERM" ]; then infocmp "$TERM" > /dev/null 2>&1 || TERM=xterm fi if [ -z "$USER_ID" ] && [ -n "$CI_PROJECT_DIR" ]; then # Work around for gitlab-runner not aligning checked out repo ownership # with our builder user. We handle that internally in kas, but we # need the exception here as well for git calls outside of kas. sudo git config --system safe.directory "$CI_PROJECT_DIR" # Account for externally specified git clone path if [ -n "$GIT_CLONE_PATH" ]; then sudo git config --system --add safe.directory "$GIT_CLONE_PATH" fi fi if [ -z "$USER_ID" ] || [ "$USER_ID" = 0 ]; then # Not a kas-container call, or we shall run everything as root GOSU="" else GROUP_ID=${GROUP_ID:-$(id -g)} groupmod -o --gid "$GROUP_ID" builder usermod -o --uid "$USER_ID" --gid "$GROUP_ID" builder >/dev/null chown -R "$USER_ID":"$GROUP_ID" /builder # copy host SSH config into home of builder if [ -d /var/kas/userdata/.ssh ]; then cp -a /var/kas/userdata/.ssh /builder/ fi # adjust timezone to host if [ -n "${KAS_HOST_TZ}" ] && [ -f "/usr/share/zoneinfo/${KAS_HOST_TZ}" ]; then echo "${KAS_HOST_TZ}" > /etc/timezone ln -sf "/usr/share/zoneinfo/${KAS_HOST_TZ}" /etc/localtime fi GOSU="gosu builder" fi # kas-container on rootless docker workaround if [ -n "$USER_ID" ] && [ "$USER_ID" -ne 0 ] && \ [ "$KAS_DOCKER_ROOTLESS" = "1" ] && [ "$(stat -c %u /repo)" -eq 0 ]; then # Docker rootless does not support keeping the user namespace # (podman option --userns=keep-id). By that, the bind mounts # are owned by root. git config --system safe.directory /repo chown_managed_dirs "$USER_ID" "$GROUP_ID" # Copy userdata to a writable location so we can chown it. if [ -d "/var/kas/userdata" ]; then mv /var/kas/userdata /var/kas/userdata.orig cp -r /var/kas/userdata.orig /var/kas/userdata chown -R "$USER_ID":"$GROUP_ID" /var/kas/userdata fi fi if [ "$PWD" = / ]; then cd /builder || exit 1 fi if [ -n "$1" ]; then case "$1" in build|checkout|clean*|diff|dump|for-all-repos|lock|menu|purge|shell|-*) # We must restore the dir owner after every kas invocation. # This is cheap as only the top-level dirs are changed (non recursive). if [ "$KAS_DOCKER_ROOTLESS" = "1" ]; then trap restore_managed_dirs_owner EXIT INT TERM $GOSU kas "$@" else # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 exec $GOSU kas "$@" fi ;; *) if [ -n "$USER_ID" ]; then echo "kas-container: this container does not support \"kas $1\"" >&2 exit 1 fi # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 exec $GOSU "$@" ;; esac else # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 exec $GOSU bash fi siemens-kas-41ad961/contrib/000077500000000000000000000000001520561422700157305ustar00rootroot00000000000000siemens-kas-41ad961/contrib/oe-git-proxy000077500000000000000000000110651520561422700202240ustar00rootroot00000000000000#!/bin/bash # oe-git-proxy is a simple tool to be via GIT_PROXY_COMMAND. It uses socat # to make SOCKS5 or HTTPS proxy connections. # It uses ALL_PROXY or all_proxy or http_proxy to determine the proxy server, # protocol, and port. # It uses NO_PROXY to skip using the proxy for a comma delimited list of # hosts, host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It # is known to work with both bash and dash shells. # # Example ALL_PROXY values: # ALL_PROXY=socks://socks.example.com:1080 # ALL_PROXY=https://proxy.example.com:8080 # # Copyright (c) 2013, Intel Corporation. # # SPDX-License-Identifier: GPL-2.0-only # # AUTHORS # Darren Hart # disable pathname expansion, NO_PROXY fields could start with "*" or be it set -f if [ $# -lt 2 -o "$1" = '--help' -o "$1" = '-h' ] ; then echo 'oe-git-proxy: error: the following arguments are required: host port' echo 'Usage: oe-git-proxy host port' echo '' echo 'OpenEmbedded git-proxy - a simple tool to be used via GIT_PROXY_COMMAND.' echo 'It uses socat to make SOCKS or HTTPS proxy connections.' echo 'It uses ALL_PROXY to determine the proxy server, protocol, and port.' echo 'It uses NO_PROXY to skip using the proxy for a comma delimited list' echo 'of hosts, host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24).' echo 'It is known to work with both bash and dash shells.runs native tools' echo '' echo 'arguments:' echo ' host proxy host to use' echo ' port proxy port to use' echo '' echo 'options:' echo ' -h, --help show this help message and exit' echo '' exit 2 fi # Locate the netcat binary if [ -z "$SOCAT" ]; then SOCAT=$(which socat 2>/dev/null) if [ $? -ne 0 ]; then echo "ERROR: socat binary not in PATH" 1>&2 exit 1 fi fi METHOD="" # Test for a valid IPV4 quad with optional bitmask valid_ipv4() { echo $1 | egrep -q "^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}(/(3[0-2]|[1-2]?[0-9]))?$" return $? } # Convert an IPV4 address into a 32bit integer ipv4_val() { IP="$1" SHIFT=24 VAL=0 for B in $( echo "$IP" | tr '.' ' ' ); do VAL=$(($VAL+$(($B<<$SHIFT)))) SHIFT=$(($SHIFT-8)) done echo "$VAL" } # Determine if two IPs are equivalent, or if the CIDR contains the IP match_ipv4() { CIDR=$1 IP=$2 if [ -z "${IP%%$CIDR}" ]; then return 0 fi # Determine the mask bitlength BITS=${CIDR##*/} [ "$BITS" != "$CIDR" ] || BITS=32 if [ -z "$BITS" ]; then return 1 fi IPVAL=$(ipv4_val $IP) IP2VAL=$(ipv4_val ${CIDR%%/*}) # OR in the unmasked bits for i in $(seq 0 $((32-$BITS))); do IP2VAL=$(($IP2VAL|$((1<<$i)))) IPVAL=$(($IPVAL|$((1<<$i)))) done if [ $IPVAL -eq $IP2VAL ]; then return 0 fi return 1 } # Test to see if GLOB matches HOST match_host() { HOST=$1 GLOB=$2 if [ -z "${HOST%%*$GLOB}" ]; then return 0 fi # Match by netmask if valid_ipv4 $GLOB; then for HOST_IP in $(getent ahostsv4 $HOST | grep ' STREAM ' | cut -d ' ' -f 1) ; do if valid_ipv4 $HOST_IP; then match_ipv4 $GLOB $HOST_IP if [ $? -eq 0 ]; then return 0 fi fi done fi return 1 } # If no proxy is set or needed, just connect directly METHOD="TCP:$1:$2" [ -z "${ALL_PROXY}" ] && ALL_PROXY=$all_proxy [ -z "${ALL_PROXY}" ] && ALL_PROXY=$http_proxy if [ -z "$ALL_PROXY" ]; then exec $SOCAT STDIO $METHOD fi # Connect directly to hosts in NO_PROXY for H in $( echo "$NO_PROXY" | tr ',' ' ' ); do if match_host $1 $H; then exec $SOCAT STDIO $METHOD fi done # Proxy is necessary, determine protocol, server, and port # extract protocol PROTO=${ALL_PROXY%://*} # strip protocol:// from string ALL_PROXY=${ALL_PROXY#*://} # extract host & port parts: # 1) drop username/password PROXY=${ALL_PROXY##*@} # 2) remove optional trailing /? PROXY=${PROXY%%/*} # 3) extract optional port PORT=${PROXY##*:} if [ "$PORT" = "$PROXY" ]; then PORT="" fi # 4) remove port PROXY=${PROXY%%:*} # extract username & password PROXYAUTH="${ALL_PROXY%@*}" [ "$PROXYAUTH" = "$ALL_PROXY" ] && PROXYAUTH= [ -n "${PROXYAUTH}" ] && PROXYAUTH=",proxyauth=${PROXYAUTH}" if [ "$PROTO" = "socks" ] || [ "$PROTO" = "socks4a" ]; then if [ -z "$PORT" ]; then PORT="1080" fi METHOD="SOCKS4A:$PROXY:$1:$2,socksport=$PORT" elif [ "$PROTO" = "socks4" ]; then if [ -z "$PORT" ]; then PORT="1080" fi METHOD="SOCKS4:$PROXY:$1:$2,socksport=$PORT" else # Assume PROXY (http, https, etc) if [ -z "$PORT" ]; then PORT="8080" fi METHOD="PROXY:$PROXY:$1:$2,proxyport=${PORT}${PROXYAUTH}" fi exec $SOCAT STDIO "$METHOD" siemens-kas-41ad961/docs/000077500000000000000000000000001520561422700152205ustar00rootroot00000000000000siemens-kas-41ad961/docs/Makefile000066400000000000000000000176421520561422700166720ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " epub3 to make an epub3" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" @echo " dummy to check syntax errors of document sources" .PHONY: clean clean: rm -rf $(BUILDDIR)/* .PHONY: html html: kas-container-usage $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." .PHONY: dirhtml dirhtml: kas-container-usage $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." .PHONY: singlehtml singlehtml: kas-container-usage $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." .PHONY: pickle pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." .PHONY: json json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." .PHONY: htmlhelp htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." .PHONY: qthelp qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/kas.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/kas.qhc" .PHONY: applehelp applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." .PHONY: devhelp devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/kas" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/kas" @echo "# devhelp" .PHONY: epub epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." .PHONY: epub3 epub3: $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 @echo @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." .PHONY: latex latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." .PHONY: latexpdf latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." .PHONY: latexpdfja latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." .PHONY: text text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." kas-container-usage: @echo Generate kas-container usage documentation @mkdir -p $(BUILDDIR) @../kas-container --help | ../scripts/kas-container-usage-to-rst.sh | awk '/KAS-COMMANDS/ {exit} {print}' \ > $(BUILDDIR)/kas-container-usage-synopsis.inc @../kas-container --help | ../scripts/kas-container-usage-to-rst.sh | awk '/KAS-COMMANDS/ {found=1} found' \ > $(BUILDDIR)/kas-container-usage-options.inc .PHONY: man man: kas-container-usage $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) -t man $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." .PHONY: texinfo texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." .PHONY: info info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." .PHONY: gettext gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." .PHONY: changes changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." .PHONY: linkcheck linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." .PHONY: doctest doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." .PHONY: coverage coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." .PHONY: xml xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." .PHONY: pseudoxml pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." .PHONY: dummy dummy: $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy @echo @echo "Build finished. Dummy builder generates no files." siemens-kas-41ad961/docs/_ext/000077500000000000000000000000001520561422700161575ustar00rootroot00000000000000siemens-kas-41ad961/docs/_ext/sphinx_kas_schema.py000066400000000000000000000114451520561422700222250ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the 'Software'), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ''' This module contains the :kasschemadesc:`` role to extract the description of a node from the schema. The `` hereby is a path to the node in the schema, separated by dots. For example, the header.version node can be accessed with :kasschemadesc:`header.properties.version`. Definitions inside the `$defs` section are accessed by their full path, e.g. :kasschemadesc:`$defs.path.to.node`. ''' __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2024' from sphinx.application import Sphinx from sphinx.util.docutils import SphinxRole from docutils.parsers.rst.states import Struct from sphinx.util.nodes import nodes from typing import Dict, Union import re from kas.configschema import CONFIGSCHEMA, __schema_definition__ class KasSchemaDescRole(SphinxRole): required_arguments = 1 def __init__(self): super().__init__() self.key_regex = re.compile(r'([a-zA-Z0-9_]+)(?:\[(\d+)\])?') def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: messages = [] if self.text.startswith('$defs.'): node = CONFIGSCHEMA['$defs'] path = self.text.split('.')[1:] else: node = CONFIGSCHEMA['properties'] path = self.text.split('.') self.env.note_dependency(__schema_definition__) try: for part in path: match = self.key_regex.match(part).groups() if match[1]: node = node[match[0]][int(match[1])] elif match[0]: node = node[match[0]] else: raise KeyError except KeyError: messages.append(self.inliner.document.reporter.error( f'Invalid path: {self.text}', line=self.lineno, )) return [], messages try: desc = node['description'] except KeyError: messages.append(self.inliner.document.reporter.error( f'Description missing for path: {self.text}', line=self.lineno, )) return [], messages default = node.get('default', None) allowed_values = node.get('enum', None) minval = node.get('minimum', None) maxval = node.get('maximum', None) memo = Struct(document=self.inliner.document, reporter=self.inliner.reporter, language=self.inliner.language) parent = nodes.paragraph() processed, msgs = self.inliner.parse(desc, self.lineno, memo, parent) parent += processed messages += msgs if default is not None: def_pg = nodes.paragraph() def_pg += nodes.strong(text='Default: ') def_pg += nodes.literal(text=str(default)) parent += def_pg if allowed_values is not None: av_pg = nodes.paragraph() av_pg += nodes.strong(text='Supported values: ') for i in range(len(allowed_values)): if i != 0: av_pg += nodes.Text(', ') av_pg += nodes.literal(text=str(allowed_values[i])) parent += av_pg if minval is not None and maxval is not None: range_pg = nodes.paragraph() range_pg += nodes.strong(text='Range: ') range_pg += nodes.literal(text=f'[{minval}, {maxval}]') parent += range_pg return parent, messages def setup(app: Sphinx) -> Dict[str, Union[bool, str]]: app.add_role('kasschemadesc', KasSchemaDescRole()) return { 'parallel_read_safe': True, 'parallel_write_safe': True, 'version': '1.0', } siemens-kas-41ad961/docs/_man/000077500000000000000000000000001520561422700161325ustar00rootroot00000000000000siemens-kas-41ad961/docs/_man/_kas-man-footer.inc000066400000000000000000000000561520561422700216100ustar00rootroot00000000000000KAS --- Part of the :manpage:`kas(1)` suite. siemens-kas-41ad961/docs/_man/kas-build-attestation.rst000066400000000000000000000002121520561422700230670ustar00rootroot00000000000000:orphan: kas build attestation ===================== .. toctree:: ../userguide/build-attestation .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-container.rst000066400000000000000000000006661520561422700214320ustar00rootroot00000000000000:orphan: kas-container manpage ===================== .. include:: ../_build/kas-container-usage-synopsis.inc DESCRIPTION ----------- .. include:: ../userguide/kas-container-description.inc .. include:: ../_build/kas-container-usage-options.inc SEE ALSO -------- :manpage:`kas(1)`, .. include:: _kas-man-footer.inc .. |SYNOPSIS| replace:: SYNOPSIS .. |OPTIONS| replace:: OPTIONS .. |KAS-COMMANDS| replace:: KAS-CONTAINER COMMANDS siemens-kas-41ad961/docs/_man/kas-credentials.rst000066400000000000000000000002071520561422700217340ustar00rootroot00000000000000:orphan: kas credential handling ======================= .. toctree:: ../userguide/credentials .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-format-changelog.rst000066400000000000000000000005461520561422700226620ustar00rootroot00000000000000:orphan: Configuration Format Changes ============================ DESCRIPTION ----------- The kas project configuration is versioned to ensure that the requested features are supported by the current kas version. .. toctree:: :maxdepth: 2 ../format-changelog SEE ALSO -------- :manpage:`kas-project-config(1)` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-build.rst000066400000000000000000000004571520561422700220410ustar00rootroot00000000000000:orphan: kas build plugin ================ .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: build :manpage: .. automodule:: kas.plugins.build :noindex: SEE ALSO ~~~~~~~~ :manpage:`kas-build-attestation(1)`, .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-checkout.rst000066400000000000000000000004021520561422700225350ustar00rootroot00000000000000:orphan: kas checkout plugin =================== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: checkout :manpage: .. automodule:: kas.plugins.checkout :noindex: .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-clean.rst000066400000000000000000000005611520561422700220200ustar00rootroot00000000000000:orphan: kas clean command ================= .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: clean :manpage: .. automodule:: kas.plugins.clean.Clean :noindex: SEE ALSO -------- :manpage:`kas-plugin-cleanall`, :manpage:`kas-plugin-cleansstate`, :manpage:`kas-plugin-purge` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-cleanall.rst000066400000000000000000000005721520561422700225130ustar00rootroot00000000000000:orphan: kas cleanall command ==================== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: cleanall :manpage: .. automodule:: kas.plugins.clean.CleanAll :noindex: SEE ALSO -------- :manpage:`kas-plugin-clean`, :manpage:`kas-plugin-cleansstate`, :manpage:`kas-plugin-purge` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-cleansstate.rst000066400000000000000000000006031520561422700232410ustar00rootroot00000000000000:orphan: kas cleansstate command ======================= .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: cleansstate :manpage: .. automodule:: kas.plugins.clean.CleanSstate :noindex: SEE ALSO -------- :manpage:`kas-plugin-clean`, :manpage:`kas-plugin-cleanall`, :manpage:`kas-plugin-purge` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-diff.rst000066400000000000000000000005641520561422700216510ustar00rootroot00000000000000:orphan: kas diff plugin =============== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: diff :manpage: .. automodule:: kas.plugins.diff :noindex: SEE ALSO -------- :manpage:`kas-project-config(1)`, :manpage:`kas-checkout(1)`, :manpage:`kas-lock(1)`, :manpage:`kas-build(1)` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-dump.rst000066400000000000000000000005641520561422700217060ustar00rootroot00000000000000:orphan: kas dump plugin =============== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: dump :manpage: .. automodule:: kas.plugins.dump :noindex: SEE ALSO -------- :manpage:`kas-project-config(1)`, :manpage:`kas-checkout(1)`, :manpage:`kas-lock(1)`, :manpage:`kas-build(1)` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-for-all-repos.rst000066400000000000000000000004261520561422700234200ustar00rootroot00000000000000:orphan: kas for-all-repos plugin ======================== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: for-all-repos :manpage: .. automodule:: kas.plugins.for_all_repos :noindex: .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-lock.rst000066400000000000000000000005301520561422700216620ustar00rootroot00000000000000:orphan: kas lock plugin =============== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: lock :manpage: .. automodule:: kas.plugins.lock :noindex: SEE ALSO -------- :manpage:`kas-project-config(1)`, :manpage:`kas-dump(1)`, :manpage:`kas-build(1)` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-menu.rst000066400000000000000000000003621520561422700217010ustar00rootroot00000000000000:orphan: kas menu plugin =============== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: menu :manpage: .. automodule:: kas.plugins.menu :noindex: .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-purge.rst000066400000000000000000000005611520561422700220600ustar00rootroot00000000000000:orphan: kas purge command ================= .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: purge :manpage: .. automodule:: kas.plugins.clean.Purge :noindex: SEE ALSO -------- :manpage:`kas-plugin-clean`, :manpage:`kas-plugin-cleanall`, :manpage:`kas-plugin-cleansstate` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-plugin-shell.rst000066400000000000000000000003661520561422700220500ustar00rootroot00000000000000:orphan: kas shell plugin ================ .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: shell :manpage: .. automodule:: kas.plugins.shell :noindex: .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas-project-config.rst000066400000000000000000000003201520561422700223440ustar00rootroot00000000000000:orphan: kas project configuration ========================= .. toctree:: ../userguide/project-configuration SEE ALSO -------- :manpage:`kas-format-changelog(1)` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_man/kas.rst000066400000000000000000000016511520561422700174450ustar00rootroot00000000000000:orphan: kas manpage =========== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :manpage: .. include:: ../intro.rst :start-line: 3 PROJECT CONFIGURATION --------------------- The project configuration file describes the build environment and the layers to be used. It is the main input to kas. For details, see :manpage:`kas-project-config(1)` BUILD ATTESTATION ----------------- Kas supports to generate build attestation. For details, see :manpage:`kas-build-attestation(1)`. CREDENTIAL HANDLING ------------------- kas provides various mechanisms to inject credentials into the build. For details, see :manpage:`kas-credentials(1)`. ENVIRONMENT VARIABLES --------------------- .. include:: ../command-line/environment-variables.inc SEE ALSO -------- :manpage:`kas-project-config(1)`, :manpage:`kas-build(1)`, :manpage:`kas-credentials(1)` .. include:: _kas-man-footer.inc siemens-kas-41ad961/docs/_static/000077500000000000000000000000001520561422700166465ustar00rootroot00000000000000siemens-kas-41ad961/docs/_static/theme_overrides.css000066400000000000000000000005551520561422700225510ustar00rootroot00000000000000/* override table width restrictions */ @media screen and (min-width: 767px) { .wy-table-responsive table td { /* !important prevents the common CSS stylesheets from overriding this as on RTD they are loaded after this stylesheet */ white-space: normal !important; } .wy-table-responsive { overflow: visible !important; } } siemens-kas-41ad961/docs/command-line.rst000066400000000000000000000004011520561422700203100ustar00rootroot00000000000000Command Line Usage ================== .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :nosubcommands: .. _env-vars-label: Environment Variables --------------------- .. include:: command-line/environment-variables.inc siemens-kas-41ad961/docs/command-line/000077500000000000000000000000001520561422700175635ustar00rootroot00000000000000siemens-kas-41ad961/docs/command-line/environment-variables.inc000066400000000000000000000405351520561422700245770ustar00rootroot00000000000000kas uses a number of environment variables to configure its behavior. The `Variables Glossary`_ provides an overview, wherein the tuple (C,K,E) denotes the scope of the variable. By default, kas does not inherit the full host shell environment into the build environment. Instead, kas constructs a controlled environment to improve reproducibility and reduce host contamination. How additional environment variables are handled depends on their scope and on the selected sub-command. Where supported, variables can be defined in the kas configuration or preserved explicitly from the host environment, for example with ``-E/--preserve-env``. When using ``kas-container``, this is more limited, especially for variables that contain file or directory paths, because the container may use a different directory layout. All directories that are passed to kas by setting the corresponding environment variables (e.g. ``KAS_WORK_DIR``, ``KAS_BUILD_DIR``, ...) must not overlap with each other, except for overlapping with ``KAS_WORK_DIR`` (i.e. the build|sstate|downloads|repo-ref dirs can be below the work dir). Environment variables that reference a file or directory must have a valid path that is accessible and readable. Variable Scope ~~~~~~~~~~~~~~ **kas-container (C)** The variable is processed or forwarded by the ``kas-container`` script. For some variables, the variable is re-written to the container's directory layout. .. note:: The ``env`` section of the `project configuration` can be used to make arbitrary environment variables available to the build environment. When invoking the build via ``kas-container``, make sure to also forward the corresponding environment variables into the container. **kas (K)** The variable is processed by kas itself. Some variables (e.g. the credentials for the awscli) are re-written to configuration files to also support older versions of the tooling. **build environment (E)** The variable is exported into the build environment. In this environment, the ``bitbake`` command is executed. **config-file (c)** The variable can be set in the ``env`` section of the :ref:`project-configuration-label`. Note, that a value provided by the calling environment takes precedence over the value in the configuration file. Variables Glossary ~~~~~~~~~~~~~~~~~~ These environment variables are processed before the configuration file is read (except if stated otherwise). By that, they cannot be defined or overwritten using the ``env`` section of the config file. +--------------------------+--------------------------------------------------+ | Environment variables | Description | +==========================+==================================================+ | ``KAS_WORK_DIR`` | The path of the kas work directory, current work | | (C, K) | directory is the default. This directory must | | | exist if set. | +--------------------------+--------------------------------------------------+ | ``KAS_BUILD_DIR`` | The path of the build directory, | | (C, K) | ``${KAS_WORK_DIR}/build`` is the default. | | | The parent directory must exist if set. | | | kas adds a ``CACHEDIR.TAG``. | +--------------------------+--------------------------------------------------+ | ``KAS_REPO_REF_DIR`` | The path to the repository reference directory. | | (C, K) | Repositories in this directory are used as | | | references when cloning. In order for kas to | | | find those repositories, they have to be named | | | in a specific way. The repo URLs are translated | | | like this: | | | ``https://github.com/siemens/meta-iot2000.git`` | | | resolves to the name | | | ``github.com.siemens.meta-iot2000.git``. | | | Repositories that are not found will be cloned | | | below this directory. Multiple instances of kas | | | can simultaneously work on the same directory, | | | as long as the underlying filesystem is POSIX | | | compatible. This directory must exist if set. | +--------------------------+--------------------------------------------------+ | ``KAS_DISTRO`` | This overwrites the respective setting in the | | ``KAS_MACHINE`` | configuration file. | | ``KAS_TARGET`` | | | ``KAS_TASK`` | | | (C, K) | | +--------------------------+--------------------------------------------------+ | ``KAS_PREMIRRORS`` | Specifies alternatives for repo URLs. Just like | | (C, K) | bitbake ``PREMIRRORS``, this variable consists | | | of new-line separated entries. Each entry | | | defines a regular expression to match a URL and, | | | space-separated, its replacement. E.g.: | | | ``http://.*\.someurl\.io/ | | | http://localmirror.net/`` | +--------------------------+--------------------------------------------------+ | ``DISTRO_APT_PREMIRRORS``| Specifies alternatives for apt URLs. Just like | | (C,c) | ``KAS_PREMIRRORS``. | +--------------------------+--------------------------------------------------+ | ``KAS_CLONE_DEPTH`` | Perform shallow git clone/fetch using --depth=N | | (C, K) | specified by this variable. This is useful in | | | case CI always starts with empty work directory | | | and this directory is always discarded after the | | | CI run. | +--------------------------+--------------------------------------------------+ | ``SSH_PRIVATE_KEY`` | Variable containing the private key that should | | (K) | be added to an internal ssh-agent. This key | | | cannot be password protected. This setting is | | | useful for CI build servers. On desktop | | | machines, an ssh-agent running outside the kas | | | environment is more useful. | +--------------------------+--------------------------------------------------+ | ``SSH_PRIVATE_KEY_FILE`` | Path to the private key file that should be | | (K) | added to an internal ssh-agent. This key cannot | | | be password protected. This setting is useful | | | for CI build servers. On desktop machines, an | | | ssh-agent running outside the kas environment is | | | more useful. | +--------------------------+--------------------------------------------------+ | ``SSH_AUTH_SOCK`` | SSH authentication socket. Used for cloning over | | (C,K,E) | SSH (alternative to ``SSH_PRIVATE_KEY`` or | | | ``SSH_PRIVATE_KEY_FILE``). | +--------------------------+--------------------------------------------------+ | ``DL_DIR`` | Environment variables that are transferred to | | ``SSTATE_DIR`` | the bitbake environment. The ``DL_DIR`` and | | ``SSTATE_MIRRORS`` | ``SSTATE_DIR`` directories are created along | | (C,K,E,c) | with their parents, if set. | +--------------------------+--------------------------------------------------+ | ``TMPDIR`` (K,E,c) | Directory for temporary files. | +--------------------------+--------------------------------------------------+ | ``http_proxy`` | These variables define the proxy configuration | | ``https_proxy`` | bitbake should use. | | ``ftp_proxy`` | | | ``no_proxy`` | | | (C,K,E) | | +--------------------------+--------------------------------------------------+ | ``GIT_PROXY_COMMAND`` (E)| Set proxy for native git fetches. ``NO_PROXY`` | | ``NO_PROXY`` (C,K,E) | is evaluated by OpenEmbedded's oe-git-proxy | | | script. | +--------------------------+--------------------------------------------------+ | ``SHELL`` | The shell to start when using the ``shell`` | | (C,K,E) | plugin and in the bitbake environment. | +--------------------------+--------------------------------------------------+ | ``TERM`` | The terminal options used in the ``shell`` | | (C,K,E) | plugin and in the bitbake environment. | +--------------------------+--------------------------------------------------+ | ``TZ`` (C) | Timezone settings. | +--------------------------+--------------------------------------------------+ | ``AWS_CONFIG_FILE`` | Path to the awscli configuration and credentials | | |aws_cred| | files that are copied to the kas home dir. | | (K,C) | | +--------------------------+--------------------------------------------------+ | |git_cred| | Allows one to set and configure the git | | (K,C) | credential helper in the `.gitconfig` of the kas | | | user. | +--------------------------+--------------------------------------------------+ | ``GITCONFIG_FILE`` | Path to a `.gitconfig` file which will be | | (K,C) | copied to the kas home dir as `.gitconfig`. | +--------------------------+--------------------------------------------------+ | ``NETRC_FILE`` | Path to a .netrc file which will be copied to | | (K,C) | the kas home dir as .netrc. | +--------------------------+--------------------------------------------------+ | ``NPMRC_FILE`` | Path to a .npmrc file which will be copied to | | (K,C) | the kas home dir as .npmrc. | +--------------------------+--------------------------------------------------+ | ``REGISTRY_AUTH_FILE`` | Path to a container registry authentication file.| | (K,C) | | +--------------------------+--------------------------------------------------+ | |ci_server_vars| | Environment variables from GitLab CI, if set | | ``CI_JOB_TOKEN`` | .netrc is configured to allow fetching from | | ``CI_JOB_URL`` | the GitLab instance. An entry will be appended | | ``CI_REGISTRY`` | in case ``NETRC_FILE`` was given as well. Note | | ``CI_REGISTRY_USER`` | that if the file already contains an entry for | | (K) | that host most tools would probably take that | | | first one. The job URL is added to the | | | provenance attestation (if enabled). | | | If ``CI_REGISTRY`` and ``CI_REGISTRY_USER`` is | | | also set, a container registry login file is | | | created, which is used by docker, podman and | | | skopeo. In case ``REGISTRY_AUTH_FILE`` was given | | | as well, the CI login data will be appended to | | | that file. | | | The required base64 encoded login data is | | | generated by kas. | +--------------------------+--------------------------------------------------+ | ``GITHUB_ACTIONS`` | Environment variables from GitHub actions or | | ``GITLAB_CI`` | GitLab CI. If set to `true`, `.gitconfig` is | | (K) | automatically imported. | | | For details, see ``GITCONFIG_FILE``. | +--------------------------+--------------------------------------------------+ | ``REMOTE_CONTAINERS`` (K)| Environment variables related to VSCode Remote | | ``REMOTE_CONTAINERS_``| Containers. If running in this environment, | | (K,E) | `.gitconfig` is automatically imported. | +--------------------------+--------------------------------------------------+ | ``BB_NUMBER_THREADS`` | Environment variables to control the concurrency.| | ``PARALLEL_MAKE`` | | | (C,K,E) | | +--------------------------+--------------------------------------------------+ | ``KAS_IMAGE_VERSION`` (C)| Select the version of the (official) kas | | | container (e.g. 4.5). | +--------------------------+--------------------------------------------------+ | |container-distro| (C) | Select the base distro and its release of the | | | container image (e.g. ``debian-bookworm``). | | | If not specified, the default (most-recent | | | supported) distro version is used. | +--------------------------+--------------------------------------------------+ | ``KAS_CONTAINER_IMAGE`` | Select the container image (full OCI path | | (C) | including tag). | +--------------------------+--------------------------------------------------+ | ``KAS_CONTAINER_ENGINE`` | Explicitly set the container engine (either | | (C) | ``docker`` or ``podman``). If not set, this is | | | auto-detected (preference: docker). | +--------------------------+--------------------------------------------------+ | ``KAS_SUDO_CMD`` | Explicitly set the sudo command (either ``sudo`` | | (C) | or ``run0``) for operations that require higher | | | privileges. If not set, this is auto-detected | | | (preference: ``sudo``). Note, that ``run0`` does | | | not preserve the environment and cannot setup | | | loopback devices. | +--------------------------+--------------------------------------------------+ | ``KAS_BUILDTOOLS_DIR`` | Explicitly set the path where kas will download | | (C,K) | and install buildtools. If not set, kas will use | | | ``KAS_BUILD_DIR/buildtools`` as the default path.| +--------------------------+--------------------------------------------------+ .. |aws_cred| replace:: ``AWS_ROLE_ARN`` ``AWS_SHARED_CREDENTIALS_FILE`` ``AWS_WEB_IDENTITY_TOKEN_FILE`` .. |git_cred| replace:: ``GIT_CREDENTIAL_HELPER`` ``GIT_CREDENTIAL_USEHTTPPATH`` .. |ci_server_vars| replace:: ``CI_SERVER_HOST`` ``CI_SERVER_PORT`` ``CI_SERVER_PROTOCOL`` ``CI_SERVER_SHELL_SSH_HOST`` ``CI_SERVER_SHELL_SSH_PORT`` .. |container-distro| replace:: ``KAS_CONTAINER_IMAGE_DISTRO`` .. only:: html For details about the access of remote resources, see :ref:`checkout-creds-label`. siemens-kas-41ad961/docs/conf.py000066400000000000000000000274361520561422700165330ustar00rootroot00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # kas documentation build configuration file, created by # sphinx-quickstart on Thu Jun 22 11:10:06 2017. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys # Prepend the project directory to the path, in order to include it: sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.insert(0, os.path.abspath( os.path.join(os.path.dirname(__file__), "_ext"))) import kas # noqa (disables pycodestyle check for this line) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinxarg.ext', 'sphinx_rtd_theme', 'sphinx_kas_schema' ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = {'.rst': 'restructuredtext'} # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = 'kas' copyright = 'Siemens and contributors, 2017-2025' author = 'Daniel Wagner, Jan Kiszka, Claudius Heine' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = kas.__version__ # The full version, including alpha/beta/rc tags. release = kas.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # # today = '' # # Else, today_fmt is used as the format for a strftime call. # # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '_man'] if tags.has('man'): # noqa: F821 exclude_patterns.remove('_man') # The reST default role (used for this markup: `text`) to use for all # documents. # # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. # # html_title = 'kas v0.9.0' # A shorter title for the navigation bar. Default is the same as html_title. # # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # # html_logo = None # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or # 32x32 pixels large. # # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. # # html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. # # html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # # html_additional_pages = {} # If false, no module index is generated. # # html_domain_indices = True # If false, no index is generated. # # html_use_index = True # If true, the index is split into individual pages for each letter. # # html_split_index = False # If true, links to the reST sources are added to the pages. # # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' # # html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. # # html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. # # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'kasdoc' html_css_files = [ 'theme_overrides.css', # override wide tables in RTD theme ] # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'kas.tex', 'kas Documentation', 'Daniel Wagner, Jan Kiszka, Claudius Heine', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # # latex_use_parts = False # If true, show page references after internal links. # # latex_show_pagerefs = False # If true, show URL addresses after external links. # # latex_show_urls = False # Documents to append as an appendix to all manuals. # # latex_appendices = [] # It false, will not define \strong, \code, itleref, \crossref ... but only # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added # packages. # # latex_keep_old_macro_names = True # If false, no module index is generated. # # latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('_man/kas', 'kas', 'a setup tool for bitbake based projects', [author], 1), ('_man/kas-container', 'kas-container', 'a setup tool for bitbake based projects', [author], 1), ('_man/kas-plugin-build', 'kas-build', 'kas build plugin', [author], 1), ('_man/kas-plugin-checkout', 'kas-checkout', 'kas checkout plugin', [author], 1), ('_man/kas-plugin-clean', 'kas-clean', 'kas clean command', [author], 1), ('_man/kas-plugin-cleanall', 'kas-cleanall', 'kas cleanall command', [author], 1), ('_man/kas-plugin-cleansstate', 'kas-cleansstate', 'kas cleansstate command', [author], 1), ('_man/kas-plugin-diff', 'kas-diff', 'kas diff plugin', [author], 1), ('_man/kas-plugin-dump', 'kas-dump', 'kas dump plugin', [author], 1), ('_man/kas-plugin-for-all-repos', 'kas-for-all-repos', 'kas for-all-repos plugin', [author], 1), ('_man/kas-plugin-lock', 'kas-lock', 'kas lock plugin', [author], 1), ('_man/kas-plugin-menu', 'kas-menu', 'kas menu plugin', [author], 1), ('_man/kas-plugin-purge', 'kas-purge', 'kas purge command', [author], 1), ('_man/kas-plugin-shell', 'kas-shell', 'kas shell plugin', [author], 1), ('_man/kas-project-config', 'kas-project-config', 'kas project configuration', [author], 1), ('_man/kas-format-changelog', 'kas-format-changelog', 'project configuration changelog', [author], 1), ('_man/kas-credentials', 'kas-credentials', 'kas credential handling', [author], 1), ('_man/kas-build-attestation', 'kas-build-attestation', 'working with SLSA / in-toto attestations', [author], 1), ] # If true, show URL addresses after external links. # # man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'kas', 'kas Documentation', author, 'kas', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. # # texinfo_appendices = [] # If false, no module index is generated. # # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. # # texinfo_no_detailmenu = False siemens-kas-41ad961/docs/devguide.rst000066400000000000000000000170641520561422700175560ustar00rootroot00000000000000Developer Guide =============== Deploy for development ---------------------- This project uses pip to manage the package. If you want to work on the project yourself you can create the necessary links via:: $ pip3 install --user -e . That will install a backlink ~/.local/bin/kas to this project. Now you are able to call it from anywhere. For local development, use the **run-kas** wrapper from the project root directory. In this case, replace ``kas`` with ``path/to/run-kas``. Making Changes -------------- These sections provide an overview of common modifications along with the required steps. Changes of the project configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When changing the project configuration, you need to update the json configuration schema (``schema-kas.json``). Further, a short description of the changes needs to be added to :doc:`format-changelog`. After making the changes, you need to update the minimum and maximum values of ``header.version``. If the version was already updated after the last release, the version bump is not required. Add a new CLI option ^^^^^^^^^^^^^^^^^^^^ Options that take a parameter (e.g. ``--format json``) must be handled in ``kas-container`` as well. To keep the handling in ``kas-container`` simple, try to choose a unique option name across all plugins. Add a new sub-command (plugin) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ To add a new sub-command, you need to create a new python file in the ``kas/plugins`` directory. It then needs to be imported and registered in ``kas/plugins/__init__.py``. Further, it needs to be registered in ``kas-container``, as well as in the ``container-entrypoint``. Each sub-command must be documented and have its own man page. The documentation is generated from the docstrings of the sub-command file and must be registered in ``docs/userguide/plugins.rst``. In addition, a manpage should be added in ``docs/_man/kas-plugin-`` and registered in ``docs/conf.py`` (as ``kas-.1``). Add support for new credentials ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Credentials are passed using environment variables. For details, see :ref:`checkout-creds-label`. These can either contain the credential directly or point to a credential file (e.g. ``.netrc``). To add support for a new credential, the following steps are required: - document the variable in :ref:`env-vars-label` - add the variable to the ``ENV_VARS`` list in ``libcmds.py::SetupHome`` - add a forward of the variable in ``kas-container`` - add the variable to the ``test_environment_variables.py`` test For variables pointing to a credential file, the following applies in addition: - the variable should end in ``_FILE`` (exceptions may apply) - ``kas-container`` - bind-mount the variable into ``/var/kas/userdata/`` - rewrite the variable to the path inside the container If the variable does not end in ``_FILE``, manual processing in the ``container-entrypoint`` script is needed to support it under rootless docker. Container image build --------------------- To build the container images kas provides, there is a script provided for your convenience. It uses docker buildx and requires BuildKit 0.13.0 or newer. To start the build both container variants, invoke:: $ scripts/build-container.sh You can limit the target type to either Yocto/OE (``kas``) or isar (``kas-isar``) via the ``--target`` options. See the script help for more options. Since release 4.3, the containers officially provided via ghcr.io are fully reproducible. To test this, you can use the following script, e.g. to validate that release:: $ scripts/reproduce-container.sh kas:4.3 Both scripts also support building/checking of the arm64 container images. See the help of both scripts for more details. Testing ------- The kas project has an extensive test suite. When adding new features or fixing bugs, it is recommended to add a test. The tests are written using the pytest framework. Decoupling from the calling environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Please make sure to decouple the tests from your local environment. To simplify this, we provide the ``monkeykas`` fixture to clean up the environment prior to each test. When adding new kas environment variables, make sure to add these to the cleanup handler as well. When writing tests, no assumptions about the values of the following environment variables should be made (they also can be unset): - ``KAS_WORK_DIR`` - ``KAS_BUILD_DIR`` As tests might want to check data in the work or build dir, we provide the following helpers to safely access the corresponding paths (by reading the value from the environment variable at call time): - ``monkeykas.get_kwd()``: absolute path to the current kas work dir - ``monkeykas.get_kbd()``: absolute path to the current kas build dir - ``monkeykas.move_to_kwd(path)``: move the path to into the ``KAS_WORK_DIR``, if needed Tests that explicitly check for correct handling of the directory layout are encouraged to parameterize these paths by temporarily setting them via ``monkeykas.setenv()``. Tests that rely on features that might be subject to the current working directory should be marked with the ``dirsfromenv`` marker. By that, various combinations of ``KAS_WORK_DIR`` and ``KAS_BUILD_DIR`` are tested. Executing the testsuite ^^^^^^^^^^^^^^^^^^^^^^^ .. note:: The menu plugin tests require the ``snack`` package to be installed. On most distros this is packaged in ``python3-newt``, on Arch Linux it is part of libnewt. To run the tests, invoke:: $ python3 -m pytest Online and offline testing ^^^^^^^^^^^^^^^^^^^^^^^^^^ Some tests require internet access to fetch resources. These tests are marked with the ``online`` marker. To run these tests, invoke:: $ python3 -m pytest -m online To run all tests except the online tests, invoke:: $ python3 -m pytest -m "not online" When adding new tests, please consider whether they require internet access or not and mark them accordingly. In general, we prefer offline tests. Measure code coverage ^^^^^^^^^^^^^^^^^^^^^ To measure the code coverage of the unit tests, the ``pytest-cov`` package is required. On Debian systems, this is provided in ``python3-pytest-cov``. Once installed, run:: $ python3 -m pytest --cov --cov-report html The coverage in HTML format can then be found in `htmlcov`. Community Resources ------------------- Project home: - https://github.com/siemens/kas Source code: - https://github.com/siemens/kas.git - ``git@github.com:siemens/kas.git`` Documentation: - https://kas.readthedocs.org Mailing list: - kas-devel@googlegroups.com - Subscription: - kas-devel+subscribe@googlegroups.com - https://groups.google.com/forum/#!forum/kas-devel/join - Archives - https://groups.google.com/forum/#!forum/kas-devel - https://www.mail-archive.com/kas-devel@googlegroups.com/ Class reference documentation ----------------------------- ``kas.kas`` Module ^^^^^^^^^^^^^^^^^^ .. automodule:: kas.kas :members: ``kas.libkas`` Module ^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.libkas :members: ``kas.libcmds`` Module ^^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.libcmds :members: ``kas.config`` Module ^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.config :members: ``kas.repos`` Module ^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.repos :members: ``kas.includehandler`` Module ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.includehandler :members: ``kas.kasusererror`` Module ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.kasusererror :members: ``kas.plugins`` Module ^^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.plugins :members: siemens-kas-41ad961/docs/format-changelog.rst000066400000000000000000000102741520561422700211730ustar00rootroot00000000000000Configuration Format Changes ============================ Version 1 (Alias '0.10') ------------------------ Added ~~~~~ - Include mechanism - Version check Version 2 --------- Changed ~~~~~~~ - Configuration file versions are now integers Fixed ~~~~~ - Including files from repos that are not defined in the current file Version 3 --------- Added ~~~~~ - ``Task`` key that allows one to specify which task to run (``bitbake -c``) Version 4 --------- Added ~~~~~ - ``Target`` key now allows one to pass a list of target names Version 5 --------- Changed behavior ~~~~~~~~~~~~~~~~ - Using ``multiconfig:*`` targets adds appropriate ``BBMULTICONFIG`` entries to the ``local.conf`` automatically. Version 6 --------- Added ~~~~~ - ``env`` key now allows one to pass custom environment variables to the bitbake build process. Version 7 --------- Added ~~~~~ - ``type`` property to ``repos`` to be able to express which version control system to use. Version 8 --------- Added ~~~~~ - ``patches`` property to ``repos`` to be able to apply additional patches to the repo. Version 9 --------- Added ~~~~~ - ``defaults`` key can now be used to set a default value for the repository property ``refspec`` and the repository patch property ``repo``. These default values will be used if the appropriate properties are not defined for a given repository or patch. Version 10 ---------- Added ~~~~~ - ``build_system`` property to pre-select OE or Isar. Version 11 ---------- Changed behavior ~~~~~~~~~~~~~~~~ - String item ``includes`` are now using repo-relative paths. File-relative is still supported by issues a deprecation warning. - bblayers.conf is generated with ``BBPATH`` and ``BBFILES`` preset to common defaults. Those can still be overwritten via ``bblayers_conf_headers``. Added ~~~~~ - ``menu_configuration`` key stores the selections done via ``kas menu`` in a configuration file. It is only evaluated by that plugin. Version 12 ---------- Added ~~~~~ - For repositories, ``url`` and ``path`` can now be overridden with a null-value to switch between version-controlled repositories and unversioned local folders. Version 13 ---------- Added ~~~~~ - Variables used in the ``env`` section can now be assigned *null* as value. In this case the variables are only exported to the bb env whitelist. Version 14 ---------- Added ~~~~~ - The keys ``commit`` and ``branch`` are introduced as replacement of ``refspec`` in ``repos`` and ``defaults``. ``refspec`` is now deprecated and will eventually be removed. - The ``overrides`` top-level entry can be used to pin floating repo branches. - ``_source_dir`` top-level entry is auto-generated when using the menu plugin and provides the path to the top repo at time of invoking the plugin. - ``_source_dir_host`` top-level entry is auto-generated by kas-container to track the source path outside of the container. Version 15 ---------- Added ~~~~~ - The key ``tag`` is introduced as a complement to ``commit`` and ``branch`` in ``repos``. Version 16 ---------- Fixed ~~~~~ - The key ``tag`` introduced in v15 was not supported in ``defaults``. It's now added. Version 17 ---------- Added ~~~~~ - The key ``artifacts`` is introduced to describe expected build artifacts. Version 18 ---------- Added ~~~~~ The repo keys ``tag`` and ``branch`` can now be set to ``null`` to remove these properties from the repo. This is needed in case a default value is set in the ``defaults`` section that should not apply to a repo. Version 19 ---------- Added ~~~~~ Various keys for signature verification of repositories were added: - top level ``signers`` object to specify keys for signature verification - in repo definitions, ``signed`` and ``allowed_signers`` are added to specify whether a repo is signed and which data to use for verification. Version 20 ---------- Added ~~~~~ - The repo key ``branch`` can now be overridden, including to a null-value. Version 21 ---------- Added ~~~~~ - The repo layers ``prio`` can be used to control the order in which the layers are added to the ``BBLAYERS`` bitbake variable. Version 22 ---------- Added ~~~~~ - Switch to nodistro which is the default distro setting in openembedded-core. siemens-kas-41ad961/docs/index.rst000066400000000000000000000005261520561422700170640ustar00rootroot00000000000000Welcome to the kas documentation, a setup tool for bitbake based projects ========================================================================= .. toctree:: :maxdepth: 2 intro userguide command-line devguide format-changelog Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` siemens-kas-41ad961/docs/intro.rst000066400000000000000000000011441520561422700171050ustar00rootroot00000000000000Introduction ============ This tool provides an easy mechanism to setup bitbake based projects. The OpenEmbedded tooling support starts at step 2 with bitbake. The downloading of sources and then configuration has to be done by hand. Usually, this is explained in a README. Instead kas is using a project configuration file and does the download and configuration phase. Key features provided by the build tool: - clone and checkout bitbake layers - create default bitbake settings (machine, arch, ...) - launch minimal build environment, reducing risk of host contamination - initiate bitbake build process siemens-kas-41ad961/docs/requirements.txt000066400000000000000000000001171520561422700205030ustar00rootroot00000000000000sphinx>=1.2.0 CommonMark>=0.5.6 sphinx-argparse>=0.2.1 sphinx-rtd-theme>=1.3.0 siemens-kas-41ad961/docs/userguide.rst000066400000000000000000000003261520561422700177470ustar00rootroot00000000000000User Guide ========== .. toctree:: :maxdepth: 2 userguide/getting-started userguide/plugins userguide/project-configuration userguide/build-attestation userguide/credentials userguide/kas-container siemens-kas-41ad961/docs/userguide/000077500000000000000000000000001520561422700172145ustar00rootroot00000000000000siemens-kas-41ad961/docs/userguide/aws-sso-warning.inc000066400000000000000000000005651520561422700227540ustar00rootroot00000000000000.. note:: When using |AWS_CONFIG_SOURCE| the entire content of ``~/.aws/sso/cache`` directory is copied into the kas workspace. This might expose all active user sessions, including those not defined in the ``AWS_CONFIG_FILE``. To mitigate security risks, log out of unnecessary profiles before starting a build or use a separate system account to run the build. siemens-kas-41ad961/docs/userguide/build-attestation.rst000066400000000000000000000072321520561422700234060ustar00rootroot00000000000000Build Attestation ================= kas supports the generation of SLSA / in-toto build attestation data. Currently, support for the following attestation formats is implemented: - `provenance v1 `_ Provenance ---------- The provenance data provides information about the build. This includes data about the build environment, as well as all primary (first-level) dependencies. The attestation will be stored in ``attestation/kas-build.provenance.json`` The following modes are supported: ``mode=min``, ``mode=max``, whereby the mode controls the amount of information that is included in the attestation. The CLI options hereby loosely follow the ``docker buildx`` provenance options, which are described in detail in `Provenance attestations `_. For compatibility with the ``docker buildx`` CLI options, we also support ``--provenance true``, which is equivalent to ``--provenance mode=min``. In min mode, the provenance attestation will contain information about: - Build timestamps - Build materials (project-config files) - Source repositories and revisions - Build platform In max mode, it will also contain the build environment, i.e. all environment variables of the ``env`` section along with their values. .. warning:: In max mode, the provenance attestation captures all environment variables specified in the ``env`` section. Make sure to not expose any secrets. For example, to build the configuration described in the file ``kas-project.yml`` and emit provenance attestations, you could run:: kas build --provenance mode=max kas-project.yml Working with sigstore cosign ---------------------------- The `cosign tool `_ from the `sigstore project `_ (`documentation `_) has native support for in-toto build predicates. However, it currently can only operate directly on the predicate but not on the enclosing attestation (cosign 2.2.4). By that, the predicate first needs to be extracted (provenance in this example):: cat build/attestation/kas-build.provenance.json | jq '.predicate' > provenance.json Attestation Signing ~~~~~~~~~~~~~~~~~~~ .. warning:: The following command operates on the public append-only transparency log. Make sure to understand the implications before executing. The following example shows how to create a signed build attestation from a provenance predicate and a local artifact, using the OIDC keyless signing workflow: .. code-block:: sh cosign attest-blob \ --type=slsaprovenance1 \ --predicate=provenance.json \ --output-signature=kas-build.dsse.json \ --output-certificate=cert.pem \ .wic .. note:: Currently the attestation can only be created for a single artifact file. For later verification, both the signed attestation (``kas-build.dsse.json``), as well as the certificate (``cert.pem``) are needed. Make sure to ship them along with the artifact. Attestation Verification ~~~~~~~~~~~~~~~~~~~~~~~~ The previously signed blob can be verified again: .. code-block:: sh cosign verify-blob-attestation \ --certificate=cert.pem \ --signature=kas-build.dsse.json \ --certificate-identity="" \ --certificate-oidc-issuer="" \ --type=slsaprovenance1 \ .wic Attestation Restoration ~~~~~~~~~~~~~~~~~~~~~~~ The dsse record (``kas-build.dsse.json``) contains the original attestation data as base64 encoded string. To restore the attestation, run:: cat kas-build.dsse.json | jq -r '.payload' | base64 -d | jq siemens-kas-41ad961/docs/userguide/credentials.rst000066400000000000000000000113231520561422700222430ustar00rootroot00000000000000.. _checkout-creds-label: Credential Handling =================== kas provides various mechanisms to inject credentials into the build. By using :ref:`env-vars-label`, a fine grained control is possible. All credentials are made available both to KAS, as well as inside the build environment. However, not all mechanisms are natively supported by all tools. As kas might need to modify credentials and config files, these are copied into the isolated environment first. .. note:: In general, file based credentials (e.g. ``.netrc``) are only copied if explicitly requested by setting the corresponding environment variable. Environment variable based credentials are automatically forwarded. .. only:: man For details about credential related environment variables, see :manpage:`kas(1)`. AWS Configuration ----------------- For AWS, conventional AWS config files, AWS SSO as well as the environment variable controlled OAuth 2.0 workflow are supported. Note, that KAS internally rewrites the ``AWS_*`` environment variables into a AWS config file to also support older versions of bitbake. .. include:: aws-sso-warning.inc Git Configuration ----------------- A ``.gitconfig`` file can be used to provide credentials as well as URL rewrites of git repositories (``insteadof``). In addition, credential helpers can be used by setting the corresponding environment variables. These are added to the ``.gitconfig`` file as well. To support the patching of git repositories, kas injects a ``[user]`` section, possibly overwriting an existing one. After patching, the original user is restored (if set). When running in a GitHub Action or GitLab CI job, the ``.gitconfig`` file is automatically injected. Otherwise, the environment variable ``GITCONFIG_FILE`` needs to point to the `.gitconfig` kas should use. GitHub Actions ~~~~~~~~~~~~~~ In combination with the `webfactory/ssh-agent `_ action, this automatically makes the required credentials available to kas and bitbake. GitLab CI ~~~~~~~~~ When running in the GitLab CI, the ``CI_JOB_TOKEN`` can be used to access git repositories via https. If ``CI_SERVER_HOST`` is also set, kas automatically adds this token to the ``.netrc`` file, where it is picked up by git. Further, kas configures git to automatically rewrite the URLs of the repositories to clone via https for repos stored on the same server. Technically this is achieved by adding `insteadof` entries to the ``.gitconfig`` file. For backwards compatibility, the git rewrite rules are only added if ``.gitconfig`` does not exist and no SSH configuration is provided (either via the kas ``SSH_`` variables or using ``.ssh/config``). If the ``CI_REGISTRY``, ``CI_REGISTRY_USER`` and ``CI_JOB_TOKEN`` variables are set, kas automatically creates a login file for the container registry at ``~/.docker/config.json``. This file is compatible with docker, podman and even skopeo. .. note:: Make sure to assign the correct permissions to the ``CI_JOB_TOKEN``. For details, see `GitLab CI/CD job token `_. Container Registry Authentication File -------------------------------------- A file named ``config.json`` is saved as ``.docker/config.json`` in the kas home directory. It contains credentials for the container registry login. The syntax is described in the `containers-auth.json specification `_. The authentication file is compatible with docker, podman and skopeo. When running in the GitLab CI, the ``CI_JOB_TOKEN`` is appended to automatically grant access according to the job permissions. Netrc File ---------- A ``.netrc`` file can be used to provide credentials for git or the HTTP(S) / FTP fetcher. When running in the GitLab CI, the ``CI_JOB_TOKEN`` is appended to automatically grant access to repositories that can be accessed by the user that triggered the CI pipeline. SSH --- The ssh folder of the calling user is automatically shared with kas. This is currently not controllable, as ssh does not obey the ``$HOME`` variable. This can be used to inject both credentials, as well as ssh configuration items into the kas environment. .. note:: Modifications to the ``.ssh/config`` file are only performed if the file is not present yet. In addition, an external ssh-agent can be made available in the kas environment by setting the ``SSH_AUTH_SOCK`` environment variable. As an alternative, ssh private keys can be added to an internal ssh agent by setting ``SSH_PRIVATE_KEY`` or ``SSH_PRIVATE_KEY_FILE``. .. note:: The use of an external ssh agent cannot be combined with options that require an internal ssh agent. .. |AWS_CONFIG_SOURCE| replace:: ``AWS_CONFIG_FILE`` siemens-kas-41ad961/docs/userguide/getting-started.rst000066400000000000000000000067231520561422700230630ustar00rootroot00000000000000Getting Started =============== Installation ------------ pipx is the recommended way to install kas. It creates an isolated Python virtual environment for kas and makes sure its dependencies are installed correctly. To install pipx, follow the instructions at https://pipx.pypa.io/stable/installation/ To install the latest version of kas using pipx run:: $ pipx install kas With menu support ~~~~~~~~~~~~~~~~~ kas menu requires an additional system dependency. On Debian-based systems, run:: $ sudo apt install python3-newt $ pipx install kas --system-site-packages Other installation methods ~~~~~~~~~~~~~~~~~~~~~~~~~~ Debian-based distributions provide a package for kas. However, this may be outdated compared to the latest release. To install kas using the system package manager, run:: $ sudo apt install kas Dependencies ------------ This project depends on - Python 3 - distro Python 3 package - jsonschema Python 3 package - PyYAML Python 3 package - GitPython Python 3 package - kconfiglib Python 3 package (optional, for menu plugin) - NEWT Python 3 distro package (optional, for menu plugin) - python-gnupg Python 3 package (optional, for signature verification) Usage ----- There are (at least) three options for using kas: - Install it locally via pip to get the ``kas`` command. - Use the container image locally. In this case, download the :doc:`kas-container ` script from the kas repository and use it in place of the ``kas`` command. The script version corresponds to the kas tool and the kas image version. - Use the container image in CI. Specify ``ghcr.io/siemens/kas/kas[-isar][:][-]`` in your CI script that requests a container image as runtime environment. Start build:: $ kas build /path/to/kas-project.yml Alternatively, experienced bitbake users can invoke usual **bitbake** steps manually, e.g.:: $ kas shell /path/to/kas-project.yml -c 'bitbake dosfsutils-native' For details about the kas input file(s), see :ref:`project-configuration-label`. Example configurations can be found in :ref:`example-configurations-label`. Directory Layout ~~~~~~~~~~~~~~~~ When invoking kas, it places download and build artifacts in the current directory by default. You can specify a different location using the environment variable ``KAS_WORK_DIR``. Repositories managed by kas are stored under their ``path`` (or ``name`` if ``path`` is not set). The build directory is named ``build`` and is relative to ``KAS_WORK_DIR`` unless explicitly set with ``KAS_BUILD_DIR``. Internal data that persists across executions is prefixed with ``.kas_``. Use Cases --------- 1. Initial build/setup:: $ mkdir $PROJECT_DIR $ cd $PROJECT_DIR $ git clone $PROJECT_URL meta-project $ kas build meta-project/kas-project.yml 2. Update/rebuild:: $ cd $PROJECT_DIR/meta-project $ git pull $ kas build kas-project.yml 3. Interactive configuration:: $ cd $PROJECT_DIR/meta-project $ kas menu $ kas build # optional, if not triggered via kas menu Community Support ----------------- If you have questions on kas, found a bug or would need an additional feature, please post to our mailing list: `kas-devel@googlegroups.com `_. You do not need to be subscribed to post but if you like to do so, this works either `via email `_ or via the `Google Groups page `_ after signing in. siemens-kas-41ad961/docs/userguide/kas-container-description.inc000066400000000000000000000050451520561422700247720ustar00rootroot00000000000000The ``kas-container`` script is a wrapper to run `kas` inside a build container. It gives fine grained control over the data that is mapped into the build and decouples the build environment from the host system. For details, see :ref:`env-vars-label`. The wrapper also takes care of mounting the necessary directories and setting up the environment variables inside the container. .. note:: The ``kas-container`` script has limited support for Git worktrees. Regular Git operations on the checked-out repository are supported. However, executing any ``git worktree ...`` command inside the container is not allowed. By default ``kas-container`` uses the official images provided by the kas project: ``ghcr.io/siemens/kas/kas[-isar]:``. To specify your own image set the ``KAS_CONTAINER_IMAGE`` environment variable. The ``kas-container`` script version should match the kas version inside the container. If kas detects that is was called from ``kas-container`` and the versions do not match, a warning is emitted. This limitation might be lessened in the future, once a stable interface between ``kas-container`` and kas is introduced. From version ``5.0`` onward, kas offers images built on several base distributions. Select a distribution by setting the environment variable ``KAS_CONTAINER_IMAGE_DISTRO`` to the desired value (e.g. ``debian-bookworm`` or ``debian-trixie``). The corresponding image tags follow the pattern ``:-`` (e.g. ``:5.0-debian-bookworm``). Alternatively, you can adjust ``KAS_CONTAINER_IMAGE_DISTRO_DEFAULT`` in the ``kas-container`` script if you copy this into your downstream layer already for encoding the supported kas version. As container backends, Docker and Podman are supported. To force the use of podman over docker, set ``KAS_CONTAINER_ENGINE=podman``. For details, see :ref:`env-vars-label`. Running under docker in `rootless mode `_ is partially supported. It is recommended to use a distinct ``KAS_WORK_DIR`` outside of the calling directory (repo-dir), as kas temporarily changes the ownership of the working directory during its operation. All files managed by kas (including the repos) must not be written to from the host. To completely remove all data managed by kas, use ``kas-container purge``. This also restores the directory owners of the dirs passed to kas, so they can be removed from the host. .. note:: The ISAR build system is not compatible with rootless execution. By that, we fall back to the system docker or podman instance. siemens-kas-41ad961/docs/userguide/kas-container.rst000066400000000000000000000006311520561422700225040ustar00rootroot00000000000000Building in a Container ======================= .. include:: kas-container-description.inc .. include:: ../_build/kas-container-usage-synopsis.inc .. include:: ../_build/kas-container-usage-options.inc .. include:: aws-sso-warning.inc .. |SYNOPSIS| replace:: Synopsis .. |OPTIONS| replace:: Options .. |KAS-COMMANDS| replace:: kas-container Commands .. |AWS_CONFIG_SOURCE| replace:: ``--aws-dir ~/.aws`` siemens-kas-41ad961/docs/userguide/plugins.rst000066400000000000000000000044331520561422700214330ustar00rootroot00000000000000Sub-commands (Plugins) ====================== kas sub-commands are implemented by a series of plugins. Each plugin typically provides a single command. ``build`` plugin ---------------- .. automodule:: kas.plugins.build .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: build ``checkout`` plugin ------------------- .. automodule:: kas.plugins.checkout .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: checkout ``clean`` plugin ---------------- .. automodule:: kas.plugins.clean ``clean`` command ^^^^^^^^^^^^^^^^^ .. automodule:: kas.plugins.clean.Clean .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: clean ``cleansstate`` command ^^^^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.plugins.clean.CleanSstate .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: cleansstate ``cleanall`` command ^^^^^^^^^^^^^^^^^^^^ .. automodule:: kas.plugins.clean.CleanAll .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: cleanall ``diff`` plugin ---------------- .. automodule:: kas.plugins.diff .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: diff ``dump`` plugin --------------- .. automodule:: kas.plugins.dump .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: dump ``for-all-repos`` plugin ------------------------ .. automodule:: kas.plugins.for_all_repos .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: for-all-repos ``lock`` plugin ------------------------ .. automodule:: kas.plugins.lock .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: lock ``menu`` plugin --------------- .. automodule:: kas.plugins.menu .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: menu ``purge`` plugin ---------------- .. automodule:: kas.plugins.clean.Purge .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: purge ``shell`` plugin ---------------- .. automodule:: kas.plugins.shell .. argparse:: :module: kas.kas :func: kas_get_argparser :prog: kas :path: shell siemens-kas-41ad961/docs/userguide/project-configuration.rst000066400000000000000000000537101520561422700242670ustar00rootroot00000000000000.. _project-configuration-label: Project Configuration ===================== Currently, JSON and YAML 1.1 are supported as the base file formats. Since YAML is arguably easier to read, this documentation focuses on the YAML format. .. code-block:: yaml # Every file needs to contain a header, that provides kas with information # about the context of this file. header: # The `version` entry in the header describes for which configuration # format version this file was created for. It is used by kas to figure # out if it is compatible with this file. The version is an integer that # is increased on every format change. version: x # The machine as it is written into the `local.conf` of bitbake. machine: qemux86-64 # The distro name as it is written into the `local.conf` of bitbake. distro: poky repos: # This entry includes the repository where the config file is located # to the bblayers.conf: meta-custom: # Here we include a list of layers from the poky repository to the # bblayers.conf: poky: url: "https://git.yoctoproject.org/git/poky" commit: 89e6c98d92887913cadf06b2adb97f26cde4849b layers: meta: meta-poky: meta-yocto-bsp: A minimal input file consists out of the ``header``, ``machine``, ``distro``, and ``repos``. Additionally, you can add ``bblayers_conf_header`` and ``local_conf_header`` which are strings that are added to the head of the respective files (``bblayers.conf`` or ``local.conf``): .. code-block:: yaml bblayers_conf_header: meta-custom: | POKY_BBLAYERS_CONF_VERSION = "2" BBPATH = "${TOPDIR}" BBFILES ?= "" local_conf_header: meta-custom: | PATCHRESOLVE = "noop" CONF_VERSION = "1" IMAGE_FSTYPES = "tar" ``meta-custom`` in these examples should be a unique name for this configuration entries. We recommend that this unique name is the **same** as the name of the containing repository/layer to ease cross-project referencing. In given examples we assume that your configuration file is part of a ``meta-custom`` repository/layer. This way it is possible to overwrite or append entries in files that include this configuration by naming an entry the same (overwriting) or using an unused name (appending). .. note:: kas internally uses ``PyYAML`` to parse YAML documents, inheriting its limitations. Notably, ``PyYAML`` only supports YAML 1.1 and does not correctly handle non-string keys in mappings. To avoid this issue, we recommend quoting keys of other types, such as octal numbers (``0001``), integers (``42``), booleans (``false``) and special values (``no``). Including in-tree configuration files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It's currently possible to include kas configuration files from the same repository/layer like this: .. code-block:: yaml header: version: x includes: - base.yml - bsp.yml - product.yml If the configuration file is inside a repository, then path is relative to the repositories base directory. If the configuration file is not in a repository, then the path is relative to the parent directory of the file. Including configuration files from other repos ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It's also possible to include configuration files from other repos like this: .. code-block:: yaml header: version: x includes: - repo: poky file: kas-poky.yml - repo: meta-bsp-collection file: hw1/kas-hw-bsp1.yml - repo: meta-custom file: products/product.yml repos: meta-custom: meta-bsp-collection: url: "https://www.example.com/git/meta-bsp-collection" commit: 3f786850e387550fdab836ed7e6dc881de23001b layers: # Additional to the layers that are added from this repository # in the hw1/kas-hw-bsp1.yml, we add here an additional bsp # meta layer: meta-custom-bsp: poky: url: "https://git.yoctoproject.org/git/poky" commit: 89e6c98d92887913cadf06b2adb97f26cde4849b layers: # If `kas-poky.yml` adds the `meta-yocto-bsp` layer and we # do not want it in our bblayers for this project, we can # overwrite it by setting: meta-yocto-bsp: excluded The files are addressed relative to the git repository path and must not traverse outside of the referenced repo. The include mechanism collects and merges the content from top to bottom and depth first. That means that settings in one include file are overwritten by settings in a latter include file and entries from the last include file can be overwritten by the current file. .. warning:: The include mechanism does not support circular references with respect to the ``repos`` entries. By that, a (transitive) include file must not change the reference of the repository it is included from. While merging, all the dictionaries are merged recursively while preserving the order in which the entries are added to the dictionary. This means that ``local_conf_header`` entries are added to the ``local.conf`` file in the same order in which they are defined in the different include files. The ``header.version`` property is always set to the highest version number found in the config files. .. note:: Internally kas iterates the repository checkout step until all referenced repositories are resolved (checked out). After each iteration, the (partial) configuration is merged and the next iteration is started. Once all repositories are available, the final configuration is build. Then, all remaining repositories are checked out. Including configuration files via the command line ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When specifying the kas configuration file on the command line, additional configurations can be included ad-hoc:: $ kas build kas-base.yml:debug-image.yml:board.yml This is equivalent to static inclusion from some ``kas-combined.yml`` like this: .. code-block:: yaml header: version: x includes: - kas-base.yml - debug.image.yml - board.yml Command line inclusion allows one to create configurations on-demand, without the need to write a kas configuration file for each possible combination. All configuration files combined via the command line either have to come from the same repository or have to live outside of any versioning control. kas will refuse any other combination in order to avoid complications and configuration flaws that can easily emerge from them. .. note:: Git submodules are considered to be part of the main repository. Hence, including config files from a submodule is supported. The repository root is always the root of the main repository (if under VCS) or the directory of the first kas config file otherwise. Working with lockfiles ~~~~~~~~~~~~~~~~~~~~~~ kas supports the use of lockfiles to pinpoint repositories to exact commit ID (e.g. SHA-1 refs for git). A lockfile hereby only overrides the commit ID defined in a kas file. When performing the checkout operation (or any other operation that performs a checkout), kas checks if a file named ``.lock.`` is found next to the currently processed kas file. If this is found, kas loads this file right before processing the current one (similar to an include file). .. note:: The locking logic applies to both files on the kas cmdline and include files. The following example shows this mechanism for a file ``kas/kas-isar.yml`` and its corresponding lockfile ``kas/kas-isar.lock.yml``. ``kas/kas-isar.yml``: .. code-block:: yaml # [...] repos: isar: url: https://github.com/ilbers/isar.git branch: next ``kas/kas-isar.lock.yml``: .. code-block:: yaml header: version: 14 overrides: repos: isar: commit: 0336610df8bb0adce76ef8c5a921c758efed9f45 The ``lock`` plugin provides helpers to simplify the creation and update of lockfiles. For details, see the plugins documentation: :mod:`kas.plugins.lock`. Configuration reference ~~~~~~~~~~~~~~~~~~~~~~~ ``header``: dict [required] :kasschemadesc:`header` ``version``: integer [required] :kasschemadesc:`header.properties.version` See the :doc:`configuration format changelog <../format-changelog>` for the format history and the latest available version. ``includes``: list [optional] :kasschemadesc:`header.properties.includes` An item in this list can have one of two types: item: string :kasschemadesc:`header.properties.includes.items.anyOf[0]` item: dict :kasschemadesc:`header.properties.includes.items.anyOf[1]` ``repo``: string [required] :kasschemadesc:`header.properties.includes.items.anyOf[1].properties.repo` The repo needs to be defined in the ``repos`` dictionary as ````. ``file``: string [required] :kasschemadesc:`header.properties.includes.items.anyOf[1].properties.file` ``build_system``: string [optional] :kasschemadesc:`build_system` Known build systems are ``openembedded`` (or ``oe``) and ``isar``. If set, this restricts the search of kas for the init script in the configured repositories to ``oe-init-build-env`` or ``isar-init-build-env``, respectively. If ``kas-container`` finds this property in the top-level kas configuration file (includes are not evaluated), it will automatically select the required container image and invocation mode. ``defaults``: dict [optional] :kasschemadesc:`defaults` This may help you to avoid repeating the same property assignment in multiple places if, for example, you wish to use the same branch for all repositories. ``repos``: dict [optional] :kasschemadesc:`defaults.properties.repos` If a default value is set for a repository property it may still be overridden by setting the same property to a different value in a given repository. ``branch``: string [optional] :kasschemadesc:`defaults.properties.repos.properties.branch` ``tag``: string [optional] :kasschemadesc:`defaults.properties.repos.properties.tag` ``patches``: dict [optional] :kasschemadesc:`defaults.properties.repos.properties.patches` If a default value is set for a patch property it may still be overridden by setting the same property to a different value in a given patch. ``repo``: string [optional] Sets the default ``repo`` property applied to all repository patches that do not override this. ``machine``: string [optional] :kasschemadesc:`machine` ``distro``: string [optional] :kasschemadesc:`distro` ``target``: string [optional] or list [optional] :kasschemadesc:`target` ``env``: dict [optional] :kasschemadesc:`env` Either a string or nothing (``null``) can be assigned as value. The former one serves as a default value whereas the latter one will lead to add the variable only to ``BB_ENV_PASSTHROUGH_ADDITIONS`` and not to the environment where kas is started. Please note, that ``null`` needs to be assigned as the nulltype (e.g. ``MYVAR: null``), not as 'null'. ``task``: string [optional] :kasschemadesc:`task` ``repos``: dict [optional] :kasschemadesc:`repos` ````: dict [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0]` ``name``: string [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.name` ``url``: string [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.url` ``type``: string [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.type` ``commit``: string [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.commit` ``branch``: string or nothing (``null``) [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.branch` ``tag``: string or nothing (``null``) [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.tag` ``path``: string [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.path` If the ``url`` and ``path`` is missing, the repository where the current configuration file is located is defined. If the ``url`` is missing and the path defined, this entry references the directory the path points to. If the ``url`` as well as the ``path`` is defined, the path is used to overwrite the checkout directory, that defaults to ``kas_work_dir`` + ``repo.name``. In case of a relative path name ``kas_work_dir`` is prepended. ``signed``: boolean [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.signed` ``allowed_signers``: list [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.allowed_signers` ``layers``: dict [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.layers` This allows combinations: .. code-block:: yaml repos: meta-foo: url: https://github.com/bar/meta-foo.git path: layers/meta-foo branch: master layers: .: contrib: This adds both ``layers/meta-foo`` and ``layers/meta-foo/contrib`` from the ``meta-foo`` repository to ``bblayers.conf``. ````: enum | dict [optional] Adds the layer with ```` that is relative to the repository root directory, to the ``bblayers.conf`` if the value of this entry is not ``disabled``. This way it is possible to overwrite the inclusion of a layer in later loaded configuration files. To re-enable it, set it to nothing (``null``). ``prio``: integer [optional] :kasschemadesc:`$defs.layerPrio.properties.prio` ``patches``: dict [optional] :kasschemadesc:`repos.additionalProperties.anyOf[0].properties.patches` ````: dict [optional] One entry in patches with its specific and unique id. All available patch entries are applied in the order of their sorted ````. ``repo``: string [required] The identifier of the repo where the path of this entry is relative to. ``path``: string [required] The path to one patch file or a quilt formatted patchset directory. ``overrides``: dict [optional] :kasschemadesc:`overrides` ``repos``: dict [optional] Maps to the top-level ``repos`` entry. ````: dict [optional] Maps to the ```` entry. ``commit``: string [optional] Pinned commit ID which overrides the ``commit`` of the corresponding repo. ``bblayers_conf_header``: dict [optional] :kasschemadesc:`bblayers_conf_header` ````: string [optional] A string that is added to the ``bblayers.conf``. The entry id (````) should be unique if lines should be added and can be the same from another included file, if this entry should be overwritten. The lines are added to ``bblayers.conf`` in alphabetic order of ```` to ensure deterministic generation of config files. ``local_conf_header``: dict [optional] :kasschemadesc:`local_conf_header` ````: string [optional] A string that is added to the ``local.conf``. It operates in the same way as the ``bblayers_conf_header`` entry. ``menu_configuration``: dict [optional] :kasschemadesc:`menu_configuration` Each variable corresponds to a Kconfig configuration variable and can be of the types string, boolean or integer. The content of this key is typically maintained by the ``kas menu`` plugin in a ``.config.yaml`` file. ``artifacts``: dict [optional] :kasschemadesc:`artifacts` Each key-value pair describes an identifier and a path relative to the kas build dir, whereby the path can contain wildcards like ``*``. Unix-style globbing is applied to all paths. In case no artifact is found, the build is considered successful, if not stated otherwise by the used plugin and mode of operation. .. note:: There are no further semantics attached to the identifiers (yet). Both the author and the consumer of the artifacts node need to agree on the semantics. Example: .. code-block:: yaml artifacts: disk-image: path/to/image.*.img firmware: path/to/firmware.bin swu: path/to/update.swu ``signers``: dict [optional] :kasschemadesc:`signers` This dict contains the public keys or certificates that are used to verify the authenticity of the repositories. In case of GPG keys, these are made available to the build environment as well by pointing the ``GNUPGHOME`` environment variable to the local keystore. A single signer configuration must not be split across multiple config files. ````: dict [optional] :kasschemadesc:`signers.additionalProperties` For each signer, a unique identifier is required. The ```` is used to reference the entry in the ``allowed_signers`` entries. ``type``: enum [optional] :kasschemadesc:`signers.additionalProperties.properties.type` ``repo``: string [optional] :kasschemadesc:`signers.additionalProperties.properties.repo` ``path``: string [optional] :kasschemadesc:`signers.additionalProperties.properties.path` ``fingerprint``: string [optional] :kasschemadesc:`signers.additionalProperties.properties.fingerprint` **GPG key fingerprint**: The fingerprint can be obtained by running ``gpg --list-keys --with-fingerprint --keyid-format=long ``. The needed string is the 40-character fingerprint without spaces. **SSH key fingerprint**: The fingerprint can be obtained by running ``ssh-keygen -lf key.pub | awk '{print $2}'``. ``gpg_keyserver``: string [optional] :kasschemadesc:`signers.additionalProperties.properties.gpg_keyserver` ``_source_dir``: string [optional] :kasschemadesc:`_source_dir` ``_source_dir_host``: string [optional] :kasschemadesc:`_source_dir_host` It provides the absolute path to the top repo outside of the container (on the host). This value is only evaluated by the ``kas-container`` script. It must not be set manually and might only be defined in the top-level ``.config.yaml`` file. ``buildtools``: dict [optional] Provides variables to define which buildtools version should be fetched and where it is (or will be) installed. Both ``version`` and ``sha256sum`` should be set. The environment variable ``KAS_BUILDTOOLS_DIR`` can be used to set the directory where buildtools will be installed, otherwise the default path (i.e., ``KAS_BUILD_DIR/buildtools``) will be used. If such directory already has buildtools installed, kas will check the ``Distro Version`` line in the version file, and if it doesn't match with ``version``, the directory will be cleaned and kas will download buildtools according to ``version``. After the download, kas will perform integrity validation by calculating the artifact's checksum and comparing it with ``sha256sum``. As for the optional variables, they are meant to be used to support cases as: mirrors, changes in the installer's file name, and fetching unofficial (i.e., custom) buildtools. Finally, the environment-setup script will run before bitbake, so the whole buildtools environment will be available. ``wget`` is the host tool required for this feature. More information on how to install or generate buildtools can be found at: |yp_doc_buildtools| ``version``: string :kasschemadesc:`buildtools.properties.version` ``sha256sum``: string :kasschemadesc:`buildtools.properties.sha256sum` ``base_url``: string [optional] :kasschemadesc:`buildtools.properties.base_url` ``filename``: string [optional] :kasschemadesc:`buildtools.properties.filename` It will be combined with to ``base_url`` to form the whole download URL, if set. If not set, kas will combine the platform architecture and ``version`` to form the standard script filename: ``{arch}-buildtools-extended-nativesdk-standalone-{version}.sh`` Example: .. code-block:: yaml buildtools: version: "5.2" sha256sum: "6f69fba75c8f3142bb49558afa3ed5dd0723a3beda169b057a5238013623462d" And for unofficial (custom) sources: .. code-block:: yaml buildtools: version: "1.0.0" sha256sum: "a6f87e5865b63f2bc28c1f605bcef3d6680d46fa8c8616a388d4e8aa0b5c100e" base_url: "https://downloads.mysources.com/yocto/buildtools/" filename: "x86_64-buildtools-beta-testing-1.0.0.sh" .. |yp_doc_buildtools| replace:: https://docs.yoctoproject.org/dev/ref-manual/system-requirements.html#downloading-a-pre-built-buildtools-tarball Buildtools archive ------------------ kas expects the buildtools installer to be a shell script (i.e., as a standard Yocto SDK). Once executed, the resulting directory should contain the elements below: - ``sysroots``: the native and target sysroots, containing (among libraries and headers) the build system's requirements: Git, tar, Python and make. - ``environment-setup-*``: the environment setup script, sourced by kas, to setup variables such as ``PATH`` in such a way that it points to the directories in ``sysroots``. - ``version-*``: the version file. Its second line contains a string as ``Distro Version: X.Y.Z``, parsed to retrieve the version number. The archive can contain other files, such as ``buildinfo``, but they are not relevant for kas. .. _example-configurations-label: Example project configurations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The following snippets show minimal but working project configurations for both OpenEmbedded and ISAR based distributions. OpenEmbedded ------------ **Poky** .. literalinclude:: ../../examples/openembedded.yml :language: YAML :lines: 25- **oe-core (nodistro)** .. literalinclude:: ../../examples/oe-core-nodistro.yml :language: YAML :lines: 25- ISAR ---- .. literalinclude:: ../../examples/isar.yml :language: YAML :lines: 25- siemens-kas-41ad961/examples/000077500000000000000000000000001520561422700161065ustar00rootroot00000000000000siemens-kas-41ad961/examples/isar.yml000066400000000000000000000026351520561422700175750ustar00rootroot00000000000000# # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2022-2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # header: version: 14 build_system: isar machine: qemuamd64 distro: debian-trixie target: mc:qemuamd64-trixie:cowsay repos: isar: url: https://github.com/ilbers/isar.git tag: v0.11 commit: f8558fcf3ecf98e58853b82d89645bcedb24b853 layers: meta: meta-isar: siemens-kas-41ad961/examples/oe-core-nodistro.yml000066400000000000000000000035761520561422700220340ustar00rootroot00000000000000# # kas - setup tool for bitbake based projects # # Copyright (c) Siemens, 2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # header: version: 19 # Optionally provide keys to verify signed repositories signers: YoctoBuildandRelease: fingerprint: 2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9 gpg_keyserver: keyserver.ubuntu.com machine: qemux86-64 distro: nodistro target: zlib-native repos: bitbake: url: https://github.com/openembedded/bitbake.git tag: yocto-5.2.3 commit: 710f98844ae30416bdf6a01b655df398b49574ec signed: true allowed_signers: - YoctoBuildandRelease # this repo does not provide any layers layers: {} core: url: https://github.com/openembedded/openembedded-core.git tag: yocto-5.2.3 commit: 347cb0861dde58613541ce692778f907943a60ea signed: true allowed_signers: - YoctoBuildandRelease layers: meta: siemens-kas-41ad961/examples/openembedded-buildtools.yml000066400000000000000000000031731520561422700234260ustar00rootroot00000000000000# # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2022-2025 # Copyright (c) Bootlin, 2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # header: version: 19 includes: - examples/openembedded.yml repos: poky: url: https://git.yoctoproject.org/poky.git # when specifying a tag, optionally provide a commit hash tag: yocto-5.2 commit: 9b96fdbb0cab02f4a6180e812b02bc9d4c41b1a5 signed: true allowed_signers: - YoctoBuildandRelease layers: meta: meta-poky: buildtools: version: "5.2" sha256sum: "6f69fba75c8f3142bb49558afa3ed5dd0723a3beda169b057a5238013623462d" siemens-kas-41ad961/examples/openembedded.yml000066400000000000000000000032701520561422700212460ustar00rootroot00000000000000# # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2022-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # header: version: 19 # Optionally provide keys to verify signed repositories signers: YoctoBuildandRelease: fingerprint: 2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9 gpg_keyserver: keyserver.ubuntu.com machine: qemux86-64 distro: poky target: zlib-native repos: poky: url: https://git.yoctoproject.org/poky.git # when specifying a tag, optionally provide a commit hash tag: yocto-5.1.1 commit: 7e081bd98fdc5435e850d1df79a5e0f1e30293d0 signed: true allowed_signers: - YoctoBuildandRelease layers: meta: meta-poky: siemens-kas-41ad961/image-tests/000077500000000000000000000000001520561422700165125ustar00rootroot00000000000000siemens-kas-41ad961/image-tests/kas-isar/000077500000000000000000000000001520561422700202245ustar00rootroot00000000000000siemens-kas-41ad961/image-tests/kas-isar/kas.yml000077700000000000000000000000001520561422700254452../../examples/isar.ymlustar00rootroot00000000000000siemens-kas-41ad961/image-tests/kas/000077500000000000000000000000001520561422700172705ustar00rootroot00000000000000siemens-kas-41ad961/image-tests/kas/kas-buildtools.yml000077700000000000000000000000001520561422700325222../../examples/openembedded-buildtools.ymlustar00rootroot00000000000000siemens-kas-41ad961/image-tests/kas/kas.yml000077700000000000000000000000001520561422700261662../../examples/openembedded.ymlustar00rootroot00000000000000siemens-kas-41ad961/kas-container000077500000000000000000000576071520561422700167730ustar00rootroot00000000000000#!/bin/sh # # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2018-2025 # # Authors: # Jan Kiszka # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. set -e KAS_CONTAINER_SCRIPT_VERSION="5.3" KAS_IMAGE_VERSION_DEFAULT="${KAS_CONTAINER_SCRIPT_VERSION}" KAS_CONTAINER_IMAGE_DISTRO_DEFAULT="" KAS_CONTAINER_IMAGE_PATH_DEFAULT="ghcr.io/siemens/kas" KAS_CONTAINER_IMAGE_NAME_DEFAULT="kas" KAS_CONTAINER_SELF_NAME="$(basename "$0")" # usage [exit_code] usage() { EXIT_CODE="$1" SELF="${KAS_CONTAINER_SELF_NAME}" printf "%b" "Usage: ${SELF} [OPTIONS] { build | shell } [KASOPTIONS] [KASFILE]\n" printf "%b" " ${SELF} [OPTIONS] { checkout | dump | lock } [KASOPTIONS] [KASFILE]\n" printf "%b" " ${SELF} [OPTIONS] { diff } [KASOPTIONS] config1 config2\n" printf "%b" " ${SELF} [OPTIONS] for-all-repos [KASOPTIONS] [KASFILE] COMMAND\n" printf "%b" " ${SELF} [OPTIONS] { clean | cleansstate | cleanall | purge} [KASFILE]\n" printf "%b" " ${SELF} [OPTIONS] menu [KCONFIG]\n" printf "%b" "\nPositional arguments:\n" printf "%b" "build\t\t\tCheck out repositories and build target.\n" printf "%b" "checkout\t\tCheck out repositories but do not build.\n" printf "%b" "diff\t\t\tCompare two kas configurations.\n" printf "%b" "dump\t\t\tCheck out repositories and write flat version\n" printf "%b" " \t\t\tof config to stdout.\n" printf "%b" "lock\t\t\tCreate and update kas project lockfiles\n" printf "%b" "shell\t\t\tRun a shell in the build environment.\n" printf "%b" "for-all-repos\t\tRun specified command in each repository.\n" printf "%b" "clean\t\t\tClean build artifacts, keep sstate cache and " \ "downloads.\n" printf "%b" "cleansstate\t\tClean build artifacts and sstate cache, " \ "keep downloads.\n" printf "%b" "cleanall\t\tClean build artifacts, sstate cache and " \ "downloads.\n" printf "%b" "purge\t\t\tRemove all data managed by kas. Run with '--dry-run'\n" printf "%b" " \t\t\tto check what would be removed\n" printf "%b" "menu\t\t\tProvide configuration menu and trigger " \ "configured build.\n" printf "%b" "\nOptional arguments:\n" printf "%b" "--isar\t\t\tUse kas-isar container to build Isar image. To force\n" printf "%b" " \t\t\tthe use of run0 over sudo, set KAS_SUDO_CMD=run0.\n" printf "%b" "--with-loop-dev Pass a loop device to the " \ "container. Only required if\n" printf "%b" "\t\t\tloop-mounting is used by recipes.\n" printf "%b" "--runtime-args\t\tAdditional arguments to pass to the " \ "container runtime\n" printf "%b" "\t\t\tfor running the build.\n" printf "%b" "-l, --log-level\t\tSet log level (default=info).\n" printf "%b" "--version\t\tprint program version.\n" printf "%b" "--ssh-dir\t\tDirectory containing SSH configurations.\n" printf "%b" "\t\t\tAvoid \$HOME/.ssh unless you fully trust the " \ "container.\n" printf "%b" "--ssh-agent\t\tForward ssh-agent socket to the container.\n" printf "%b" "--aws-dir\t\tDirectory containing AWScli configuration.\n" printf "%b" "\t\t\tAvoid \$HOME/.aws unless you fully trust the " \ "container.\n" printf "%b" "--git-credential-store\tFile path to the git credential " \ "store\n" printf "%b" "--no-proxy-from-env\tDo not inherit proxy settings from " \ "environment.\n" printf "%b" "--repo-ro\t\tMount current repository read-only\n" \ "\t\t\t(default for build command)\n" printf "%b" "--repo-rw\t\tMount current repository writable\n" \ "\t\t\t(default for shell command)\n" printf "%b" "-h, --help\t\tShow this help message and exit.\n" printf "%b" "\n" printf "%b" "You can force the use of podman over docker using " \ "KAS_CONTAINER_ENGINE=podman.\n" exit "${EXIT_CODE:-1}" } fatal_error() { echo "${KAS_CONTAINER_SELF_NAME}: Error: $*" >&2 exit 1 } warning() { echo "${KAS_CONTAINER_SELF_NAME}: Warning: $*" >&2 } debug(){ if [ -n "${KAS_VERBOSE}" ]; then echo "${KAS_CONTAINER_SELF_NAME}: Debug: $*" >&2 fi } trace() { [ -n "${KAS_VERBOSE}" ] && echo "+ $*" >&2 "$@" } prepare_sudo_cmd() { if [ -z "${KAS_SUDO_CMD}" ]; then # Try to auto-detect a privileged executor if command -v sudo >/dev/null; then KAS_SUDO_CMD="sudo" elif command -v run0 >/dev/null; then KAS_SUDO_CMD="run0" else fatal_error "No privileged executor found, need sudo or run0." fi fi case "$KAS_SUDO_CMD" in sudo) _KAS_SUDO_CMD="sudo --preserve-env";; run0) _KAS_SUDO_CMD="run0 --background= --unit=kas-container@$$";; *) fatal_error "Unsupported KAS_SUDO_CMD ('${KAS_SUDO_CMD}'), use sudo or run0.";; esac } enable_isar_mode() { if [ -n "${ISAR_MODE}" ]; then return fi ISAR_MODE=1 KAS_CONTAINER_IMAGE_NAME_DEFAULT="kas-isar" KAS_ISAR_ARGS="--privileged" if [ "${KAS_CONTAINER_ENGINE}" = "podman" ]; then prepare_sudo_cmd # sudo is needed for a privileged podman container KAS_CONTAINER_COMMAND="${_KAS_SUDO_CMD} ${KAS_CONTAINER_COMMAND}" # preserved user PATH may lack sbin needed by privileged podman export PATH="${PATH}:/usr/sbin" elif [ "${KAS_DOCKER_ROOTLESS}" = "1" ]; then prepare_sudo_cmd DOCKER_HOST_DEFAULT="$(docker context inspect default --format '{{.Endpoints.docker.Host}}')" export DOCKER_HOST="${DOCKER_HOST:-$DOCKER_HOST_DEFAULT}" debug "kas-isar does not support rootless docker. Using system docker in $DOCKER_HOST" # force use of well-known system docker socket KAS_CONTAINER_COMMAND="${_KAS_SUDO_CMD} ${KAS_CONTAINER_COMMAND}" KAS_DOCKER_ROOTLESS=0 fi } enable_oe_mode() { if [ "${KAS_CONTAINER_ENGINE}" = "podman" ]; then # The container entry point expects that the current userid # calling "podman run" has a 1:1 mapping KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} --userns=keep-id" fi } enable_unpriv_userns_docker() { if [ -f /etc/os-release ] && grep -q 'NAME="Ubuntu"' /etc/os-release && [ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ] && [ "$(cat /proc/sys/kernel/apparmor_restrict_unprivileged_userns)" = "1" ]; then if [ -f /etc/apparmor.d/rootlesskit ]; then debug "AppArmor restricts unprivileged userns, using \"rootlesskit\" profile" KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} --security-opt apparmor=rootlesskit" else warning "AppArmor restricts unprivileged userns but no suitable apparmor " \ "profile found. Consider setting apparmor_restrict_unprivileged_userns=0" fi fi } # Params: NAME CREATE_MODE check_and_expand() { eval _varval=\"\$"$1"\" [ -z "$_varval" ] && return case "$2" in required) [ ! -d "$_varval" ] && fatal_error "Variable $1 set, but \"$_varval\" is not a directory." ;; create) [ ! -d "$_varval" ] && trace mkdir "$_varval" ;; createrec) trace mkdir -p "$_varval" ;; esac realpath -e "$_varval" } # Params: FILE # Returns: root repo dir of file repo_path_of_file() { _DIR="$(dirname "$1")" _REPO_DIR=$(git -C "${_DIR}" rev-parse --show-toplevel 2>/dev/null) \ || _REPO_DIR=$(hg --cwd "${_DIR}" root 2>/dev/null) \ || _REPO_DIR=${_DIR} echo "$_REPO_DIR" } # Params: ARG process_file_arg() { _KAS_FILES= _KAS_FIRST_FILE= _KAS_REPO_DIR= # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 for FILE in $(IFS=':'; echo $ARG); do if ! KAS_REAL_FILE="$(realpath -qe "$FILE")"; then fatal_error "configuration file '${FILE}' not found" fi if [ -z "${_KAS_FILES}" ]; then _KAS_FIRST_FILE="${KAS_REAL_FILE}" _KAS_FILES="${KAS_REAL_FILE}" _KAS_REPO_DIR=$(repo_path_of_file "${_KAS_FIRST_FILE}") else _KAS_FILES="${_KAS_FILES}:${KAS_REAL_FILE}" fi done KAS_FILES="${KAS_FILES} ${_KAS_FILES}" KAS_FIRST_FILES="${KAS_FIRST_FILES} ${_KAS_FIRST_FILE}" KAS_REPO_DIRS="${KAS_REPO_DIRS} ${_KAS_REPO_DIR}" } # Params: NAME CONTAINER_PATH MODE # If the dir is not below KAS_WORK_DIR, the dir is mounted into the container. forward_dir() { eval _varval=\"\$"$1"\" [ -z "$_varval" ] && return FW_DIR_REL=$(realpath -q --relative-base="${KAS_WORK_DIR}" "$_varval") if [ "${FW_DIR_REL}" = "$_varval" ]; then KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} -v ${FW_DIR_REL}:$2:$3 -e $1=$2" else KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} -e $1=/work/${FW_DIR_REL}" fi } check_docker_rootless() { KAS_DOCKER_ROOTLESS=0 if [ "$(docker context show)" = "rootless" ]; then KAS_DOCKER_ROOTLESS=1 fi } enable_docker_rootless() { warning "Rootless docker used, only limited functionality available." if [ "${KAS_WORK_DIR}" = "${KAS_REPO_DIR}" ]; then warning "On docker rootless a exclusive KAS_WORK_DIR should be used" \ "as kas temporarily changes the ownership of this directory." fi if [ "${KAS_REPO_MOUNT_OPT}" = "rw" ]; then fatal_error "Docker rootless requires read-only repo." fi KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} -e KAS_DOCKER_ROOTLESS=1" } KAS_GIT_OVERLAY_FILE="" kas_container_cleanup() { if [ -f "${KAS_GIT_OVERLAY_FILE}" ]; then trace rm -f "${KAS_GIT_OVERLAY_FILE}" fi } trap kas_container_cleanup EXIT INT TERM set_container_image_var() { # if the image is explicitly set, use that if [ -n "${KAS_CONTAINER_IMAGE}" ]; then return fi KAS_IMAGE_VERSION="${KAS_IMAGE_VERSION:-${KAS_IMAGE_VERSION_DEFAULT}}" KAS_CONTAINER_IMAGE_DISTRO="${KAS_CONTAINER_IMAGE_DISTRO:-${KAS_CONTAINER_IMAGE_DISTRO_DEFAULT}}" KAS_CONTAINER_IMAGE_NAME="${KAS_CONTAINER_IMAGE_NAME:-${KAS_CONTAINER_IMAGE_NAME_DEFAULT}}" KAS_CONTAINER_IMAGE_PATH="${KAS_CONTAINER_IMAGE_PATH:-${KAS_CONTAINER_IMAGE_PATH_DEFAULT}}" KAS_CONTAINER_IMAGE="${KAS_CONTAINER_IMAGE_PATH}/${KAS_CONTAINER_IMAGE_NAME}:${KAS_IMAGE_VERSION}" if [ -n "${KAS_CONTAINER_IMAGE_DISTRO}" ]; then KAS_CONTAINER_IMAGE="${KAS_CONTAINER_IMAGE}-${KAS_CONTAINER_IMAGE_DISTRO}" fi } # SC2034: DIR appears unused (ignore, as they are used inside eval) # shellcheck disable=2034 setup_kas_dirs() { KAS_WORK_DIR="${KAS_WORK_DIR:-$(pwd)}" KAS_WORK_DIR="$(check_and_expand KAS_WORK_DIR required)" KAS_BUILD_DIR="$(check_and_expand KAS_BUILD_DIR create)" KAS_REPO_REF_DIR="$(check_and_expand KAS_REPO_REF_DIR required)" DL_DIR="$(check_and_expand DL_DIR createrec)" SSTATE_DIR="$(check_and_expand SSTATE_DIR createrec)" KAS_BUILDTOOLS_DIR="$(check_and_expand KAS_BUILDTOOLS_DIR createrec)" } setup_kas_dirs KAS_CONTAINER_ENGINE="${KAS_CONTAINER_ENGINE:-${KAS_DOCKER_ENGINE}}" if [ -z "${KAS_CONTAINER_ENGINE}" ]; then # Try to auto-detect a container engine if command -v docker >/dev/null 2>&1 && docker -v 2>/dev/null | grep -q '^Docker'; then KAS_CONTAINER_ENGINE=docker elif command -v podman >/dev/null 2>&1; then KAS_CONTAINER_ENGINE=podman else fatal_error "no container engine found, need docker or podman" fi fi KAS_RUNTIME_ARGS="--log-driver=none --user=root" case "${KAS_CONTAINER_ENGINE}" in docker) KAS_CONTAINER_COMMAND="docker" enable_unpriv_userns_docker check_docker_rootless ;; podman) KAS_CONTAINER_COMMAND="podman" KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} --security-opt label=disable" ;; *) fatal_error "unknown container engine '${KAS_CONTAINER_ENGINE}'" ;; esac # parse kas-container options while [ $# -gt 0 ]; do case "$1" in --isar) enable_isar_mode shift 1 ;; --with-loop-dev) if ! KAS_LOOP_DEV=$(/sbin/losetup -f 2>/dev/null); then if [ "$(id -u)" -eq 0 ]; then fatal_error "loop device not available!" fi prepare_sudo_cmd if ! [ "$KAS_SUDO_CMD" = "sudo" ]; then fatal_error '--with-loop-dev requires sudo for device setup.' fi sudo_command="/sbin/losetup -f" sudo_message="[sudo] enter password to setup loop" sudo_message="$sudo_message devices by calling" sudo_message="$sudo_message '$sudo_command': " # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 if ! KAS_LOOP_DEV=$(sudo -p "$sudo_message" $sudo_command \ 2>/dev/null); then fatal_error "loop device setup unsuccessful!" \ "try calling '$sudo_command' with root" \ "permissions manually." fi fi KAS_WITH_LOOP_DEV="--device ${KAS_LOOP_DEV}" shift 1 ;; --runtime-args|--docker-args) [ $# -gt 0 ] || usage KAS_RUNTIME_ARGS="${KAS_RUNTIME_ARGS} $2" shift 2 ;; --ssh-dir) [ $# -gt 2 ] || usage KAS_SSH_DIR="$2" shift 2 ;; --ssh-agent) if [ -z "${SSH_AUTH_SOCK}" ]; then fatal_error "no SSH agent running" fi KAS_SSH_AUTH_SOCK=$(realpath -e "$SSH_AUTH_SOCK") shift 1 ;; --aws-dir) [ $# -gt 2 ] || usage KAS_AWS_DIR="$2" shift 2 ;; --git-credential-store) [ $# -gt 2 ] || usage KAS_GIT_CREDENTIAL_STORE="$2" shift 2 ;; --no-proxy-from-env) KAS_NO_PROXY_FROM_ENV=1 shift 1 ;; --repo-ro) KAS_REPO_MOUNT_OPT="ro" shift 1 ;; --repo-rw) KAS_REPO_MOUNT_OPT="rw" shift 1 ;; -l | --log-level) if [ "$2" = "debug" ]; then KAS_VERBOSE=1 fi KAS_OPTIONS_DIRECT="${KAS_OPTIONS_DIRECT} -l $2" shift 2 ;; --version) echo "${KAS_CONTAINER_SELF_NAME} $KAS_IMAGE_VERSION_DEFAULT" exit 0 ;; -h | --help) usage 0 ;; --*) usage ;; clean|cleansstate|cleanall|purge) KAS_REPO_MOUNT_OPT_DEFAULT="ro" KAS_CMD=$1 shift 1 break ;; shell|lock) KAS_REPO_MOUNT_OPT_DEFAULT="rw" KAS_CMD=$1 shift 1 break ;; build|checkout|for-all-repos|menu) KAS_REPO_MOUNT_OPT_DEFAULT="ro" KAS_CMD=$1 shift 1 break ;; diff) KAS_REPO_MOUNT_OPT_DEFAULT="ro" KAS_CMD=$1 shift 1 break ;; dump) if printf '%s\0' "$@" | grep -xqz -- '--inplace\|-i'; then KAS_REPO_MOUNT_OPT_DEFAULT="rw" else KAS_REPO_MOUNT_OPT_DEFAULT="ro" fi KAS_CMD=$1 shift 1 break ;; *) usage ;; esac done [ -n "${KAS_CMD}" ] || usage KAS_EXTRA_BITBAKE_ARGS=0 KAS_FILES= # parse kas sub-command options while [ $# -gt 0 ] && [ $KAS_EXTRA_BITBAKE_ARGS -eq 0 ]; do case "$1" in --format|--indent|--provenance|--skip|--target|--task) KAS_OPTIONS="${KAS_OPTIONS} $1 $2" shift 1 shift 1 || KAS_OPTIONS="--help" ;; -c|--cmd|--command) KAS_BITBAKE_C_OPTION_ARGS="$2" shift 1 shift 1 || KAS_OPTIONS="--help" ;; -E|--preserve-env) fatal_error "$1 is not supported with ${KAS_CONTAINER_SELF_NAME}" ;; --) KAS_EXTRA_BITBAKE_ARGS=$# ;; -*) KAS_OPTIONS="${KAS_OPTIONS} $1" shift 1 ;; *) ARG="$1" shift 1 if [ "$KAS_CMD" = "for-all-repos" ]; then if [ $# -gt 0 ]; then KAS_REPO_CMD="$1" shift 1 else KAS_REPO_CMD="$ARG" unset ARG fi fi process_file_arg "$ARG" ;; esac done if [ -n "${KAS_FIRST_FILES}" ]; then KAS_REPO_DIR=$(echo "${KAS_REPO_DIRS}" | awk '{print $1}') else KAS_REPO_DIR=$(pwd) fi SOURCE_DIR_HOST=$( grep -e "^_source_dir_host: " "${KAS_WORK_DIR}/.config.yaml" 2>/dev/null | \ sed 's/_source_dir_host:[ ]\+//') if [ -n "${SOURCE_DIR_HOST}" ]; then KAS_REPO_DIR="${SOURCE_DIR_HOST}" fi if [ "${KAS_CMD}" = "menu" ]; then if [ -z "${KAS_FIRST_FILES}" ]; then KAS_FIRST_FILES="Kconfig" fi # When using the menu plugin, we need to track the KAS_REPO_DIR outside # of the container to later allow a simple `kas-container build`. For # that, we tell the kas menu plugin via an env-var about the location # on the host. This data is then added to the .config.yaml where it can # be evaluated by the next invocation of kas-container. KAS_REPO_DIR=$(check_and_expand KAS_REPO_DIR required) if ! [ "${KAS_REPO_DIR}" = "${KAS_WORK_DIR}" ]; then set -- "$@" -e _KAS_REPO_DIR_HOST="${KAS_REPO_DIR}" fi if [ "$(echo "${KAS_FIRST_FILES}" | wc -w)" -ne "1" ]; then fatal_error "menu plugin only supports a single Kconfig file" fi BUILD_SYSTEM=$(tr '\n' '\f' 2>/dev/null < "${KAS_FIRST_FILES}" | \ sed -e 's/\(.*\fconfig KAS_BUILD_SYSTEM\f\(.*\)\|.*\)/\2/' \ -e 's/\f\([[:alpha:]].*\|$\)//' \ -e 's/.*default \"\(.*\)\".*/\1/') else if [ -z "${KAS_FIRST_FILES}" ]; then KAS_FIRST_FILES="${KAS_WORK_DIR}/.config.yaml" fi # We only get the first build system and let kas check if mixed _KAS_FIRST_FILE=$(echo "${KAS_FIRST_FILES}" | awk '{print $1}') BUILD_SYSTEM=$(grep -e "^build_system: " "${_KAS_FIRST_FILE}" 2>/dev/null | \ sed 's/build_system:[ ]\+//') fi if [ "${BUILD_SYSTEM}" = "isar" ]; then enable_isar_mode elif [ -z "${ISAR_MODE}" ]; then enable_oe_mode fi # clean can be executed without config, hence manually forward the build system if [ "${ISAR_MODE}" = "1" ] && echo "${KAS_CMD}" | grep -qe "^clean\|purge"; then KAS_OPTIONS="${KAS_OPTIONS} --isar" fi set_container_image_var if [ "${KAS_DOCKER_ROOTLESS}" = "1" ]; then KAS_REPO_MOUNT_OPT_DEFAULT="ro" fi KAS_REPO_MOUNT_OPT="${KAS_REPO_MOUNT_OPT:-${KAS_REPO_MOUNT_OPT_DEFAULT}}" if [ "$(id -u)" -eq 0 ] && [ "${KAS_ALLOW_ROOT}" != "yes" ] ; then fatal_error "Running as root - may break certain recipes." \ "Better give a regular user docker access. Set" \ "KAS_ALLOW_ROOT=yes to override." fi if [ "${KAS_DOCKER_ROOTLESS}" = "1" ]; then enable_docker_rootless fi if [ "${KAS_CMD}" = "diff" ]; then if [ "$(echo "${KAS_FILES}" | wc -w)" -eq "2" ]; then _KAS_REPO_DIR1="$(echo "${KAS_REPO_DIRS}" | awk '{print $1}')" _KAS_REPO_DIR2="$(echo "${KAS_REPO_DIRS}" | awk '{print $2}')" _KAS_FILES1="$(echo "${KAS_FILES}" | awk '{print $1}' | sed 's|'"${_KAS_REPO_DIR1}"'/|/repo/|g')" _KAS_FILES2="$(echo "${KAS_FILES}" | awk '{print $2}' | sed 's|'"${_KAS_REPO_DIR2}"'/|/repo2/|g')" KAS_FILES="${_KAS_FILES1} ${_KAS_FILES2}" set -- "$@" -v "${_KAS_REPO_DIR2}:/repo2:${KAS_REPO_MOUNT_OPT}" fi else KAS_FILES="$(echo "${KAS_FILES}" | sed 's|'"${KAS_REPO_DIR}"'/|/repo/|g')" fi set -- "$@" -v "${KAS_REPO_DIR}:/repo:${KAS_REPO_MOUNT_OPT}" \ -v "${KAS_WORK_DIR}":/work:rw -e KAS_WORK_DIR=/work \ --workdir=/repo \ -e KAS_CONTAINER_SCRIPT_VERSION="${KAS_CONTAINER_SCRIPT_VERSION}" \ -e USER_ID="$(id -u)" -e GROUP_ID="$(id -g)" --rm --init forward_dir KAS_BUILD_DIR "/build" "rw" forward_dir DL_DIR "/downloads" "rw" forward_dir KAS_REPO_REF_DIR "/repo-ref" "rw" forward_dir SSTATE_DIR "/sstate" "rw" forward_dir KAS_BUILDTOOLS_DIR "/buildtools" "rw" if git_com_dir=$(git -C "${KAS_REPO_DIR}" rev-parse --git-common-dir 2>/dev/null) \ && [ "$git_com_dir" != "$(git -C "${KAS_REPO_DIR}" rev-parse --git-dir)" ]; then # If (it's a git repo) and the common dir isn't the git-dir, it is shared worktree and # we have to mount the common dir in the container to make git work # The mount path inside the container is different from the host path. Hence, we over-mount # the .git file to point to the correct path. KAS_GIT_OVERLAY_FILE=$(mktemp) sed "s|gitdir: ${git_com_dir}/|gitdir: /repo-common/|" "${KAS_REPO_DIR}/.git" > "${KAS_GIT_OVERLAY_FILE}" set -- "$@" -v "${git_com_dir}:/repo-common:${KAS_REPO_MOUNT_OPT}" \ -v "${KAS_GIT_OVERLAY_FILE}:/repo/.git:ro" # if the workdir is the same as the repo dir, it is the same shared worktree if [ "${KAS_WORK_DIR}" = "${KAS_REPO_DIR}" ]; then set -- "$@" -v "${KAS_GIT_OVERLAY_FILE}:/work/.git:ro" fi fi KAS_SSH_DIR="$(check_and_expand KAS_SSH_DIR required)" if [ -n "${KAS_SSH_DIR}" ] ; then set -- "$@" -v "${KAS_SSH_DIR}":/var/kas/userdata/.ssh:ro fi if [ -n "${KAS_SSH_AUTH_SOCK}" ]; then if [ ! -S "${KAS_SSH_AUTH_SOCK}" ]; then fatal_error "passed SSH_AUTH_SOCK '${KAS_SSH_AUTH_SOCK}' is not a socket" fi set -- "$@" -v "${KAS_SSH_AUTH_SOCK}":/ssh-agent/ssh-auth-sock \ -e SSH_AUTH_SOCK=/ssh-agent/ssh-auth-sock fi KAS_AWS_DIR="$(check_and_expand KAS_AWS_DIR required)" if [ -n "${KAS_AWS_DIR}" ] ; then set -- "$@" -v "${KAS_AWS_DIR}":/var/kas/userdata/.aws:ro \ -e AWS_CONFIG_FILE="${AWS_CONFIG_FILE:-/var/kas/userdata/.aws/config}" \ -e AWS_SHARED_CREDENTIALS_FILE="${AWS_SHARED_CREDENTIALS_FILE:-/var/kas/userdata/.aws/credentials}" fi if [ -n "${AWS_WEB_IDENTITY_TOKEN_FILE}" ] ; then if [ ! -f "${AWS_WEB_IDENTITY_TOKEN_FILE}" ]; then echo "Passed AWS_WEB_IDENTITY_TOKEN_FILE '${AWS_WEB_IDENTITY_TOKEN_FILE}' is not a file" exit 1 fi set -- "$@" -v "$(realpath -e "${AWS_WEB_IDENTITY_TOKEN_FILE}")":/var/kas/userdata/.aws/web_identity_token:ro \ -e AWS_WEB_IDENTITY_TOKEN_FILE="${AWS_CONFIG_FILE:-/var/kas/userdata/.aws/web_identity_token}" \ -e AWS_ROLE_ARN="${AWS_ROLE_ARN}" fi KAS_GIT_CREDENTIAL_HELPER_DEFAULT="" if [ -n "${KAS_GIT_CREDENTIAL_STORE}" ] ; then if [ ! -f "${KAS_GIT_CREDENTIAL_STORE}" ]; then fatal_error "passed KAS_GIT_CREDENTIAL_STORE '${KAS_GIT_CREDENTIAL_STORE}' is not a file" fi KAS_GIT_CREDENTIAL_HELPER_DEFAULT="store --file=/var/kas/userdata/.git-credentials" set -- "$@" -v "$(realpath -e "${KAS_GIT_CREDENTIAL_STORE}")":/var/kas/userdata/.git-credentials:ro fi GIT_CREDENTIAL_HELPER="${GIT_CREDENTIAL_HELPER:-${KAS_GIT_CREDENTIAL_HELPER_DEFAULT}}" if [ -n "${GIT_CREDENTIAL_HELPER}" ] ; then set -- "$@" -e GIT_CREDENTIAL_HELPER="${GIT_CREDENTIAL_HELPER}" fi if [ -f "${NETRC_FILE}" ]; then set -- "$@" -v "$(realpath -e "${NETRC_FILE}")":/var/kas/userdata/.netrc:ro \ -e NETRC_FILE="/var/kas/userdata/.netrc" fi if [ -f "${NPMRC_FILE}" ]; then set -- "$@" -v "$(realpath -e "${NPMRC_FILE}")":/var/kas/userdata/.npmrc:ro \ -e NPMRC_FILE="/var/kas/userdata/.npmrc" fi if [ -f "${GITCONFIG_FILE}" ]; then set -- "$@" -v "$(realpath -e "${GITCONFIG_FILE}")":/var/kas/userdata/.gitconfig:ro \ -e GITCONFIG_FILE="/var/kas/userdata/.gitconfig" fi if [ -f "${REGISTRY_AUTH_FILE}" ]; then set -- "$@" -v "$(realpath -e "${REGISTRY_AUTH_FILE}")":/var/kas/userdata/.docker/config.json:ro \ -e REGISTRY_AUTH_FILE="/var/kas/userdata/.docker/config.json" fi if [ -t 1 ]; then set -- "$@" -t -i fi if [ -n "${SSTATE_MIRRORS}" ]; then if echo "${SSTATE_MIRRORS}" | grep -q "file:///"; then warning "SSTATE_MIRRORS contains a local path." \ "Make sure to make this path available inside the container." fi set -- "$@" -e "SSTATE_MIRRORS=${SSTATE_MIRRORS}" fi # propagate timezone information to entrypoint (requires systemd 239) if command -v timedatectl >/dev/null; then set -- "$@" -e "KAS_HOST_TZ=$(timedatectl show -p Timezone --value 2>/dev/null)" fi for var in TERM KAS_DISTRO KAS_MACHINE KAS_TARGET KAS_TASK KAS_CLONE_DEPTH \ KAS_PREMIRRORS DISTRO_APT_PREMIRRORS BB_NUMBER_THREADS PARALLEL_MAKE \ GIT_CREDENTIAL_USEHTTPPATH \ TZ; do if [ -n "$(eval echo \$${var})" ]; then set -- "$@" -e "${var}=$(eval echo \"\$${var}\")" fi done # propagate only supported SHELL settings case "$SHELL" in /bin/sh|/bin/bash|/bin/dash) set -- "$@" -e "SHELL=$SHELL" ;; *) set -- "$@" -e "SHELL=/bin/bash" ;; esac if [ -z "${KAS_NO_PROXY_FROM_ENV+x}" ]; then for var in http_proxy https_proxy ftp_proxy no_proxy NO_PROXY; do if [ -n "$(eval echo \$${var})" ]; then set -- "$@" -e "${var}=$(eval echo \$${var})" fi done fi # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 set -- "$@" ${KAS_ISAR_ARGS} ${KAS_WITH_LOOP_DEV} ${KAS_RUNTIME_ARGS} \ ${KAS_CONTAINER_IMAGE} ${KAS_OPTIONS_DIRECT} ${KAS_CMD} ${KAS_OPTIONS} if [ -n "${KAS_BITBAKE_C_OPTION_ARGS}" ]; then set -- "$@" -c "${KAS_BITBAKE_C_OPTION_ARGS}" fi # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=2086 set -- "$@" ${KAS_FILES} if [ "$KAS_CMD" = "for-all-repos" ]; then set -- "$@" "${KAS_REPO_CMD}" fi # rotate any extra bitbake args from the front to the end of the argument list while [ $KAS_EXTRA_BITBAKE_ARGS -gt 0 ]; do arg="$1" shift 1 set -- "$@" "$arg" KAS_EXTRA_BITBAKE_ARGS=$((KAS_EXTRA_BITBAKE_ARGS - 1)) done # shellcheck disable=SC2086 trace ${KAS_CONTAINER_COMMAND} run "$@" siemens-kas-41ad961/kas-docker000077700000000000000000000000001520561422700207122kas-containerustar00rootroot00000000000000siemens-kas-41ad961/kas/000077500000000000000000000000001520561422700150465ustar00rootroot00000000000000siemens-kas-41ad961/kas/__init__.py000066400000000000000000000030461520561422700171620ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ kas - setup tool for bitbake based projects """ from .__version__ import __version__ from .configschema import CONFIGSCHEMA, \ __file_version__, __compatible_file_version__, \ __schema_definition__ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' __all__ = ['__version__', '__file_version__', '__compatible_file_version__', '__schema_definition__', 'CONFIGSCHEMA'] siemens-kas-41ad961/kas/__main__.py000066400000000000000000000024621520561422700171440ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ The main entry point of kas, a setup tool for bitbake based projects """ from .kas import main __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' main() siemens-kas-41ad961/kas/__version__.py000066400000000000000000000024131520561422700177010ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains the version of kas. """ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2020' __version__ = '5.3' siemens-kas-41ad961/kas/attestation.py000066400000000000000000000205371520561422700177660ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module provides infrastructure to generate provenance attestation of the build process. """ import os import distro import logging import hashlib import base64 import sys from enum import Enum from urllib.parse import urlparse, urlunparse from pathlib import Path from datetime import datetime, timezone from kas import __version__ as KASVERSION SLSA_PROVENANCE_TYPE = 'https://slsa.dev/provenance/v1' KAS_BUILDER_ID = 'https://github.com/siemens/kas' KAS_BUILD_TYPE = 'https://kas.readthedocs.io/en' \ f'/{KASVERSION}/userguide/project-configuration.html' INTOTO_STATEMENT_TYPE = 'https://in-toto.io/Statement/v1' def date_to_rfc3339(dt): return dt.astimezone(timezone.utc) \ .strftime('%Y-%m-%dT%H:%M:%S.%fZ') def file_digest_slow(f, algorithm): """ Implementation of hashlib.file_digest for python < 3.11 """ hash = hashlib.new(algorithm) while True: data = f.read() if not data: break hash.update(data) return hash class Provenance: """ Create a build attestation predicate in SLSA provenance format for a kas build. """ class Mode(Enum): MIN = 1 MAX = 10 def __init__(self, ctx, t_started, t_finished, mode=Mode.MIN): self._ctx = ctx self._t_started = t_started self._t_finished = t_finished self._mode = mode @staticmethod def _url_with_protocol(url): if url.startswith('git@'): return f'ssh://{url}' if url.startswith('ssh://'): return f'{url}' if url.startswith('http://') or url.startswith('https://'): return f'{url}' @staticmethod def _strip_credentials(url): """ Returns the url with all credentials removed """ try: parsed = urlparse(url) except ValueError: return url netloc = parsed.hostname or '' if parsed.port: netloc += f':{parsed.port}' safe_url = (parsed.scheme, netloc, parsed.path, parsed.params, parsed.query, parsed.fragment) return urlunparse(safe_url) @staticmethod def _get_filetype(f: Path): if f.suffix == '.json': return 'json' return 'yaml' def _make_relative_path(self, path: Path): top_repo_path = Path(self._ctx.config.handler.get_top_repo_path()) workdir = Path(self._ctx.kas_work_dir) if path.is_relative_to(workdir): return path.relative_to(workdir) else: return path.relative_to(top_repo_path) def type_(self): return SLSA_PROVENANCE_TYPE def as_dict(self): res_deps = [] tracked_repos = [] for r in self._ctx.config.get_repos(): if r.operations_disabled: if not r.url or not r.revision: continue digest = {f'{r.get_type()}Commit': r.revision} annotations = { 'dirty': r.dirty, 'layers': [str(layer.path.relative_to(r.path)) for layer in r.layers] } cleanurl = self._strip_credentials(r.url) dep = { 'name': r.name, 'uri': f'{r.get_type()}+{self._url_with_protocol(cleanurl)}', 'digest': digest, 'annotations': annotations } res_deps.append(dep) tracked_repos.append(r) # (abspath, relpath) config_files = [(Path(c), self._make_relative_path(Path(c))) for c in self._ctx.config.filenames] for ca, cr in config_files: if any([r.contains_path(cr) for r in tracked_repos]): logging.debug(f'Config file {cr} is tracked') continue with open(ca, 'rb') as f: content = f.read() rd = { 'name': str(cr), 'content': base64.b64encode(content).decode('utf-8'), 'mediaType': f'application/vnd.kas+{self._get_filetype(ca)}' } res_deps.append(rd) bd = { 'buildType': KAS_BUILD_TYPE, 'externalParameters': { 'command': self._ctx.args.cmd, 'config': [str(c) for _, c in config_files], 'target': self._ctx.args.target, 'task': self._ctx.args.task, 'extra_bitbake_args': self._ctx.args.extra_bitbake_args, }, 'internalParameters': {}, 'resolvedDependencies': res_deps } if self._mode == self.Mode.MAX: bd['internalParameters']['env'] = \ self._ctx.config.get_environment() b_versions = { 'kas': KASVERSION, 'distro.name': distro.id(), 'distro.version': distro.version() } rd = { 'builder': { 'id': KAS_BUILDER_ID, 'version': b_versions }, 'metadata': { 'invocationId': os.environ.get('CI_JOB_URL', ''), 'startedOn': date_to_rfc3339(self._t_started), 'finishedOn': date_to_rfc3339(self._t_finished), }, 'byproducts': [] } p = { 'buildDefinition': bd, 'runDetails': rd } return p class Statement: """ Create a statement in in-toto format for a kas build. """ def __init__(self, predicate, ctx, t_started, t_finished): self._predicate = predicate self._ctx = ctx self._t_started = t_started self._t_finished = t_finished def _check_artifact_timestamp(self, name, path): """ Warn if artifact timestamp is not within the build range. """ logging.debug(f'Found artifact {name}:{path} in build dir') fullpath = Path(self._ctx.build_dir) / path mtime = datetime.fromtimestamp(fullpath.stat().st_mtime) if mtime < self._t_started or mtime > self._t_finished: logging.warning( f'Artifact {name}:{path.name} mtime {mtime.strftime("%c")}' f' not in build range ' f'[{self._t_started.strftime("%c")} - ' f'{self._t_finished.strftime("%c")}]') def as_dict(self): pt = self._predicate.type_() pp = self._predicate.as_dict() subjects = [] for n, s in self._ctx.config.get_artifacts(missing_ok=False): self._check_artifact_timestamp(n, s) fullpath = Path(self._ctx.build_dir) / s with open(fullpath, "rb") as f: if sys.version_info < (3, 11): digest = file_digest_slow(f, "sha256") else: digest = hashlib.file_digest(f, "sha256") rd = { 'name': s.name, 'digest': {'sha256': digest.hexdigest()} } subjects.append(rd) if len(subjects) == 0: logging.warning('Attestation does not contain any artifacts.') st = { '_type': INTOTO_STATEMENT_TYPE, 'subject': subjects, 'predicateType': pt, 'predicate': pp } return st siemens-kas-41ad961/kas/config.py000066400000000000000000000230171520561422700166700ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2021 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains the implementation of the kas configuration. """ import os import json import copy from pathlib import Path from .repos import Repo from .includehandler import IncludeHandler from .kasusererror import ArtifactNotFoundError from .configschema import CONFIGSCHEMA __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2021' CONFIG_YAML_FILE = '.config.yaml' class Config: """ Implements the kas configuration based on config files. """ def __init__(self, ctx, filename, target=None, task=None): self._override_target = target self._override_task = task self._build_dir = ctx.build_dir self.__config = {} if not filename: filename = os.path.join(ctx.kas_work_dir, CONFIG_YAML_FILE) self.filenames = [os.path.abspath(configfile) for configfile in filename.split(':')] update = ctx.args.update if hasattr(ctx.args, 'update') else False self.handler = IncludeHandler(self.filenames, not update) self.repo_dict = {} self.repo_cfg_hashes = {} @property def _config(self): if not self.__config: raise RuntimeError("Config has not been imported yet.") return self.__config def get_build_system(self): """ Returns the pre-selected build system """ return self._config.get('build_system', '') def find_missing_repos(self, repo_paths={}): """ Returns repos that are in config but not on disk and updates the internal config dictionary. """ (self.__config, missing_repo_names) = \ self.handler.get_config(repos=repo_paths) return missing_repo_names def get_config(self, remove_includes=False, apply_overrides=False): """ Returns a copy of the config dict """ config = copy.deepcopy(self._config) if remove_includes and 'includes' in config['header']: del config['header']['includes'] if apply_overrides: overrides = config.get('overrides', {}) if overrides: config.update(overrides) del config['overrides'] return config def get_lockfiles(self): """ Returns the list of in-use lockfiles. """ return self.handler.get_lockfiles() def get_repos_config(self): """ Returns the repository configuration """ return self._config.get('repos', {}) def get_repos(self): """ Returns the list of repos. """ # Always keep repo_dict and repos synchronous # when calling get_repos self.repo_dict = self._get_repo_dict() return list(self.repo_dict.values()) def get_repo(self, name): """ Returns a `Repo` instance for the configuration with the key `name`. """ repo_defaults = self._config.get('defaults', {}).get('repos', {}) overrides = self._config.get('overrides', {}) \ .get('repos', {}).get(name, {}) config = self.get_repos_config()[name] or {} top_repo_path = self.handler.get_top_repo_path() # Check if we have this repo with an identical config already. # As this function is called across various places and with different # configurations (e.g. due to updates from transitive includes), # we cache the results. args = (name, config, repo_defaults, top_repo_path, overrides) return self._get_or_create_repo(args) def _get_or_create_repo(self, args): """ Get a repo from the cache and insert it if not existing. Creating repos is expensive due to external commands being called. """ encoded = json.dumps(args, sort_keys=True).encode() if encoded in self.repo_cfg_hashes: return self.repo_cfg_hashes[encoded] repo = Repo.factory(*args) self.repo_cfg_hashes[encoded] = repo return repo def _get_repo_dict(self): """ Returns a dictionary containing the repositories with their names (as it is defined in the config file) as keys and the `Repo` instances as values. """ return {name: self.get_repo(name) for name in self.get_repos_config()} def get_bitbake_targets(self): """ Returns a list of bitbake targets """ if self._override_target: return self._override_target environ_targets = [i for i in os.environ.get('KAS_TARGET', '').split() if i] if environ_targets: return environ_targets def_target = CONFIGSCHEMA['properties']['target']['default'] target = self._config.get('target', def_target) if isinstance(target, str): return [target] return target def get_bitbake_task(self): """ Returns the bitbake task """ if self._override_task: return self._override_task default = CONFIGSCHEMA['properties']['task']['default'] return os.environ.get('KAS_TASK', self._config.get('task', default)) def _get_conf_header(self, header_name): """ Returns the local.conf header """ header = '' for key, value in sorted(self._config.get(header_name, {}).items()): header += f'# {key}\n{value}\n' return header def get_bblayers_conf_header(self): """ Returns the bblayers.conf header """ return self._get_conf_header('bblayers_conf_header') def get_local_conf_header(self): """ Returns the local.conf header """ return self._get_conf_header('local_conf_header') def get_machine(self): """ Returns the machine """ default = CONFIGSCHEMA['properties']['machine']['default'] return os.environ.get('KAS_MACHINE', self._config.get('machine', default)) def get_distro(self): """ Returns the distro """ default = CONFIGSCHEMA['properties']['distro']['default'] return os.environ.get('KAS_DISTRO', self._config.get('distro', default)) def get_environment(self): """ Returns the configured environment variables from the configuration file with possible overwritten values from the environment. """ env = self._config.get('env', {}) return {var: os.environ.get(var, env[var]) for var in env} def get_multiconfig(self): """ Returns the multiconfig array as bitbake string """ multiconfigs = set() for target in self.get_bitbake_targets(): if target.startswith('multiconfig:') or target.startswith('mc:'): multiconfigs.add(target.split(':')[1]) return ' '.join(multiconfigs) def get_artifacts(self, missing_ok=True): """ Returns the found artifacts after glob expansion, relative to the build_dir as a list of tuples (name, path). If missing_ok=False, raises an ArtifactNotFoundError if no artifact for a given name is found. """ arts = self._config.get('artifacts', {}) foundfiles = [] for name, art in arts.items(): files = list(Path(self._build_dir).glob(art)) if not missing_ok and len(files) == 0: raise ArtifactNotFoundError(name, art) foundfiles.extend([(name, f) for f in files]) return [(n, f.relative_to(self._build_dir)) for n, f in foundfiles] def get_signers_config(self, keytype=None): """ Returns the keys from the configuration """ signers = self._config.get('signers', {}) if not keytype: return signers else: return {k: v for k, v in signers.items() if v.get('type', 'gpg') == keytype} def get_buildtools(self): """ Returns the buildtools keys: version, download URL and archive filename. These are provided so kas knows which buildtools archive to fetch and from what source. """ return self._config.get('buildtools') siemens-kas-41ad961/kas/configschema.py000066400000000000000000000036031520561422700200500ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the 'Software'), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ''' This module contains the schema of the configuration file. ''' __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2024' import json import os def _load_schema(): global CONFIGSCHEMA global __file_version__ global __compatible_file_version__ global __schema_definition__ cwd = os.path.dirname(os.path.realpath(__file__)) __schema_definition__ = os.path.join(cwd, 'schema-kas.json') with open(__schema_definition__, 'r') as f: CONFIGSCHEMA = json.load(f) header_node = CONFIGSCHEMA['properties']['header'] version_node = header_node['properties']['version']['anyOf'][1] __file_version__ = version_node["maximum"] __compatible_file_version__ = version_node["minimum"] _load_schema() siemens-kas-41ad961/kas/context.py000066400000000000000000000150061520561422700171060ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains the implementation of the kas context. """ import distro import os import logging from enum import Enum from kas.kasusererror import KasUserError from kas import __version__ __context__ = None def get_distro_id_base(): """ Returns a compatible distro id. """ return distro.like() or distro.id() def create_global_context(args): """ Creates global context as singleton. """ global __context__ __context__ = Context(args) return __context__ def get_context(): """ Returns singleton global context. """ return __context__ class ManagedEnvironment(Enum): """ Managed environments are well-known executors (like CI systems) that kas can detect and adapt to. """ GITHUB_ACTIONS = 1 GITLAB_CI = 2 VSCODE_REMOTE_CONTAINERS = 3 def __str__(self): if self == self.GITHUB_ACTIONS: return 'GitHub Actions' if self == self.GITLAB_CI: return 'GitLab CI' if self == self.VSCODE_REMOTE_CONTAINERS: return 'VSCode Remote Containers' return f'{self.name}' class Context: """ Implements the kas build context. """ def __init__(self, args): work_dir = os.environ.get('KAS_WORK_DIR', os.getcwd()) self.__kas_work_dir = os.path.abspath(work_dir) build_dir = os.environ.get('KAS_BUILD_DIR', os.path.join(self.__kas_work_dir, 'build')) self.__kas_build_dir = os.path.abspath(build_dir) ref_dir = os.environ.get('KAS_REPO_REF_DIR', None) self.__kas_repo_ref_dir = os.path.abspath(ref_dir) if ref_dir else None clone_depth = os.environ.get('KAS_CLONE_DEPTH', '0') if not clone_depth.isdigit(): raise KasUserError('KAS_CLONE_DEPTH must be a number') self.repo_clone_depth = max(int(clone_depth), 0) self.setup_initial_environ() self.check_container_call() # Register the paths that kas created and exclusively owns self.managed_paths = set() if not os.environ.get('KAS_BUILD_DIR'): self.managed_paths.add(self.__kas_build_dir) self.keyhandler = {} self.config = None self.args = args def setup_initial_environ(self): """ Sets the environment variables for processes that are started by kas. """ self.environ = {} distro_bases = get_distro_id_base().lower().split() for distro_base in distro_bases: if distro_base in ['fedora', 'suse', 'opensuse']: self.environ = {'LC_ALL': 'en_US.utf8', 'LANG': 'en_US.utf8', 'LANGUAGE': 'en_US'} break elif distro_base in ['debian', 'ubuntu', 'gentoo', 'arch']: self.environ = {'LC_ALL': 'en_US.UTF-8', 'LANG': 'en_US.UTF-8', 'LANGUAGE': 'en_US:en'} break if self.environ == {}: logging.warning('kas: No supported distros found in %s. ' 'No default locales set.', distro_bases) for key in ['http_proxy', 'https_proxy', 'ftp_proxy', 'no_proxy', 'SSH_AUTH_SOCK', 'BB_NUMBER_THREADS', 'PARALLEL_MAKE']: val = os.environ.get(key, None) if val: self.environ[key] = val # make remote containers environment available in kas if self.managed_env == ManagedEnvironment.VSCODE_REMOTE_CONTAINERS: for k in os.environ.keys(): if k.startswith('REMOTE_CONTAINERS_'): self.environ[k] = os.environ[k] @staticmethod def check_container_call(): container_v = os.environ.get('KAS_CONTAINER_SCRIPT_VERSION') if not container_v: # not a kas-container call (or from a too old script) return if container_v != __version__: logging.warning(f'kas-container ({container_v}) and ' f'kas ({__version__}) versions do not match') @staticmethod def _get_managed_env(): """ Detects if kas is running in well-known environment (e.g. a CI system). Returns the identifier of the CI system or None. """ if os.environ.get('GITHUB_ACTIONS', False) == 'true': return ManagedEnvironment.GITHUB_ACTIONS if os.environ.get('GITLAB_CI', False) == 'true': return ManagedEnvironment.GITLAB_CI if os.environ.get('REMOTE_CONTAINERS', False) == 'true': return ManagedEnvironment.VSCODE_REMOTE_CONTAINERS return None @property def build_dir(self): """ The path to the build directory """ return self.__kas_build_dir @property def kas_work_dir(self): """ The path to the kas work directory """ return self.__kas_work_dir @property def kas_repo_ref_dir(self): """ The reference directory for the repo """ return self.__kas_repo_ref_dir @property def force_checkout(self): return getattr(self.args, 'force_checkout', None) @property def update(self): return getattr(self.args, 'update', None) @property def managed_env(self): return self._get_managed_env() siemens-kas-41ad961/kas/includehandler.py000066400000000000000000000405511520561422700204060ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2021 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module implements how includes of configuration files are handled in kas. """ import os from pathlib import Path from collections import OrderedDict from collections.abc import Mapping from functools import cached_property import functools import logging import json import yaml from jsonschema.validators import validator_for from .kasusererror import KasUserError from .repos import Repo from . import __file_version__, __compatible_file_version__, __version__ from . import CONFIGSCHEMA __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2021' SOURCE_DIR_OVERRIDE_KEY = '_source_dir' SOURCE_DIR_HOST_OVERRIDE_KEY = '_source_dir_host' PROJECT_CONFIG_URL = f'https://kas.readthedocs.io/en/{__version__}/' \ 'userguide/project-configuration.html' class LoadConfigException(KasUserError): """ Class for exceptions that appear while loading a configuration file. """ def __init__(self, message, filename): super().__init__(f'{message}: {filename}') class IncludeException(KasUserError): """ Class for exceptions that appear in the include mechanism. """ pass class ConfigFile(): def __init__(self, filename, is_external, is_lockfile): self.filename = Path(filename) self.config = {} # src_dir must only be set by auto-generated config file self.src_dir = None self.is_external = is_external self.is_lockfile = is_lockfile @staticmethod def load(filename, is_external=False, is_lockfile=False, is_main_file=True): """ Load the configuration file and test if version is supported. """ cf = ConfigFile(filename, is_external, is_lockfile) (_, ext) = os.path.splitext(filename) if ext == '.json': with open(filename, 'rb') as fds: cf.config = json.load(fds) elif ext in ['.yml', '.yaml']: try: with open(filename, 'rb') as fds: cf.config = yaml.safe_load(fds) except yaml.YAMLError as e: msg = f'Error in line {e.problem_mark.line + 1}' \ if hasattr(e, 'problem_mark') else '' raise LoadConfigException( f'Configuration file is not valid YAML: {msg}', filename) else: raise LoadConfigException('Config file extension not recognized', filename) validator_class = validator_for(CONFIGSCHEMA) validator = validator_class(CONFIGSCHEMA) validation_error = False for error in sorted(validator.iter_errors(cf.config), key=str): validation_error = True logging.error('Config file validation Error:\n%s', error.message) logging.error('For a list of supported configuration elements, ' 'see %s', PROJECT_CONFIG_URL) logging.debug('Validation against this schema failed:\n%s', json.dumps(error.schema, indent=2)) if validation_error: raise LoadConfigException('Error(s) occured while validating the ' 'config file', filename) try: version_value = int(cf.config['header']['version']) except ValueError: # Be compatible: version string '0.10' is equivalent to file # version 1 This check is already done in the config schema so # here just set the right version version_value = 1 if version_value < __compatible_file_version__ or \ version_value > __file_version__: raise LoadConfigException('This version of kas is compatible with ' f'version {__compatible_file_version__} ' f'to {__file_version__}, ' f'file has version {version_value}', filename) if cf.config.get('proxy_config'): logging.warning('Obsolete ''proxy_config'' detected. ' 'This has no effect and will be rejected soon.') cf.src_dir = cf.config.get(SOURCE_DIR_OVERRIDE_KEY, None) if cf.src_dir is not None and not is_main_file: raise LoadConfigException( f'{SOURCE_DIR_OVERRIDE_KEY!r} is only allowed in the ' 'top-level kas configuration', filename) return cf class IncludeHandler: """ Implements a handler where every configuration file should contain a dictionary as the base type with and 'includes' key containing a list of includes. The includes can be specified in two ways: as a string containing the path, relative to the repository root from the current file, or as a dictionary. The dictionary must have a 'file' key containing the path to the include file and a 'repo' key containing the key of the repository. The path is interpreted relative to the repository root path, which is lazy resolved by the first access of a method. The includes are read and merged from the deepest level upwards. In case ``use_lock`` is ``True``, kas checks if a file ``.lock.`` exists next to the first entry in ``top_files``. This filename is then appended to the list of ``top_files``. """ def __init__(self, top_files, use_lock=True): self.top_files = top_files self.use_lock = use_lock self.config_files = [] def get_lock_filename(self, kasfile=None): """ Returns the lockfile name for the given kas config file. """ file = Path(kasfile or self.top_files[0]) return file.parent / (file.stem + '.lock' + file.suffix) @cached_property def top_repo_path(self): """ Lazy resolve top repo path as we might need a prepared environment """ return Repo.get_root_path(os.path.dirname(self.top_files[0])) def get_lockfiles(self): """ Returns a list of lockfiles in the order the configuration files were parsed. """ return list(filter(lambda x: x.is_lockfile, self.config_files)) def get_top_repo_path(self): return self.top_repo_path def ensure_from_same_repo(self): """ Ensure that all concatenated config files belong to the same repository """ repo_paths = [Repo.get_root_path(os.path.dirname(configfile), fallback=False) for configfile in self.top_files] if len(set(repo_paths)) > 1: raise IncludeException('All concatenated config files must ' 'belong to the same repository or all ' 'must be outside of versioning control') def sanitize_include_path(self, base_path, include): """ Ensure the created path is with the base_path. Returns the resolved path of base_path / include. """ repo = Path(base_path) candidate = (repo / Path(include)).resolve() if not candidate.is_relative_to(repo.resolve()): raise IncludeException( f'include {candidate} resolves outside repository {repo}') return str(candidate) def get_config(self, repos=None): """ Parameters: repos -- A dictionary that maps repo names to directory paths Returns: (config, repos) config -- A dictionary containing the configuration repos -- A list of missing repo names that are needed \ to create a complete configuration """ repos = repos or {} def _internal_include_handler(filename, repo_path, is_external=False, is_lockfile=False, is_main_file=False): """ Recursively loads include files and finds missing repos. Includes are done in the following way: topfile.yml: ------- header: includes: - include1.yml - repo: repo1 file: include-repo1.yml - repo: repo2 file: include-repo2.yml - include3.yml ------- Includes are merged in in this order: ['include1.yml', 'include2.yml', 'include-repo1.yml', 'include-repo2.yml', 'include-repo2.yml', 'topfile.yml'] On conflict the latter includes overwrite previous ones and the current file overwrites every include. (evaluation depth first and from top to bottom) """ missing_repos = [] configs = [] try: current_config = \ ConfigFile.load(filename, is_external, is_lockfile, is_main_file) # if lockfile exists, inject it after current file lockfile = self.get_lock_filename(filename) if Path(lockfile).exists(): (cfg, rep) = _internal_include_handler( lockfile, repo_path, is_external=is_external, is_lockfile=True ) configs.extend(cfg) missing_repos.extend(rep) # src_dir must only be set by auto-generated config file if current_config.src_dir: self.top_repo_path = current_config.src_dir repo_path = current_config.src_dir except FileNotFoundError: raise LoadConfigException('Configuration file not found', filename) if not isinstance(current_config.config, Mapping): raise IncludeException('Configuration file does not contain a ' 'dictionary as base type') header = current_config.config.get('header', {}) for include in header.get('includes', []): if isinstance(include, str): includefile = self.sanitize_include_path( repo_path, include) if not os.path.exists(includefile): alternate = self.sanitize_include_path( os.path.dirname(current_config.filename), include) if os.path.exists(alternate): logging.warning( 'Falling back to file-relative addressing ' 'of local include "%s"', include) logging.warning( 'Update your layer to repo-relative ' 'addressing to avoid this warning') includefile = alternate (cfg, rep) = _internal_include_handler( includefile, repo_path, is_external=is_external ) configs.extend(cfg) missing_repos.extend(rep) elif isinstance(include, Mapping): includerepo = include.get('repo', None) includedir = repos.get(includerepo, None) if includedir is not None: incexternal = bool(includedir != self.top_repo_path) try: includefile = include['file'] except KeyError: raise IncludeException( f'"file" is not specified: {include}') abs_includedir = Path(includedir).absolute() (cfg, rep) = _internal_include_handler( self.sanitize_include_path(abs_includedir, includefile), abs_includedir, is_external=incexternal) configs.extend(cfg) missing_repos.extend(rep) else: missing_repos.append(includerepo) logging.debug('config file %s (%s)', current_config.filename, 'external' if is_external else 'internal') configs.append(current_config) # Remove all possible duplicates in missing_repos missing_repos = list(OrderedDict.fromkeys(missing_repos)) return (configs, missing_repos) def _internal_dict_merge(dest, upd): """ Merges upd recursively into a copy of dest. The order is preserved as in the original dict as dict-insertion orders are preserved from Python 3.6 onwards. If keys in upd intersect with keys in dest we will do a manual merge (helpful for non-dict types like FunctionWrapper). """ if (not isinstance(dest, Mapping)) \ or (not isinstance(upd, Mapping)): raise IncludeException('Cannot merge using non-dict') dest = dest.copy() updkeys = list(upd.keys()) if set(list(dest.keys())) & set(updkeys): for key in updkeys: val = upd[key] try: dest_subkey = dest.get(key, None) except AttributeError: dest_subkey = None if isinstance(dest_subkey, Mapping) \ and isinstance(val, Mapping): ret = _internal_dict_merge(dest_subkey, val) dest[key] = ret else: dest[key] = upd[key] return dest try: for k in upd: dest[k] = upd[k] except AttributeError: # this mapping is not a dict for k in upd: dest[k] = upd[k] return dest self.config_files = [] missing_repos = [] self.ensure_from_same_repo() for idx, configfile in enumerate(self.top_files): cfgs, reps = _internal_include_handler(configfile, self.get_top_repo_path(), is_main_file=(idx == 0)) self.config_files.extend(cfgs) for repo in reps: if repo not in missing_repos: missing_repos.append(repo) config_files = self.config_files if not self.use_lock: config_files = [x for x in config_files if not x.is_lockfile] config = functools.reduce(_internal_dict_merge, map(lambda x: x.config, config_files)) # the merged config must have the highest (used) version number header_version = max([int(cfg.config['header']['version']) for cfg in config_files]) config['header']['version'] = header_version return config, missing_repos siemens-kas-41ad961/kas/kas.py000066400000000000000000000200451520561422700161770ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module is the main entry point for kas, setup tool for bitbake based projects. In case of user errors (e.g. invalid configuration, repo fetch failure) kas exits with error code 2, while exiting with 1 for internal errors. When cancelled by SIGINT, kas exits with 130. For details on error handling, see :mod:`kas.kasusererror`. """ import argparse import asyncio import distro import traceback import logging import signal import sys import os from .kasusererror import KasUserError, CommandExecError try: import colorlog HAVE_COLORLOG = True except ImportError: HAVE_COLORLOG = False from . import __version__, __file_version__, __compatible_file_version__ from . import plugins __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' DEFAULT_LOG_LEVEL = 'info' def create_logger(): """ Setup the logging environment """ log = logging.getLogger() # root logger set_global_loglevel(DEFAULT_LOG_LEVEL.upper()) format_str = '%(asctime)s - %(levelname)-8s - %(message)s' date_format = '%Y-%m-%d %H:%M:%S' if HAVE_COLORLOG and os.isatty(2): cformat = '%(log_color)s' + format_str colors = {'DEBUG': 'reset', 'INFO': 'reset', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red', 'CRITICAL': 'bold_red'} formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors) else: formatter = logging.Formatter(format_str, date_format) stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) log.addHandler(stream_handler) return logging.getLogger(__name__) def cleanup_logger(): """ Cleanup the logging environment """ for handler in logging.root.handlers[:]: if isinstance(handler, logging.StreamHandler): logging.root.removeHandler(handler) def set_global_loglevel(level): """ Configure the global log level. Implemented as function to monkey-patch it out in tests. """ logging.getLogger().setLevel(level) def register_signal_handlers(loop): """ Register the signal handlers which should be handled by the event loop. Implemented as function to monkey-patch it out in tests. """ loop.add_signal_handler(signal.SIGTERM, interruption) loop.add_signal_handler(signal.SIGINT, interruption) def interruption(): """ Gracefully cancel all tasks in the event loop """ loop = asyncio.get_event_loop() for sig in [signal.SIGINT, signal.SIGTERM]: loop.remove_signal_handler(sig) loop.add_signal_handler(sig, termination) pending = asyncio.all_tasks(loop) if pending: logging.debug(f'waiting for {len(pending)} tasks to terminate') [t.cancel() for t in pending] def termination(): """ Forcefully terminate the process """ logging.error('kas terminated forcefully') os._exit(130) def shutdown_loop(loop): """ Waits for completion of the event loop but ignores any exceptions. The tasks are either already cancelled or will be transitively cancelled shortly. As this is the final cleanup, we cannot check for exceptions as these might lead in an unclosed event loops. """ pending = asyncio.all_tasks(loop) if len(pending): logging.debug("Cleanup %d remaining tasks", len(pending)) loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) loop.close() def kas_get_argparser(): """ Creates an argparser for kas with all plugins. """ # Load plugins here so that the commands and arguments introduced by the # plugins can be seen by sphinx when it calls this function to build the # documentation plugins.load() parser = argparse.ArgumentParser(description='kas - setup tool for ' 'bitbake based project') verstr = f'%(prog)s {__version__} ' \ f'(configuration format version {__file_version__}, ' \ f'earliest compatible version {__compatible_file_version__})' parser.add_argument('--version', action='version', version=verstr) parser.add_argument('-l', '--log-level', choices=['debug', 'info', 'warning', 'error', 'critical'], default=f'{DEFAULT_LOG_LEVEL}', help=f'Set log level (default: {DEFAULT_LOG_LEVEL})') subparser = parser.add_subparsers(help='sub command help', dest='cmd') for plugin in plugins.all(): plugin_parser = subparser.add_parser( plugin.name, help=plugin.helpmsg, formatter_class=ArgumentChoicesHelpFormatter) plugin.setup_parser(plugin_parser) return parser class ArgumentChoicesHelpFormatter(argparse.HelpFormatter): """Help message formatter which adds choices to argument help. If the default METAVAR is used, this will do nothing, as the default METAVAR shows the available choices already. If the METAVAR is overridden, and %(choice)s is not present in the help string, add them. """ def _get_help_string(self, action): help = action.help if action.choices and action.metavar is not None: if "%(choices)" not in action.help: help += " Possible choices: %(choices)s." return help def kas(argv): """ The actual main entry point of kas. """ create_logger() parser = kas_get_argparser() args = parser.parse_args(argv) if args.log_level: set_global_loglevel(args.log_level.upper()) logging.info('%s %s started on %s %s', os.path.basename(sys.argv[0]), __version__, distro.name(), distro.codename() or distro.version()) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) register_signal_handlers(loop) try: plugin_class = plugins.get(args.cmd) if plugin_class: plugin = plugin_class() plugin.run(args) else: parser.print_help() except CommandExecError as err: logging.error('%s', err) raise except KasUserError as err: logging.error('%s', err) raise except asyncio.CancelledError: logging.error('kas execution cancelled') raise except Exception as err: logging.error('%s', err) raise finally: shutdown_loop(loop) cleanup_logger() def main(): """ The main function that operates as a wrapper around kas. """ try: kas(sys.argv[1:]) except CommandExecError as err: sys.exit(err.ret_code if err.forward else 2) except KasUserError: sys.exit(2) except asyncio.CancelledError: sys.exit(130) except Exception: traceback.print_exc() sys.exit(1) if __name__ == '__main__': main() siemens-kas-41ad961/kas/kasusererror.py000066400000000000000000000071331520561422700201530ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2023 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module provides a common base class for all exceptions which are related to user or configuration errors. These exceptions should be caught and reported to the user using a meaningful message instead of a stacktrace. When handling errors in KAS, never return directly using `sys.exit`, but instead throw an exception derived from :class:`KasUserError` (for user errors), or one derived from `Exception` for internal errors. These are then handled centrally, mapped to correct return codes and pretty printed. """ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2023' class KasUserError(Exception): """ User or input error. Derive all user error exceptions from this class. """ pass class CommandExecError(KasUserError): """ Failure in execution of a shell command. The `forward_error_code` parameter can be used to request the receiver of the exception to `sys.exit` with that code instead of a generic one. Only use this in special cases, where the return code can actually be related to a single shell command. """ def __init__(self, command, ret_code, forward_ret_code=False): self.ret_code = ret_code self.forward = forward_ret_code message = ' '.join([f"'{c}'" if ' ' in c else c for c in command]) super().__init__(f'Command "{message}" failed with error {ret_code}') class ArgsCombinationError(KasUserError): """ Invalid combination of CLI arguments provided """ def __init__(self, message): super().__init__(f'Invalid combination of arguments: {message}') class ArtifactNotFoundError(KasUserError, FileNotFoundError): """ A configured artifact is not found (or the glob matches 0 elements). """ def __init__(self, name, artifact): super().__init__(f'No artifact found for {name}:"{artifact}"') class EnvSetButNotFoundError(KasUserError, FileNotFoundError): """ A environment variable pointing to a file or directory is set, but the path it points to does not exist. """ def __init__(self, env_name, path): super().__init__(f'Environment variable "{env_name}" is set, but the ' f'path does not exist: {path}') class MissingModuleError(KasUserError): """ An optional module is missing for the requested operation """ def __init__(self, module, operation) -> None: super().__init__(f'Module "{module}" is required for: {operation}') siemens-kas-41ad961/kas/keyhandler.py000066400000000000000000000251051520561422700175510ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens, 2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module provide the infrastructure to setup and work with local GnuPG keyrings. """ import logging import subprocess from pathlib import Path from git.config import GitConfigParser from kas.kasusererror import KasUserError, MissingModuleError try: import gnupg HAVE_GNUPG = True except ImportError: HAVE_GNUPG = False class KeyImportError(KasUserError): """ Raised when a key could not be imported. """ def __init__(self, name, message): super().__init__(f'Could not import key "{name}": {message}') class KeyHandler: @property def env(self): return {} def prepare_validation(self, repo): """ Configure the repository for signature validation (e.g. define set of allowed keys). Currently only supported for git repos. """ if repo.get_type() != 'git': raise KasUserError('Only git repositories are supported') def validate_allowed_signer(self, repo, output): """ Process the output of a signature validation and validate it against the list of allowed signers. Must only be called if the validation by the VCS was completed successfully. """ assert repo.get_type() == 'git' def get_key_repr(self, keyname): """ Returns a human readable representation of the key. Derived classes may override this to provide a more meaningful representation of the key. """ return keyname class GPGKeyHandler(KeyHandler): def __init__(self, gnupghome, signers, confinst): if not HAVE_GNUPG: raise MissingModuleError('python-gnupg', 'signature verification') if logging.getLogger().level <= logging.DEBUG: logging.getLogger('gnupg').setLevel(logging.INFO) self.gpg = gnupg.GPG(gnupghome=str(gnupghome)) self.fingerprints = {} self._reset_trust() self._import_keys(signers, confinst) def _reset_trust(self): all_keys = self.gpg.list_keys() for key in all_keys: fingerprint = key['fingerprint'] logging.debug(f'Resetting trust level for key "{fingerprint}"') self.gpg.trust_keys(fingerprint, 'TRUST_NEVER') def _import_keys(self, signers, confinst): """ Import the keys based on the config entries. The schema ensures, that only valid combinations can be passed to this function. """ for name, loc in signers.items(): fingerprint = loc.get('fingerprint') keyserver = loc.get('gpg_keyserver') if 'path' in loc: repo = confinst.get_repo(loc['repo']) keyfile = Path(repo.path) / Path(loc['path']) import_result = self.gpg.import_keys_file(keyfile) elif self._key_cached(fingerprint): logging.debug(f'Key "{name}" with fingerprint ' f'"{fingerprint}" already cached, ' f'skipping download') self.gpg.trust_keys(fingerprint, 'TRUST_ULTIMATE') self.fingerprints[name] = fingerprint continue else: import_result = self.gpg.recv_keys(keyserver, fingerprint) if import_result.count == 0: raise KeyImportError(name, 'No keys imported') if import_result.count > 1: raise KeyImportError(name, 'Multiple keys imported') # an import result can also have expired keys which are not # imported but show up in the results array. Ignore them. actual_fp = [k for k in import_result.results if k['fingerprint']][0]['fingerprint'] if fingerprint and actual_fp != fingerprint: raise KeyImportError(name, 'Fingerprints do not match: ' f'Key has "{actual_fp}". ' f'Expected "{fingerprint}".') # we operate on a kas-local keystore, so we can trust the key self.gpg.trust_keys(actual_fp, 'TRUST_ULTIMATE') self.fingerprints[name] = actual_fp logging.debug(f'Imported key "{name}" with fingerprint ' f'"{actual_fp}"') def _key_cached(self, fingerprint): keys = self.gpg.list_keys(keys=fingerprint) return len(keys) > 0 def _fingerprint(self, keyname): fingerprint = self.fingerprints.get(keyname) if not fingerprint: raise KasUserError(f'Key "{keyname}" not found') return self.gpg.list_keys(keys=fingerprint)[0]['fingerprint'] def _keyid(self, fingerprint): keyname = [k for k, v in self.fingerprints.items() if v == fingerprint] if not keyname: return None return keyname[0] @property def env(self): return {'GNUPGHOME': str(self.gpg.gnupghome)} def validate_allowed_signer(self, repo, output): super().validate_allowed_signer(repo, output) allowed_fps = [self._fingerprint(x) for x in repo.allowed_signers] validsigs = [x for x in output.split('\n') if 'VALIDSIG' in x] sigs = [x.split()[-1] for x in validsigs] if not sigs: return (False, None) for sig in sigs: if sig in allowed_fps: return (True, self._keyid(sig)) return (False, self._keyid(sigs[0])) def get_key_repr(self, keyname): fingerprint = self._fingerprint(keyname) uids = self.gpg.list_keys(keys=fingerprint)[0]['uids'] fp_formatted = ' '.join([(fingerprint[i:i + 4]) for i in range(0, len(fingerprint), 4)]) uidstr = ' aka '.join([f'"{uid}"' for uid in uids]) return f'Fingerprint {fp_formatted} from {uidstr}' class SSHKeyHandler(KeyHandler): def __init__(self, workdir, signers, confinst): self.workdir = workdir self.signers = {} for name, loc in signers.items(): repo = confinst.get_repo(loc['repo']) pubfile = Path(repo.path) / Path(loc['path']) keydata = subprocess.check_output(['ssh-keygen', '-lf', pubfile.absolute()]) keyparts = keydata.decode('utf-8').split() size, fp = keyparts[0:2] comment = ' '.join(keyparts[2:-1]) rawtype = keyparts[-1].replace('(', '').replace(')', '') keytype = self._key_name_from_sn(rawtype, size) if 'fingerprint' in loc and loc['fingerprint'] != fp: raise KeyImportError(name, 'Fingerprints do not match: ' f'Key has "{fp}". ' f'Expected "{loc["fingerprint"]}".') rawkey = subprocess.check_output(['ssh-keygen', '-mRFC4716', '-ef', pubfile]) key = self._key_from_rfc4716(rawkey.decode('utf-8')) self.signers[name] = { 'type': keytype.lower(), 'key': key, 'comment': comment, 'fingerprint': fp, 'size': size } @staticmethod def _key_name_from_sn(name, size): """ Convert the key type and size from the output of ssh-keygen -lf to the ssh key type string. """ # RFC 4253 section 6.6 if name == 'DSA': return 'ssh-dss' if name == 'RSA': return 'ssh-rsa' # RFC 5656 section 6.2 if name == 'ECDSA' and int(size) in [256, 384, 521]: return f'ecdsa-sha2-nistp{size}' # RFC 8709 section 6 if name == 'ED25519': return 'ssh-ed25519' raise KasUserError(f'Unsupported key type "{name}" with size "{size}"') @staticmethod def _key_from_rfc4716(data): parts = [] for line in data.splitlines(): if not line.startswith('----') and not line.startswith('Comment:'): parts.append(line) return ''.join(parts) def _keyid(self, fingerprint): for k, v in self.signers.items(): if v['fingerprint'] == fingerprint: return k return None def prepare_validation(self, repo): super().prepare_validation(repo) trustpath = self.workdir / repo.name trustpath.mkdir(exist_ok=True) allowedSignersFile = trustpath / 'allowedSigners' with open(allowedSignersFile, 'w') as f: for k in repo.allowed_signers: signer = self.signers.get(k) f.write(f'"{signer["comment"]}" namespaces="git" ' f'{signer["type"]} {signer["key"]}\n') gitconfig = Path(repo.path) / '.git/config' with GitConfigParser(gitconfig, read_only=False) as config: config.add_section('gpg "ssh"') config['gpg "ssh"']['allowedSignersFile'] = str(allowedSignersFile) config.write() def validate_allowed_signer(self, repo, output): super().validate_allowed_signer(repo, output) fp = output.split()[-1] return (True, self._keyid(fp)) def get_key_repr(self, keyname): signer = self.signers.get(keyname) return f'Fingerprint {signer["fingerprint"]} ({signer["type"]}) ' \ f'from "{signer["comment"]}"' siemens-kas-41ad961/kas/libcmds.py000066400000000000000000000540721520561422700170450ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains common commands used by kas plugins. """ import tempfile import logging import shutil import os import pprint import configparser import json import base64 from pathlib import Path from git.config import GitConfigParser from .libkas import (ssh_cleanup_agent, ssh_setup_agent, ssh_no_host_key_check, get_build_environ, repos_fetch, repos_apply_patches, add_cachedir_tag) from .context import ManagedEnvironment as ME from .context import get_context from .includehandler import IncludeException from .kasusererror import EnvSetButNotFoundError, ArgsCombinationError from .repos import SignatureValidator __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' KAS_USER_NAME = 'kas User' KAS_USER_EMAIL = 'kas@example.com' HTTPS_PORT_DEFAULT = 443 SSH_PORT_DEFAULT = 22 class Macro: """ Contains commands and provides method to run them. """ def __init__(self, use_common_setup=True): if use_common_setup: repo_loop = Loop('repo_setup_loop') repo_loop.add(SetupReposStep()) # setup commands are pairs of setup / cleanup commands self.setup_commands = [ (SetupDir(), None) ] if ('SSH_PRIVATE_KEY' in os.environ or 'SSH_PRIVATE_KEY_FILE' in os.environ): if 'SSH_AUTH_SOCK' in os.environ: raise ArgsCombinationError( 'Internal SSH agent (e.g. for "SSH_PRIVATE_KEY") can ' 'only be started if no external one is passed.') self.setup_commands.append((SetupSSHAgent(), CleanupSSHAgent())) self.setup_commands += [(x, None) for x in [ SetupHome(), MakeNonInteractive(), InitSetupRepos(), repo_loop, FinishSetupRepos(), ReposCheckout(), ReposApplyPatches(), SetupEnviron(), WriteBBConfig(), ]] else: self.setup_commands = [] self.commands = [] def add(self, command): """ Appends commands to the command list. """ self.commands.append(command) def run(self, ctx, skip=None): """ Runs a command from the command list with respect to the configuration. """ def _run_single(command): command_name = str(command) if command_name in (skip or []): return False logging.debug('execute %s', command_name) command.execute(ctx) return True cleanup_commands = [] try: for cmd in self.setup_commands: if _run_single(cmd[0]) and cmd[1]: cleanup_commands.insert(0, cmd[1]) for cmd in self.commands: _run_single(cmd) finally: for cmd in cleanup_commands: _run_single(cmd) class Command: """ An abstract class that defines the interface of a command. """ def execute(self, ctx): """ This method executes the command. """ pass class Loop(Command): """ A class that defines a set of commands as a loop. """ def __init__(self, name): self.commands = [] self.name = name def __str__(self): return self.name def add(self, command): """ Appends a command to the loop. """ self.commands.append(command) def execute(self, ctx): """ Executes the loop. """ loop_name = str(self) def executor(command): command_name = str(command) logging.debug('Loop %s: execute %s', loop_name, command_name) return command.execute(ctx) while all(executor(c) for c in self.commands): pass class SetupHome(Command): """ Sets up the home directory of kas. """ # A list of environment variables that SETUP_HOME uses # This should be kept up to date with any code in execute() ENV_VARS = [ 'GIT_CREDENTIAL_HELPER', 'GIT_CREDENTIAL_USEHTTPPATH', 'GITCONFIG_FILE', 'AWS_CONFIG_FILE', 'AWS_ROLE_ARN', 'AWS_SHARED_CREDENTIALS_FILE', 'AWS_WEB_IDENTITY_TOKEN_FILE', 'NETRC_FILE', 'NPMRC_FILE', 'REGISTRY_AUTH_FILE', ] def __init__(self): super().__init__() self.tmpdirname = None def __del__(self): if self.tmpdirname: shutil.rmtree(self.tmpdirname) def __str__(self): return 'setup_home' @staticmethod def _path_from_env(name): """ Returns the path a env var points to (if set). If the path does not exist, raises an EnvSetButNotFoundError. """ rawpath = os.environ.get(name, None) if not rawpath: return None path = Path(rawpath) if not path.exists(): raise EnvSetButNotFoundError(name, path) return path @staticmethod def _ssh_config_present(): """ Checks if the .ssh/config file exists or any manual ssh config option is set. """ ssh_vars = ['SSH_PRIVATE_KEY', 'SSH_PRIVATE_KEY_FILE', 'SSH_AUTH_SOCK'] if any(e in os.environ for e in ssh_vars): return True ssh_path = os.path.expanduser('~/.ssh') if os.path.exists(os.path.join(ssh_path, 'config')): return True return False def _setup_netrc(self): netrc_file = self._path_from_env('NETRC_FILE') if netrc_file: shutil.copy(netrc_file, self.tmpdirname + "/.netrc") if os.environ.get('CI_SERVER_HOST', False) \ and os.environ.get('CI_JOB_TOKEN', False): with open(self.tmpdirname + '/.netrc', 'a') as fds: fds.write('machine ' + os.environ['CI_SERVER_HOST'] + '\n' 'login gitlab-ci-token\n' 'password ' + os.environ['CI_JOB_TOKEN'] + '\n') def _setup_npmrc(self): npmrc_file = self._path_from_env('NPMRC_FILE') if not npmrc_file: return shutil.copy(npmrc_file, self.tmpdirname + "/.npmrc") def _setup_registry_auth(self): os.makedirs(self.tmpdirname + "/.docker") reg_auth_file = self._path_from_env('REGISTRY_AUTH_FILE') if reg_auth_file: shutil.copy(reg_auth_file, self.tmpdirname + "/.docker/config.json") elif not os.path.exists(self.tmpdirname + '/.docker/config.json'): with open(self.tmpdirname + '/.docker/config.json', 'w') as fds: fds.write("{}") if os.environ.get('CI_REGISTRY', False) \ and os.environ.get('CI_JOB_TOKEN', False) \ and os.environ.get('CI_REGISTRY_USER', False): with open(self.tmpdirname + '/.docker/config.json', 'r+') as fds: data = json.loads(fds.read()) token = os.environ['CI_JOB_TOKEN'] base64_token = base64.b64encode(token.encode()).decode() auths = data.get('auths', {}) auths.update( {os.environ['CI_REGISTRY']: {"auth": base64_token}}) data['auths'] = auths fds.seek(0) fds.write(json.dumps(data, indent=4)) fds.truncate() def _setup_aws_creds(self): aws_dir = self.tmpdirname + "/.aws" conf_file = aws_dir + "/config" shared_creds_file = aws_dir + "/credentials" sso_cache_dir = aws_dir + "/sso/cache" os.makedirs(aws_dir) aws_conf_file = self._path_from_env('AWS_CONFIG_FILE') aws_shared_creds_file = \ self._path_from_env('AWS_SHARED_CREDENTIALS_FILE') if aws_conf_file and aws_shared_creds_file: shutil.copy(aws_conf_file, conf_file) shutil.copy(aws_shared_creds_file, shared_creds_file) # OAuth 2.0 workflow credentials aws_web_identity_token_file = \ self._path_from_env('AWS_WEB_IDENTITY_TOKEN_FILE') if aws_web_identity_token_file and os.environ.get('AWS_ROLE_ARN'): webid_token_file = aws_dir + '/web_identity_token' config = configparser.ConfigParser() if os.path.exists(conf_file): config.read(conf_file) if 'default' not in config: config['default'] = {} config['default']['role_arn'] = os.environ.get('AWS_ROLE_ARN') config['default']['web_identity_token_file'] = webid_token_file with open(aws_dir + '/config', 'w') as fds: config.write(fds) shutil.copy(aws_web_identity_token_file, webid_token_file) # SSO workflow if aws_conf_file: aws_cache_dir_conf = \ os.path.join(os.path.dirname(aws_conf_file), "sso/cache") aws_cache_dir_home = os.path.join(Path.home(), ".aws/sso/cache") # In kas-container the directory passed in --aws-dir is not # mounted in $HOME. Look for sso/cache in directory containing # AWS_CONFIG_FILE first to maintain the same behavior between # kas and kas-container. aws_cache_dir = None if os.path.isdir(aws_cache_dir_conf): aws_cache_dir = aws_cache_dir_conf elif os.path.isdir(aws_cache_dir_home): aws_cache_dir = aws_cache_dir_home if aws_cache_dir: shutil.copy(aws_conf_file, conf_file) shutil.copytree(aws_cache_dir, sso_cache_dir) @staticmethod def _setup_gitlab_ci_ssh_rewrite(config): ci_host = os.environ.get('CI_SERVER_HOST', None) ci_port = os.environ.get('CI_SERVER_PORT', HTTPS_PORT_DEFAULT) ci_prot = os.environ.get('CI_SERVER_PROTOCOL', 'https') # added in GitLab 15.11. Set sensible defaults for older versions. ci_ssh_host = os.environ.get('CI_SERVER_SHELL_SSH_HOST', ci_host) ci_ssh_port = os.environ.get('CI_SERVER_SHELL_SSH_PORT', SSH_PORT_DEFAULT) for host in [ci_host, f'{ci_host}:{ci_port}']: section = f'url "{ci_prot}://{host}/"' config.add_value(section, 'insteadOf', f'git@{ci_ssh_host}:') config.add_value(section, 'insteadOf', f'git@{ci_ssh_host}:{ci_ssh_port}') config.add_value(section, 'insteadOf', f'ssh://git@{ci_ssh_host}/') config.add_value(section, 'insteadOf', f'ssh://git@{ci_ssh_host}:{ci_ssh_port}/') def _setup_gitconfig(self): gitconfig_host = self._path_from_env('GITCONFIG_FILE') gitconfig_kas = self.tmpdirname + '/.gitconfig' # when running in a externally managed environment, # always try to read the gitconfig if not gitconfig_host and get_context().managed_env: gitconfig_host = Path('~/.gitconfig').expanduser() if not gitconfig_host.exists(): gitconfig_host = None if gitconfig_host: shutil.copy(gitconfig_host, gitconfig_kas) with GitConfigParser(gitconfig_kas, read_only=False) as config: if os.environ.get('GIT_CREDENTIAL_HELPER', False): config['credential'] = { 'helper': os.environ.get('GIT_CREDENTIAL_HELPER') } if os.environ.get('GIT_CREDENTIAL_USEHTTPPATH', False): config['credential']['useHttpPath'] = \ os.environ.get('GIT_CREDENTIAL_USEHTTPPATH') if get_context().managed_env == ME.GITLAB_CI and \ not gitconfig_host: ci_project_dir = self._path_from_env('CI_PROJECT_DIR') if ci_project_dir: logging.debug('Adding git safe.directory %s', ci_project_dir) config.add_value('safe', 'directory', str(ci_project_dir)) ci_server = os.environ.get('CI_SERVER_HOST', None) if ci_server and not self._ssh_config_present(): logging.debug('Adding GitLab CI ssh -> https rewrites') self._setup_gitlab_ci_ssh_rewrite(config) config.write() def execute(self, ctx): managed_env = get_context().managed_env if managed_env: logging.info(f'Running on {managed_env}') if not self.tmpdirname: self.tmpdirname = tempfile.mkdtemp() def_umask = os.umask(0o077) self._setup_netrc() self._setup_npmrc() self._setup_registry_auth() self._setup_gitconfig() self._setup_aws_creds() os.umask(def_umask) ctx.environ['HOME'] = self.tmpdirname class MakeNonInteractive(Command): """ Make execution environment non-interactive """ def __str__(self): return 'make_non_interactive' def execute(self, ctx): ctx.environ['GIT_TERMINAL_PROMPT'] = 'false' class MakeInteractive(Command): """ Make execution environment interactive """ def __str__(self): return 'make_interactive' def execute(self, ctx): ctx.environ['GIT_TERMINAL_PROMPT'] = 'true' class SetupDir(Command): """ Creates the build directory. """ def __str__(self): return 'setup_dir' def execute(self, ctx): if not os.path.exists(ctx.build_dir): os.mkdir(ctx.build_dir) add_cachedir_tag(ctx.build_dir, "kas build directory") class SetupSSHAgent(Command): """ Sets up the ssh agent configuration. """ def __str__(self): return 'setup_ssh_agent' def execute(self, ctx): ssh_tools = ['ssh', 'ssh-add', 'ssh-agent'] for tool in ssh_tools: if shutil.which(tool) is None: raise RuntimeError('SSH setup requested but could ' f'not find "{tool}" in PATH') ssh_setup_agent() ssh_no_host_key_check() class CleanupSSHAgent(Command): """ Removes all the identities and stops the ssh-agent instance. """ def __str__(self): return 'cleanup_ssh_agent' def execute(self, ctx): ssh_cleanup_agent() class SetupEnviron(Command): """ Sets up the kas environment. """ def __str__(self): return 'setup_environ' def execute(self, ctx): ctx.environ.update(get_build_environ(ctx.config.get_build_system())) class WriteBBConfig(Command): """ Writes bitbake configuration files into the build directory. """ def __str__(self): return 'write_bbconfig' def execute(self, ctx): def _get_layer_path_under_topdir(ctx, layer): """ Returns a path relative to ${TOPDIR}. TOPDIR is a BB variable pointing to the build directory. It is not expanded by KAS, hence we avoid absolute paths pointing into the build host. """ # we need to walk up, which would require pathlib.Path >= 3.12 relpath = os.path.relpath(layer.path, ctx.build_dir) return '${TOPDIR}/' + relpath def _write_bblayers_conf(ctx): filename = ctx.build_dir + '/conf/bblayers.conf' if not os.path.isdir(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) layers_sorted = \ sorted([layer for repo in ctx.config.get_repos() for layer in repo.layers]) with open(filename, 'w') as fds: fds.write(ctx.config.get_bblayers_conf_header()) fds.write('BBLAYERS ?= " \\\n ') fds.write(' \\\n '.join( [_get_layer_path_under_topdir(ctx, layer) for layer in layers_sorted])) fds.write('"\n') fds.write('BBPATH ?= "${TOPDIR}"\n') fds.write('BBFILES ??= ""\n') def _write_local_conf(ctx): filename = ctx.build_dir + '/conf/local.conf' with open(filename, 'w') as fds: fds.write(ctx.config.get_local_conf_header()) fds.write(f'MACHINE ??= "{ctx.config.get_machine()}"\n') fds.write(f'DISTRO ??= "{ctx.config.get_distro()}"\n') fds.write('BBMULTICONFIG ?= ' f'"{ctx.config.get_multiconfig()}"\n') _write_bblayers_conf(ctx) _write_local_conf(ctx) class ReposApplyPatches(Command): """ Applies the patches defined in the configuration to the repositories. """ def __str__(self): return 'repos_apply_patches' def _vcs_operate_as_kas(self, gitconfig): # currently only implemented for git with GitConfigParser(gitconfig, read_only=False) as config: # in case no user is defined, we keep the kas user user_orig = { 'name': config.get('user', 'name', fallback=KAS_USER_NAME), 'email': config.get('user', 'email', fallback=KAS_USER_EMAIL) } config.set_value('user', 'name', KAS_USER_NAME) config.set_value('user', 'email', KAS_USER_EMAIL) config.write() return user_orig def _vcs_restore_user(self, gitconfig, user): # currently only implemented for git with GitConfigParser(gitconfig, read_only=False) as config: config['user'] = user config.write() def execute(self, ctx): if 'HOME' not in ctx.environ: raise ArgsCombinationError('Apply patches requires setup_home') gitconfig = ctx.environ['HOME'] + '/.gitconfig' user = self._vcs_operate_as_kas(gitconfig) repos_apply_patches(ctx.config.get_repos()) self._vcs_restore_user(gitconfig, user) class InitSetupRepos(Command): """ Prepares setting up repos including the include logic """ def __str__(self): return 'init_setup_repos' def execute(self, ctx): ctx.missing_repo_names = ctx.config.find_missing_repos() ctx.missing_repo_names_old = None class SetupReposStep(Command): """ Single step of the checkout repos loop """ def __str__(self): return 'setup_repos_step' def execute(self, ctx): if not ctx.missing_repo_names: return False if ctx.missing_repo_names == ctx.missing_repo_names_old: raise IncludeException('Could not fetch all repos needed by ' 'includes. Missing repos: {}' .format(', '.join(ctx.missing_repo_names))) logging.debug('Missing repos for complete config:\n%s', pprint.pformat(ctx.missing_repo_names)) ctx.missing_repos = [] for repo_name in ctx.missing_repo_names: if repo_name not in ctx.config.get_repos_config(): # we don't have this repo yet (e.g. due to transitive incl.) continue ctx.missing_repos.append((repo_name, ctx.config.get_repo(repo_name))) repos_fetch([v for k, v in ctx.missing_repos]) # import keys from old config and validate against this state SignatureValidator.import_keys(ctx) for _, repo in ctx.missing_repos: # check signature prior to checkout and config dict update # to avoid TOCTOU issues SignatureValidator.ensure_valid_if_signed(ctx, repo) repo.checkout() ctx.config.repo_dict.update( {id: repo for id, repo in ctx.missing_repos}) repo_paths = {r: ctx.config.repo_dict[r].path for r in ctx.config.repo_dict} ctx.missing_repo_names_old = ctx.missing_repo_names ctx.missing_repo_names = \ ctx.config.find_missing_repos(repo_paths) return ctx.missing_repo_names class FinishSetupRepos(Command): """ Finalizes the repo setup loop """ def __str__(self): return 'finish_setup_repos' def execute(self, ctx): # now fetch everything with complete config repos_fetch(ctx.config.get_repos()) config_str = pprint.pformat(ctx.config.get_config(), sort_dicts=False) logging.debug('Configuration from config file:\n%s', config_str) class ReposCheckout(Command): """ Ensures that the right revision of each repo is checked out. """ def __str__(self): return 'repos_checkout' def execute(self, ctx): SignatureValidator.import_keys(ctx) for repo in ctx.config.get_repos(): SignatureValidator.ensure_valid_if_signed(ctx, repo) repo.checkout() siemens-kas-41ad961/kas/libkas.py000066400000000000000000000575051520561422700167010ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains the core implementation of kas. """ import argparse import re import os import sys import logging import tempfile import asyncio import errno import hashlib import hmac import pathlib import platform import shutil import signal import stat from subprocess import Popen, PIPE, run as subprocess_run from urllib.parse import quote from .context import get_context from .kasusererror import KasUserError, CommandExecError from .configschema import CONFIGSCHEMA __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' class InitBuildEnvError(KasUserError): """ Error related to the OE / ISAR environment setup scripts """ pass class EnvNotValidError(KasUserError): """ The caller environment is not suited for the requested operation """ pass class TaskExecError(KasUserError): """ Similar to :class:`kas.kasusererror.CommandExecError` but for kas internal tasks """ def __init__(self, command, ret_code): self.ret_code = ret_code super().__init__(f'{command} failed: error code {ret_code}') class LogOutput: """ Handles the log output of executed applications """ def __init__(self, live): self.live = live self.stdout = [] self.stderr = [] def log_stdout(self, line): """ This method is called when a line is received over stdout. """ if self.live: logging.info(line.strip()) self.stdout.append(line) def log_stderr(self, line): """ This method is called when a line is received over stderr. """ if self.live: logging.error(line.strip()) self.stderr.append(line) async def _read_stream(stream, callback): """ This asynchronous method reads from the output stream of the application and transfers each line to the callback function. """ while True: line = await stream.readline() try: line = line.decode('utf-8') except UnicodeDecodeError as err: logging.warning('Could not decode line from stream, ignoring: %s', err) if line: callback(line) else: break def _filter_stderr(capture_stderr, ret, out, err=None): if capture_stderr: return (ret, out, err or '') else: return (ret, out) def _report_cmd_error(ret, cwd, cmdstr, cmd, stderr): if stderr: msg = f'Command "{cwd}$ {cmdstr}" failed:\n{stderr}' logging.error(msg.rstrip('\n')) raise CommandExecError(cmd, ret) async def run_cmd_async(cmd, cwd, env=None, fail=True, liveupdate=False, capture_stderr=False): """ Run a command asynchronously. """ env = env or get_context().environ cmdstr = ' '.join(cmd) logging.debug('%s$ %s', cwd, cmdstr) logo = LogOutput(liveupdate) orig_fd = signal.set_wakeup_fd(-1, warn_on_full_buffer=False) signal.set_wakeup_fd(orig_fd, warn_on_full_buffer=False) try: process = await asyncio.create_subprocess_exec( *cmd, cwd=cwd, env=env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, preexec_fn=os.setpgrp) except FileNotFoundError as ex: if fail: raise ex return _filter_stderr(capture_stderr, errno.ENOENT, str(ex)) except PermissionError as ex: if fail: raise ex return _filter_stderr(capture_stderr, errno.EPERM, str(ex)) # Process termination is a complicated thing. We need to ensure that # the event-loop ThreadedChildWatcher thread fires before the loop is # terminated. The best we can do it to ask the process to terminate # (SIGINT) and wait for it. We need to shield the process wait to avoid # that it is killed by the cancellation of the task, as we want a # controlled termination. Forced terminations can leak an orphaned process. # https://github.com/pytest-dev/pytest-asyncio/issues/708#issuecomment-1868488942 tasks = [ asyncio.ensure_future(_read_stream(process.stdout, logo.log_stdout)), asyncio.ensure_future(_read_stream(process.stderr, logo.log_stderr)) ] try: await asyncio.gather(*[asyncio.shield(t) for t in tasks]) ret = await asyncio.shield(process.wait()) except asyncio.CancelledError: try: process.terminate() except ProcessLookupError: # Process already exited between the cancel and us reaching here. pass logging.debug('Command "%s" cancelled', cmdstr) await process.wait() raise if ret and fail: _report_cmd_error(ret, cwd, cmdstr, cmd, ''.join(logo.stderr)) return _filter_stderr(capture_stderr, ret, ''.join(logo.stdout), ''.join(logo.stderr)) def run_cmd(cmd, cwd, env=None, fail=True, capture_stderr=False): """ Runs a command synchronously. """ env = env or get_context().environ cmdstr = ' '.join(cmd) logging.debug('%s$ %s', cwd, cmdstr) try: ret = subprocess_run(cmd, env=env, cwd=cwd, stdout=PIPE, stderr=PIPE) if ret.returncode and fail: _report_cmd_error(ret.returncode, cwd, cmdstr, cmd, ret.stderr.decode('utf-8')) except FileNotFoundError as ex: if fail: raise ex return _filter_stderr(capture_stderr, errno.ENOENT, str(ex)) except PermissionError as ex: if fail: raise ex return _filter_stderr(capture_stderr, errno.EPERM, str(ex)) return _filter_stderr(capture_stderr, ret.returncode, ret.stdout.decode('utf-8'), ret.stderr.decode('utf-8')) def find_program(paths, name): """ Find a file within the paths array and returns its path. """ for path in paths.split(os.pathsep): prg = os.path.join(path, name) if os.path.isfile(prg): return prg return None def repos_fetch(repos): """ Fetches the list of repositories to the kas_work_dir. .. note:: termination point of the asyncio event loop. """ if len(repos) == 0: return tasks = [] for repo in repos: tasks.append(asyncio.ensure_future(repo.fetch_async())) loop = asyncio.get_event_loop() try: loop.run_until_complete(asyncio.gather(*tasks)) except CommandExecError as e: [t.cancel() for t in tasks] raise TaskExecError('fetch repos', e.ret_code) except KasUserError: [t.cancel() for t in tasks] raise def repos_apply_patches(repos): """ Applies the patches to the repositories. .. note:: termination point of the asyncio event loop. """ if len(repos) == 0: return tasks = [] for repo in repos: tasks.append(asyncio.ensure_future(repo.apply_patches_async())) loop = asyncio.get_event_loop() try: loop.run_until_complete(asyncio.gather(*tasks)) except CommandExecError as e: [t.cancel() for t in tasks] raise TaskExecError('apply patches', e.ret_code) except KasUserError: [t.cancel() for t in tasks] raise def get_buildtools_dir(): # Set the dest. directory for buildtools's setup env_path = os.environ.get("KAS_BUILDTOOLS_DIR") if env_path: return pathlib.Path(env_path).resolve() # defaults to KAS_BUILD_DIR/buildtools return (pathlib.Path(get_context().build_dir) / 'buildtools').resolve() def get_buildtools_filename(): arch = platform.machine() ctx = get_context() conf_buildtools = ctx.config.get_buildtools() version = conf_buildtools['version'] if 'filename' in conf_buildtools: filename = conf_buildtools['filename'] else: filename = ( f"{arch}-buildtools-extended-" f"nativesdk-standalone-{version}.sh" ) return filename def get_buildtools_path(): return get_buildtools_dir() / get_buildtools_filename() def get_buildtools_url(): ctx = get_context() conf_buildtools = ctx.config.get_buildtools() filename = get_buildtools_filename() version = conf_buildtools['version'] if 'base_url' in conf_buildtools: base_url = conf_buildtools['base_url'] else: default = ( CONFIGSCHEMA['properties']['buildtools']['properties'] ['base_url']['default'] ) base_url = f"{default}/yocto-{version}/buildtools/" return f"{base_url}/{quote(filename)}" def check_sha256sum(filename, expected_checksum): hash_sha256 = hashlib.sha256() with open(filename, "rb") as f: for chunk in iter(lambda: f.read(8192), b""): hash_sha256.update(chunk) actual_checksum = hash_sha256.hexdigest() logging.info( f"Buildtools installer's checksum (sha256) is: " f"{actual_checksum}" ) return hmac.compare_digest(actual_checksum, expected_checksum) def download_buildtools(): ctx = get_context() conf_buildtools = ctx.config.get_buildtools() version = conf_buildtools['version'] buildtools_dir = get_buildtools_dir() # Enable extended buildtools tarball buildtools_url = get_buildtools_url() tmpbuildtools = get_buildtools_path() logging.info(f"Downloading Buildtools {version}") # Download installer fetch_cmd = ['wget', '-q', '-O', str(tmpbuildtools), buildtools_url] (ret, _) = run_cmd(fetch_cmd, cwd=ctx.kas_work_dir) if ret != 0: raise InitBuildEnvError("Could not download buildtools installer") # Check if the installer's sha256sum matches if not check_sha256sum(tmpbuildtools, conf_buildtools['sha256sum']): raise InitBuildEnvError( "sha256sum mismatch: installer may be corrupted" ) # Make installer executable st = tmpbuildtools.stat() tmpbuildtools.chmod(st.st_mode | stat.S_IEXEC) # Run installer (in an isolated environment) installer_cmd = [str(tmpbuildtools), '-d', str(buildtools_dir), '-y'] env = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin'} (ret, _) = run_cmd(installer_cmd, cwd=ctx.kas_work_dir, env=env) if ret != 0: raise InitBuildEnvError("Could not run buildtools installer") def get_buildtools_version(): try: version_file = list(get_buildtools_dir().glob("version-*")) if len(version_file) != 1: raise ValueError("Invalid number of version files") with version_file[0].resolve().open('r') as f: lines = f.readlines() for line in lines: if line.startswith("Distro Version"): return lines[1].split(':', 1)[1].strip() except Exception as e: logging.warning(f"Unable to read buildtools version: {e}") return -1 def get_build_environ(build_system): """ Creates the build environment variables. """ # nasty side effect function: running oe/isar-init-build-env also # creates the conf directory init_repo = None if build_system in ['openembedded', 'oe']: scripts = ['oe-init-build-env'] elif build_system == 'isar': scripts = ['isar-init-build-env'] else: scripts = ['oe-init-build-env', 'isar-init-build-env'] permutations = \ [(repo, script) for repo in get_context().config.get_repos() for script in scripts] for (repo, script) in permutations: if os.path.exists(repo.path + '/' + script): if init_repo: raise InitBuildEnvError( 'Multiple init scripts found ' f'({repo.name} vs. {init_repo.name}). ' 'Resolve ambiguity by removing one of the repos') init_repo = repo init_script = script if not init_repo: raise InitBuildEnvError('Did not find any init-build-env script') conf_buildtools = get_context().config.get_buildtools() buildtools_env = "" if conf_buildtools: # Create the dest. directory if it doesn't exist buildtools_dir = get_buildtools_dir() buildtools_dir.mkdir(parents=True, exist_ok=True) if not any(buildtools_dir.iterdir()): # Directory is empty, try to fetch from upstream logging.info(f"Buildtools ({buildtools_dir}): directory is empty") download_buildtools() else: # Fetch buildtools when versions differ in non-empty dir found_version = get_buildtools_version() if found_version != conf_buildtools['version']: logging.warning("Buildtools: version mismatch") logging.info(f"Required version: {conf_buildtools['version']}") logging.info(f"Found version: {found_version}") shutil.rmtree(os.path.realpath(buildtools_dir)) os.makedirs(os.path.realpath(buildtools_dir)) download_buildtools() envfiles = list(get_buildtools_dir().glob("environment-setup-*")) if len(envfiles) == 1: # Ignore missing pkg-config error until oe-core fix is merged buildtools_env = ( "source {} || true\n".format(envfiles[0].resolve()) ) else: logging.error( f"Expected 1 environment setup file, found {len(envfiles)}." "Invalid or misconfigured buildtools package." ) return -1 with tempfile.TemporaryDirectory() as temp_dir: if logging.getLogger().getEffectiveLevel() == logging.DEBUG: init_script_log = pathlib.Path(temp_dir) / '.init_script.log' else: init_script_log = '/dev/null' script = f"""#!/bin/bash set -e {buildtools_env} source {init_script} $1 > {init_script_log} env """ get_bb_env_file = pathlib.Path(temp_dir) / "get_bb_env" get_bb_env_file.write_text(script) get_bb_env_file.chmod(0o775) env = {} env['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin' (_, output) = run_cmd([str(get_bb_env_file), get_context().build_dir], cwd=init_repo.path, env=env) if init_script_log != '/dev/null': with open(init_script_log) as log: msg = f'{init_script} output:\n' for line in log.readlines(): msg += line logging.debug(msg.rstrip('\n')) env = {} for line in output.splitlines(): try: (key, val) = line.split('=', 1) env[key] = val except ValueError: pass conf_env = get_context().config.get_environment() env_vars = ['SSTATE_DIR', 'SSTATE_MIRRORS', 'DL_DIR', 'TMPDIR'] env_vars.extend(conf_env) env.update(conf_env) if 'BB_ENV_PASSTHROUGH_ADDITIONS' in env: passthrough_additions = env['BB_ENV_PASSTHROUGH_ADDITIONS'] + ' ' + \ ' '.join(env_vars) env.update({'BB_ENV_PASSTHROUGH_ADDITIONS': passthrough_additions}) elif 'BB_ENV_EXTRAWHITE' in env: extra_white = env['BB_ENV_EXTRAWHITE'] + ' ' + ' '.join(env_vars) env.update({'BB_ENV_EXTRAWHITE': extra_white}) env_vars.extend(['SSH_AUTH_SOCK', 'SHELL', 'TERM', 'GIT_PROXY_COMMAND', 'NO_PROXY']) for env_var in env_vars: if env_var in os.environ: env[env_var] = os.environ[env_var] # filter out 'None' values env = {k: v for (k, v) in env.items() if v is not None} return env def ssh_add_key_file(env, key_path): """ Adds an ssh key file to the ssh-agent """ with open(key_path) as f: key = f.read() ssh_add_key(env, key) def ssh_add_key(env, key): """ Adds an ssh key to the ssh-agent """ # The ssh-agent needs the key to end with a newline, otherwise it # unhelpfully prompts for a password if not key.endswith('\n'): key += '\n' process = Popen(['ssh-add', '-'], stdin=PIPE, stdout=None, stderr=PIPE, env=env) (_, error) = process.communicate(input=str.encode(key)) if process.returncode and error: logging.error('failed to add ssh key: %s', error.decode('utf-8')) def ssh_cleanup_agent(): """ Removes the identities and stops the ssh-agent instance """ ctx = get_context() # remove the identities (ret, _) = run_cmd(['ssh-add', '-D'], cwd=ctx.kas_work_dir, env=ctx.environ, fail=False) if ret != 0: logging.error('failed to delete SSH identities') # stop the ssh-agent (ret, _) = run_cmd(['ssh-agent', '-k'], cwd=ctx.kas_work_dir, env=ctx.environ, fail=False) if ret != 0: logging.error('failed to stop SSH agent') def ssh_setup_agent(envkeys=None): """ Starts the ssh-agent """ ctx = get_context() env = ctx.environ envkeys = envkeys or ['SSH_PRIVATE_KEY', 'SSH_PRIVATE_KEY_FILE'] (_, output) = run_cmd(['ssh-agent', '-s'], env=env, cwd=ctx.kas_work_dir) for line in output.split('\n'): matches = re.search(r"(\S+)\=(\S+)\;", line) if matches: env[matches.group(1)] = matches.group(2) found = False for envkey in envkeys: if envkey == 'SSH_PRIVATE_KEY_FILE': key_path = os.environ.get(envkey) if key_path: found = True logging.info(f"adding SSH key from file '{key_path}'") ssh_add_key_file(env, key_path) else: key = os.environ.get(envkey) if key: found = True logging.info("adding SSH key from env-var 'SSH_PRIVATE_KEY'") ssh_add_key(env, key) if found is not True: warning = "None of the following environment keys were set: " + \ ", ".join(envkeys) logging.warning(warning) def ssh_no_host_key_check(): """ Disables ssh host key check """ home = os.path.expanduser('~') ssh_dir = home + '/.ssh' if not os.path.exists(ssh_dir): os.mkdir(ssh_dir) ssh_config = ssh_dir + "/config" generated_content = 'Host *\n\tStrictHostKeyChecking no\n\n' try: with open(ssh_config, 'x') as fds: fds.write(generated_content) except FileExistsError: with open(ssh_config, 'r') as fds: content = fds.read() if content != generated_content: logging.warning("%s already exists, " "not touching it to disable StrictHostKeyChecking", ssh_config) def add_cachedir_tag(dir, comment=None): """ Create a CACHEDIR.TAG below dir. If a comment is provided, add this to the tag as well. """ cachetag = pathlib.Path(dir) / 'CACHEDIR.TAG' if cachetag.exists(): return with open(cachetag, 'w') as f: logging.debug(f'create CACHEDIR.TAG in {str(dir)}') f.write('Signature: 8a477f597d28d172789f06886806bc55\n') f.write('# This file is a cache directory tag created by kas.\n' '# For information about cache directory tags, see:\n' '# https://bford.info/cachedir/spec.html\n') if comment: f.write(f'#\n# {comment}\n') def setup_parser_common_args(parser): from kas.libcmds import Macro setup_cmds = [str(s) for (s, _) in Macro().setup_commands] parser.add_argument('--skip', help='Skip build steps. To skip more than one step, ' 'use this argument multiple times.', default=[], action='append', metavar='STEP', choices=setup_cmds) parser.add_argument('--force-checkout', action='store_true', help='Always checkout the desired commit/branch/tag ' 'of each repository, discarding any local changes') parser.add_argument('--update', action='store_true', help='Pull new upstream changes to the desired ' 'branch even if it is already checked out locally') def setup_parser_config_arg(parser): parser.add_argument('config', help='Config file(s), separated by colon. Using ' '.config.yaml in KAS_WORK_DIR if none is ' 'specified.', nargs='?') def setup_parser_preserve_env_arg(parser): parser.add_argument('-E', '--preserve-env', help='Keep current user environment block', action='store_true') class ExtendConstAction(argparse._AppendConstAction): """Add an 'extend_const' action similar to 'append_const'. Based on the existing 'append_const' and 'extend' actions. """ def __init__(self, option_strings, dest, const, default=None, required=False, help=None, metavar=None): super(argparse._AppendConstAction, self).__init__( option_strings=option_strings, dest=dest, nargs=0, const=const, default=default, required=required, help=help, metavar=metavar) def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest, None) if items is None: items = [] if isinstance(items, list): items = items[:] else: import copy items = copy.copy(items) items.extend(self.const) setattr(namespace, self.dest, items) def setup_parser_keep_config_unchanged_arg(parser): # Skip the tasks which would change the config of the build # environment steps = [ 'setup_dir', 'finish_setup_repos', 'repos_checkout', 'repos_apply_patches', 'write_bbconfig', ] parser.add_argument('-k', '--keep-config-unchanged', help='Skip steps that change the configuration', action=ExtendConstAction, dest='skip', const=steps) def run_handle_preserve_env_arg(ctx, os, args, SetupHome): if args.preserve_env: # Warn if there's any settings that setup_home would apply # but are now ignored for var in SetupHome.ENV_VARS: if var in os.environ: logging.warning('Environment variable "%s" ignored ' 'because user environment is being used', var) if not os.isatty(sys.stdout.fileno()): raise EnvNotValidError( '--preserve-env can only be run from a tty') ctx.environ = os.environ.copy() logging.warning("Preserving the current environment block may " "have unintended side effects on the build.") siemens-kas-41ad961/kas/plugins/000077500000000000000000000000001520561422700165275ustar00rootroot00000000000000siemens-kas-41ad961/kas/plugins/__init__.py000066400000000000000000000042201520561422700206360ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains and manages kas plugins """ PLUGINS = {} def register_plugins(mod): """ Register all kas plugins found in a module """ for plugin in getattr(mod, '__KAS_PLUGINS__', []): PLUGINS[plugin.name] = plugin def load(): """ Import all kas plugins """ # sort alphabetically from . import build from . import checkout from . import clean from . import diff from . import dump from . import for_all_repos from . import lock from . import menu from . import shell register_plugins(build) register_plugins(checkout) register_plugins(clean) register_plugins(diff) register_plugins(dump) register_plugins(for_all_repos) register_plugins(lock) register_plugins(menu) register_plugins(shell) def get(name): """ Lookup a kas plugin class by name """ return PLUGINS.get(name, None) def all(): """ Get a list of all loaded kas plugin classes """ return PLUGINS.values() siemens-kas-41ad961/kas/plugins/build.py000066400000000000000000000140611520561422700202020ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas build`` command. When this command is executed, kas will checkout repositories, setup the build environment and then invoke bitbake to build the targets selected in the chosen config file. When running with ``--provenance `` kas will generate an provenance attestation for the build. The attestation will be stored in ``attestation/kas-build.provenance.json`` in the build directory. For details about provenance, see the build attestation chapter. .. note:: In provenance mode, the command returns with a non-zero exit code in case no artifact is found for at least one entry. """ import logging import subprocess import sys import json import asyncio from pathlib import Path from datetime import datetime from kas.context import create_global_context from kas.config import Config from kas.libkas import find_program, run_cmd_async from kas.libkas import setup_parser_keep_config_unchanged_arg from kas.libcmds import Macro, Command from kas.libkas import setup_parser_common_args, setup_parser_config_arg from kas.kasusererror import CommandExecError from kas.attestation import Provenance, Statement __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2024' class Build: """ This class implements the build plugin for kas. """ name = 'build' helpmsg = ( 'Checks out all necessary repositories and builds using bitbake as ' 'specified in the configuration file.' ) @classmethod def setup_parser(cls, parser): """ Setup the argument parser for the build plugin """ setup_parser_common_args(parser) setup_parser_config_arg(parser) setup_parser_keep_config_unchanged_arg(parser) parser.add_argument('extra_bitbake_args', nargs='*', help='Extra arguments to pass to bitbake ' '(typically requires separation via \'--\')') parser.add_argument('--target', action='append', help='Select target to build') parser.add_argument('-c', '--cmd', '--task', dest='task', help='Select which task should be executed') parser.add_argument('--provenance', choices=['true', 'mode=min', 'mode=max'], help='Enable provenance attestation generation') def run(self, args): """ Executes the build command of the kas plugin. """ if args.config and args.config.startswith('-'): args.extra_bitbake_args.insert(0, args.config) args.config = None ctx = create_global_context(args) ctx.config = Config(ctx, args.config, args.target, args.task) macro = Macro() macro.add(BuildCommand(args.extra_bitbake_args)) macro.run(ctx, args.skip) class BuildCommand(Command): """ Implements the bitbake build step. """ def __init__(self, extra_bitbake_args): super().__init__() self.extra_bitbake_args = extra_bitbake_args def __str__(self): return 'build' def _generate_attestation(self, ctx, time_started, time_finished, mode): """ Generate the provenance attestation for the build. """ predicate = Provenance(ctx, time_started, time_finished, mode) stmt = Statement(predicate, ctx, time_started, time_finished).as_dict() att_dir = Path(ctx.build_dir) / 'attestation' att_dir.mkdir(parents=True, exist_ok=True) with open(att_dir / 'kas-build.provenance.json', 'w') as f: f.write(json.dumps(stmt, indent=4)) f.write('\n') def execute(self, ctx): """ Executes the bitbake build command. """ # Start bitbake build of image bitbake = find_program(ctx.environ['PATH'], 'bitbake') cmd = [bitbake, '-c', ctx.config.get_bitbake_task()] \ + self.extra_bitbake_args + ctx.config.get_bitbake_targets() time_started = datetime.now() if sys.stdout.isatty(): logging.info('%s$ %s', ctx.build_dir, ' '.join(cmd)) ret = subprocess.call(cmd, env=ctx.environ, cwd=ctx.build_dir) if ret != 0: raise CommandExecError(cmd, ret) else: loop = asyncio.get_event_loop() loop.run_until_complete(run_cmd_async(cmd, cwd=ctx.build_dir, liveupdate=True)) time_finished = datetime.now() if ctx.args.provenance: mode = Provenance.Mode.MAX if ctx.args.provenance == 'mode=max' \ else Provenance.Mode.MIN self._generate_attestation(ctx, time_started, time_finished, mode) __KAS_PLUGINS__ = [Build] siemens-kas-41ad961/kas/plugins/checkout.py000066400000000000000000000045611520561422700207140ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Konsulko Group, 2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas checkout`` command. When this command is executed, kas will checkout repositories and set up the build directory as specified in the chosen config file. This command is useful if you need to inspect the configuration or modify any of the checked out layers before starting a build. For example, to setup the configuration described in the file ``kas-project.yml`` you could run:: kas checkout kas-project.yml """ from kas.context import create_global_context from kas.config import Config from kas.libcmds import Macro from kas.libkas import setup_parser_common_args, setup_parser_config_arg __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' class Checkout: name = 'checkout' helpmsg = ( 'Checks out all necessary repositories and sets up the build ' 'directory as specified in the configuration file.' ) @classmethod def setup_parser(cls, parser): setup_parser_common_args(parser) setup_parser_config_arg(parser) def run(self, args): ctx = create_global_context(args) ctx.config = Config(ctx, args.config) macro = Macro() macro.run(ctx, args.skip) __KAS_PLUGINS__ = [Checkout] siemens-kas-41ad961/kas/plugins/clean.py000066400000000000000000000225241520561422700201700ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens, 2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas {clean,cleansstate,cleanall}`` commands. In case a configuration file is provided, it will be used to determine the build system and the files managed by kas. """ import os import shutil import logging import subprocess from pathlib import Path from kas.context import create_global_context, get_context from kas.config import Config, CONFIG_YAML_FILE from kas.includehandler import ConfigFile from kas.libcmds import Macro from kas.kasusererror import KasUserError from kas import plugins __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens, 2025' class Clean(): """ Clean the build artifacts by removing the build artifacts directory. """ name = 'clean' helpmsg = ( 'Clean build artifacts, keep sstate cache and downloads.' ) config_files = None @classmethod def setup_parser(cls, parser): parser.add_argument('--dry-run', action='store_true', default=False, help='Do not remove anything, just print what ' 'would be removed') parser.add_argument('--isar', action='store_true', default=False, help='Use ISAR build directory layout') parser.add_argument('config', help='Config file(s), separated by colon. Using ' '.config.yaml in KAS_WORK_DIR if existing ' 'and none is specified.', nargs='?') def run(self, args): ctx = create_global_context(args) default_conf_file = Path(ctx.kas_work_dir) / CONFIG_YAML_FILE build_system = None if args.config: self.config_files = args.config elif default_conf_file.exists(): self.config_files = str(default_conf_file) if self.config_files: # By definition, build_system key must be present in the first # config file to take effect. cf = ConfigFile.load(self.config_files.split(':')[0]) build_system = cf.config.get('build_system') if args.isar: build_system = 'isar' logging.debug('Run clean in "%s" mode' % (build_system or 'default')) if args.dry_run: logging.warning('Dry run, not removing anything') tmpdirs = Path(ctx.build_dir).glob('tmp*') dirs_to_remove = [] for tmpdir in tmpdirs: logging.info(f'Removing {tmpdir}') if build_system == 'isar': dirs_to_remove.append(tmpdir) else: if not args.dry_run: shutil.rmtree(tmpdir) if len(dirs_to_remove) == 0: return clean_args = ['sudo', '--prompt', '[sudo] enter password for %U ' 'to clean ISAR artifacts'] clean_args.extend(['rm', '-rf']) clean_args.extend([p.as_posix() for p in dirs_to_remove]) logging.debug(' '.join(clean_args)) if not args.dry_run: subprocess.check_call(clean_args) @staticmethod def clear_dir_content(directory): """ Clear the contents of a directory without removing the dir itself. """ for item in directory.iterdir(): if item.is_dir(): shutil.rmtree(item) else: item.unlink() class CleanSstate(Clean): """ Removes the build artifacts and the empties the sstate cache. """ name = 'cleansstate' helpmsg = ( 'Clean build artifacts and sstate cache.' ) def run(self, args): super().run(args) ctx = get_context() sstate_dir = Path(os.environ.get('SSTATE_DIR', Path(ctx.build_dir) / 'sstate-cache')) if sstate_dir.exists(): logging.info(f'Removing {sstate_dir}/*') if not args.dry_run: self.clear_dir_content(sstate_dir) class CleanAll(CleanSstate): """ Removes the build artifacts, empties the sstate cache and the downloads. """ name = 'cleanall' helpmsg = ( 'Clean build artifacts, sstate-cache and downloads.' ) def run(self, args): super().run(args) ctx = get_context() downloads_dir = Path(os.environ.get('DL_DIR', Path(ctx.build_dir) / 'downloads')) if downloads_dir.exists(): logging.info(f'Removing {downloads_dir}/*') if not args.dry_run: self.clear_dir_content(downloads_dir) class Purge(CleanAll): """ Clears the contents of the build directory, sstate-cache, downloads and the repos managed by kas (including referenced repos in ``KAS_REPO_REF_DIR``, if set). In ``KAS_WORK_DIR`` it will remove the default configuration file and the ``KAS_BUILD_DIR`` (if present). To preserve the reference repositories, run with ``--preserve-repo-refs``. This command requires a configuration file to locate the managed repos. .. note:: Before purging, kas needs to checkout and resolve all repos to locate the repos managed by kas. """ name = 'purge' helpmsg = ( 'Purge all data managed by kas, including managed repos.' ) @classmethod def setup_parser(cls, parser): super().setup_parser(parser) parser.add_argument('--preserve-repo-refs', action='store_true', default=False, help='Do not remove the reference repositories') def run(self, args): super().run(args) ctx = get_context() if not self.config_files: raise KasUserError('Purge requires a config file to locate ' 'managed repos.') ctx.config = Config(ctx, self.config_files) # to read the config, we need all repos (but no build env), macro = Macro() macro.run(ctx, skip=['repos_apply_patches', 'write_bb_config', 'setup_environ']) for r in ctx.config.get_repos(): if r.operations_disabled: logging.debug(f'Skipping {r.name} as not managed by kas') continue logging.info(f'Removing {r.path}') if not args.dry_run: shutil.rmtree(r.path) if ctx.kas_repo_ref_dir and not args.preserve_repo_refs: ref_repo = Path(ctx.kas_repo_ref_dir) / r.qualified_name if ref_repo.exists(): logging.info(f'Removing {ref_repo}') if not args.dry_run: shutil.rmtree(ref_repo) build_dir = Path(ctx.build_dir) logging.info(f'Removing {build_dir}/*') if not args.dry_run: self.clear_dir_content(build_dir) if os.environ.get("KAS_BUILDTOOLS_DIR"): buildtools_dir = ( Path(os.environ.get("KAS_BUILDTOOLS_DIR")).resolve() ) if buildtools_dir.exists(): logging.info(f'Removing {buildtools_dir}') if not args.dry_run: shutil.rmtree(buildtools_dir) work_dir = Path(ctx.kas_work_dir) default_config = work_dir / CONFIG_YAML_FILE if default_config.exists(): logging.info(f'Removing {default_config}') if not args.dry_run: default_config.unlink() clean_paths = list(ctx.managed_paths) # Plugins can register additional paths by providing get_managed_paths. # These paths must be relative to the work_dir. for plugin in plugins.all(): if hasattr(plugin, 'get_managed_paths'): ppaths = plugin.get_managed_paths() clean_paths.extend([work_dir / p for p in ppaths]) for path in [Path(p) for p in clean_paths if Path(p).exists()]: logging.info(f'Removing {path}') if not args.dry_run: if path.is_file() or path.is_symlink(): path.unlink() else: shutil.rmtree(path) __KAS_PLUGINS__ = [Clean, CleanSstate, CleanAll, Purge] siemens-kas-41ad961/kas/plugins/diff.py000066400000000000000000000265321520561422700200210ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens, 2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas diff`` command. This plugin compares two kas configurations and outputs the differences. The diff includes both content differences in the configuration files and repository differences if commit IDs or tags have changed. Additionally, you can use the ``--format json`` option to output the diff in JSON format. .. note:: The text output of the plugin should not considered to be stable. If stable output is needed, use a machine readable format like json. """ import json import difflib from kas.context import create_global_context from kas.config import Config from kas.libcmds import Macro from kas.libkas import setup_parser_common_args __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens, 2025' class Diff: """ kas plugin to compute diff of two kas configurations. """ name = 'diff' helpmsg = ( 'Compare two kas configurations.' ) @classmethod def setup_parser(cls, parser): setup_parser_common_args(parser) parser.add_argument('config1', help='The first config file to be compared.') parser.add_argument('config2', help='The second config file to be compared.') parser.add_argument('--format', choices=['json', 'text'], default='text', help='Diff output format (default: text)') parser.add_argument('--oneline', action='store_true', help='Use git oneline output for differing ' 'commits.') parser.add_argument('--no-color', action='store_true', help='Disable colored highlighting of diffs.') parser.add_argument('--commit-only', action='store_true', help='This will not display the differences in ' 'the kas configurations; it will only list ' 'commits resulting from different ' 'repository revisions.') parser.add_argument('--content-only', action='store_true', help='This will only display the differences in ' 'the kas configurations and will not ' 'include repository differences.') @staticmethod def compare_dicts(dict1, dict2, parent_key=''): """ Deep compare dictionaries. Returns a dictionary with the differences. """ diff = {"values_changed": {}} def add_change(key, old_value, new_value): diff["values_changed"][key] = { "old_value": old_value, "new_value": new_value } def deep_compare(d1, d2, parent_key): keys = set(d1.keys()).union(set(d2.keys())) for key in keys: full_key = f"{parent_key}.{key}" if parent_key else key if key in d1 and key in d2: if isinstance(d1[key], dict) and isinstance(d2[key], dict): deep_compare(d1[key], d2[key], full_key) elif d1[key] != d2[key]: add_change(full_key, d1[key], d2[key]) elif key in d1: add_change(full_key, d1[key], None) else: add_change(full_key, None, d2[key]) deep_compare(dict1, dict2, parent_key) return diff @staticmethod def formatting_diff_output(oldfile, newfile, diff_output, oneline, no_color, commit_only, content_only): """ Format the diff output. """ def print_unified_diff(oldv, newv, key): # unified diff expects newline terminated input strings if '\n' not in oldv: oldv += '\n' if '\n' not in newv: newv += '\n' diff = difflib.unified_diff(oldv.splitlines(keepends=True), newv.splitlines(keepends=True)) print(f"{' ' * 5}{key}:") for line in diff: if line.startswith('+++') or line.startswith('---'): continue elif line.startswith('@@'): print(f"{COLORS_FILES}{' ' * 9}{line}{COLORS_ENDC}", end='') elif line[0] == '+': print(f"{COLORS_NEW}+{' ' * 8}{line[1:]}{COLORS_ENDC}", end='') elif line[0] == '-': print(f"{COLORS_OLD}-{' ' * 8}{line[1:]}{COLORS_ENDC}", end='') else: print(f"{' ' * 8}{line}", end='') if no_color: COLORS_OLD = '' COLORS_NEW = '' COLORS_COMMIT = '' COLORS_FILES = '' COLORS_ENDC = '' else: COLORS_OLD = '\033[31m' COLORS_NEW = '\033[32m' COLORS_COMMIT = '\033[33m' COLORS_FILES = '\033[34m' COLORS_ENDC = '\033[0m' prefix_old = "-" prefix_new = "+" format_dict = {'old_value': {'color': COLORS_OLD, 'prefix': prefix_old}, 'new_value': {'color': COLORS_NEW, 'prefix': prefix_new}} if oldfile or newfile: print(f"kas diff {oldfile} {newfile}") print(prefix_old * 3, f" {oldfile}") print(prefix_new * 3, f" {newfile}") vc_dict = diff_output.get('values_changed', {}) vcs_dict = diff_output.get('vcs', {}) if vc_dict and not commit_only: print(f"{COLORS_FILES}@@ config changed @@{COLORS_ENDC}") for key in vc_dict.keys(): oldval = vc_dict[key]['old_value'] newval = vc_dict[key]['new_value'] if isinstance(oldval, str) and isinstance(newval, str): print_unified_diff(oldval, newval, key) continue for k, i in format_dict.items(): value = vc_dict[key][k] if not value: continue value = str(value).replace('\n', f"\n{i['prefix']}") print(f"{i['color']}" f"{i['prefix']}{' ' * 4}{key}: {value}" f"{COLORS_ENDC}") if vcs_dict and not content_only: for key in vcs_dict.keys(): print(f"{COLORS_FILES}@@ {key} commits diff @@{COLORS_ENDC}") for li in vcs_dict[key]: msg = li['message'] if oneline: msg = msg.split('\n')[0] print(f"{COLORS_COMMIT}" f"{li['commit'][:7]}{COLORS_ENDC}" f" {msg}") else: print(f"{COLORS_COMMIT}{li['commit']}: {COLORS_ENDC}" f"{li['author']} {li['commit_date']}") indented_msg = '\n'.join([' ' * 4 + s for s in msg.split('\n')]) print(indented_msg) if key != list(vcs_dict.keys())[-1]: print("---") def run(self, args): args.skip += [ 'setup_environ', 'write_bbconfig' ] ctx = create_global_context(args) ctx.config = Config(ctx, args.config1) macro = Macro() macro.run(ctx, args.skip) config1 = ctx.config.get_config(remove_includes=True, apply_overrides=True) repos1 = ctx.config.get_repos() args.skip += [ 'setup_dir', 'setup_home', 'setup_ssh_agent' ] ctx.config = Config(ctx, args.config2) macro.run(ctx, args.skip) config2 = ctx.config.get_config(remove_includes=True, apply_overrides=True) repos2 = ctx.config.get_repos() diff = Diff.compare_dicts(config1, config2) diff_output = {} if diff: # combine the diff output and the repo diff to a dict diff_output.update(diff) vcs_dict = {} # check for commit IDs/tags of repos and compare them for key in diff.get('values_changed', {}): if 'commit' in key or 'tag' in key: commit1 = diff['values_changed'][key]['old_value'] commit2 = diff['values_changed'][key]['new_value'] repo_name_arr = key.split('.') if len(repo_name_arr) < 3 or repo_name_arr[0] != 'repos': continue repo_name = repo_name_arr[1] # we do not known which config contains the latest commit, # so we need to check both configs for each_repo in repos1: if each_repo.path and each_repo.url \ and each_repo.name == repo_name: repo_diff = each_repo.diff(commit2, commit1) if len(repo_diff.get(each_repo.name)) > 0: vcs_dict.update(repo_diff) for each_repo in repos2: if each_repo.path and each_repo.url \ and each_repo.name == repo_name: repo_diff = each_repo.diff(commit1, commit2) if len(repo_diff.get(each_repo.name)) > 0: vcs_dict.update(repo_diff) if vcs_dict: diff_output['vcs'] = vcs_dict if args.format == 'json': print(json.dumps(diff_output, indent=4)) else: Diff.formatting_diff_output(args.config1, args.config2, diff_output, args.oneline, args.no_color, args.commit_only, args.content_only) __KAS_PLUGINS__ = [Diff] siemens-kas-41ad961/kas/plugins/dump.py000066400000000000000000000254661520561422700200630ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas dump`` command. When this command is executed in default mode, kas will parse all referenced config files, expand includes and print a flattened yaml version of the configuration to stdout. This config is semantically identical to the input, but does not include any references to other configuration files. The output of this command can be used to further analyze the build configuration. When running with ``--lock``, a locking spec is created which only contains the exact commit of each repository. This can be used to pin the commit of floating branches and tags, while still keeping an easy update path. For details on the locking support, see :class:`kas.plugins.lock`. .. note:: The options to create and update lock files have been moved to the lock plugin. When running with ``--resolve-local``, VCS tracking information of the root repo (the one with the kas-project.yml) is added to the output. The generated file can be used as single input to kas to reproduce the build environment. If the root repo is not under version control or contains uncommitted changes, a warning is emitted. Please note: - the dumped config is semantically identical but not bit-by-bit identical - all referenced repositories are checked out to resolve cross-repo configs - all branches are resolved before patches are applied - the ordering of the keys is kept unless ``--sort`` is used. If you intend to store the flattened configs for comparison, it is recommended to sort the keys. For example, to get a single config representing the final build config of ``kas-project.yml:target-override.yml`` you could run:: kas dump kas-project.yml:target-override.yml > kas-project-expanded.yml The generated config can be used as input for kas:: kas build kas-project-expanded.yml """ import sys import json import yaml import logging import argparse from typing import TypeVar, TextIO from collections import OrderedDict from kas.context import get_context from kas.plugins.checkout import Checkout from kas.kasusererror import KasUserError, ArgsCombinationError __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2022' LOCKFILE_VERSION_MIN = 14 SCHEMA_VERSION_MIN = 7 class OutputFormatError(KasUserError): def __init__(self, format): super().__init__(f'invalid format {format}') class IoTarget: StrOrTextIO = TypeVar('StrOrTextIO', str, TextIO) target: StrOrTextIO managed: bool def __init__(self, target, managed): self.target = target self.managed = managed class IoTargetMonitor: """ Simple monitor to unify access to file targets that need to be closed (files) and ambient ones (stdout / stderr) """ def __init__(self, target: IoTarget): self._target = target self._file = None def __enter__(self): if self._target.managed: self._file = open(self._target.target, 'w') return self._file return self._target.target def __exit__(self, exc_type, exc_value, traceback): if self._target.managed: self._file.close() class Dump(Checkout): """ Implements a kas plugin that combines multiple kas configurations and dumps the result. """ name = 'dump' helpmsg = ( 'Expand and dump the final config to stdout. When resolving branches, ' 'this is done before patches are applied.' ) class KasYamlDumper(yaml.Dumper): """ Yaml formatter (dumper) that generates output in a formatting which is similar to kas example input files. """ def represent_data(self, data): if isinstance(data, str): if data.count('\n') > 0: return self.represent_scalar( 'tag:yaml.org,2002:str', data, style='|') return self.represent_scalar('tag:yaml.org,2002:str', data) elif isinstance(data, OrderedDict): return self.represent_mapping( 'tag:yaml.org,2002:map', data.items()) elif data is None: return self.represent_scalar('tag:yaml.org,2002:null', '') return super().represent_data(data) @staticmethod def setup_parser_format_args(parser): parser.add_argument('--indent', type=int, default=2, help='Line indent (# of spaces, default: 2)') parser.add_argument('--sort', action='store_true', default=False, help='Alphanumerically sort keys in output') @classmethod def setup_parser(cls, parser): super().setup_parser(parser) Dump.setup_parser_format_args(parser) lk_or_env = parser.add_mutually_exclusive_group() parser.add_argument('--format', choices=['yaml', 'json'], default='yaml', help='Output format (default: yaml)') parser.add_argument('--resolve-refs', action='store_true', help='Replace floating refs with exact SHAs. ' 'Overrides are removed') parser.add_argument('--resolve-local', action='store_true', help='Add tracking information of root repo') lk_or_env.add_argument('--resolve-env', action='store_true', help='Set env defaults to captured env value') lk_or_env.add_argument('--lock', action='store_true', help='Create lockfile with exact SHAs') parser.add_argument('-i', '--inplace', action='store_true', help=argparse.SUPPRESS) @staticmethod def dump_config(config: dict, target: IoTarget, format: str, indent: int, sorted: bool): """ Dump the configuration to the target in the specified format. """ with IoTargetMonitor(target) as f: if format == 'json': json.dump(config, f, indent=indent, sort_keys=sorted) f.write('\n') elif format == 'yaml': yaml.dump( config, f, indent=indent, sort_keys=sorted, Dumper=Dump.KasYamlDumper) else: raise OutputFormatError(format) def run(self, args): def _filter_enabled(repos): return [(k, r) for k, r in repos if not r.operations_disabled] def _filter_local(repos): return [(k, r) for k, r in repos if r.operations_disabled and r.name] args.skip += [ 'setup_dir', 'repos_apply_patches', 'setup_environ', 'write_bbconfig', ] super().run(args) ctx = get_context() schema_v = LOCKFILE_VERSION_MIN if args.lock else SCHEMA_VERSION_MIN config_expanded = {'header': {'version': schema_v}} if args.lock \ else ctx.config.get_config(remove_includes=True) repos = ctx.config.repo_dict.items() output = IoTarget(target=sys.stdout, managed=False) if args.inplace and not args.lock: raise ArgsCombinationError('--inplace requires --lock') if args.resolve_local and args.lock: raise ArgsCombinationError( '--resolve-local cannot be used with --lock') if args.inplace and args.lock: from kas.plugins.lock import Lock logging.warning('The --inplace option is deprecated. ' 'Migrate to the "lock" command.') return Lock().run(args) if args.lock: args.resolve_refs = True # when locking, only consider repos managed by kas repos = _filter_enabled(repos) config_expanded['overrides'] = \ {'repos': {k: {'commit': r.revision} for k, r in repos}} if args.resolve_refs and not args.lock: for k, r in _filter_enabled(repos): if r.commit or r.branch or r.tag: config_expanded['repos'][k]['commit'] = r.revision elif r.refspec: config_expanded['repos'][k]['refspec'] = r.revision # as the refs are resolved, the overrides are redundant if 'overrides' in config_expanded: del config_expanded['overrides'] if args.resolve_local: for k, r in _filter_local(repos): if r.revision: if r.dirty: logging.warning(f'Repository {r.name} (root repo) ' 'contains uncommitted changes.') if config_expanded['repos'][k] is None: config_expanded['repos'][k] = {} config_expanded['repos'][k]['url'] = r.url config_expanded['repos'][k]['commit'] = r.revision else: logging.warning(f'Repository {r.name} (root repo) ' 'is not under version control.') if args.resolve_env and 'env' in config_expanded: config_expanded['env'] = ctx.config.get_environment() self.dump_config(config_expanded, output, args.format, args.indent, args.sort) __KAS_PLUGINS__ = [Dump] siemens-kas-41ad961/kas/plugins/for_all_repos.py000066400000000000000000000127271520561422700217400ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Konsulko Group, 2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas for-all-repos`` command. When this command is executed, kas will checkout the repositories listed in the chosen config file and then execute a specified command in each repository. It can be used to query the repository status, automate actions such as archiving the layers used in a build or to execute any other required commands. For example, to print the commit hashes used by each repository used in the file ``kas-project.yml`` (assuming they are all git repositories) you could run:: kas for-all-repos kas-project.yml 'git rev-parse HEAD' The environment for executing the command in each repository is extended to include the following variables: * ``KAS_REPO_NAME``: The name of the current repository determined by either the name property or by the key used for this repo in the config file. * ``KAS_REPO_PATH``: The path of the local directory where this repository is checked out, relative to the directory where ``kas`` is executed. * ``KAS_REPO_URL``: The URL from which this repository was cloned, or an empty string if no remote URL was given in the config file. * ``KAS_REPO_COMMIT``: The commit ID which was checked out for this repository, or an empty string if no commit was given in the config file. * ``KAS_REPO_BRANCH``: The branch which was checked out for this repository, or an empty string if no branch was given in the config file. * ``KAS_REPO_TAG``: The tag which was checked out for this repository, or an empty string if no tag was given in the config file. * ``KAS_REPO_REFSPEC``: The refspec which was checked out for this repository, or an empty string if no refspec was given in the config file. This variable is obsolete and will be removed when support for refspec keys is removed as well. Migrate your repos to commit/branch and use the related variables instead. """ import logging import os import subprocess from kas.context import create_global_context from kas.config import Config from kas.libcmds import Macro, Command, SetupHome from kas.libkas import setup_parser_common_args, setup_parser_config_arg from kas.libkas import setup_parser_keep_config_unchanged_arg from kas.libkas import setup_parser_preserve_env_arg from kas.libkas import run_handle_preserve_env_arg from kas.kasusererror import CommandExecError __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' class ForAllRepos: name = 'for-all-repos' helpmsg = ( 'Runs a specified command in all checked out repositories.' ) @classmethod def setup_parser(cls, parser): setup_parser_common_args(parser) setup_parser_config_arg(parser) setup_parser_preserve_env_arg(parser) setup_parser_keep_config_unchanged_arg(parser) parser.add_argument('command', help='Command to be executed as a string.') def run(self, args): ctx = create_global_context(args) ctx.config = Config(ctx, args.config) run_handle_preserve_env_arg(ctx, os, args, SetupHome) macro = Macro() macro.add(ForAllReposCommand(args.command)) macro.run(ctx, args.skip) class ForAllReposCommand(Command): def __init__(self, command): super().__init__() self.command = command def __str__(self): return 'for-all-repos' def execute(self, ctx): for repo in ctx.config.get_repos(): env = { **ctx.environ, 'KAS_REPO_NAME': repo.name, 'KAS_REPO_PATH': repo.path, 'KAS_REPO_URL': '' if repo.operations_disabled else repo.url, 'KAS_REPO_COMMIT': '' if repo.operations_disabled else (repo.commit or ''), 'KAS_REPO_BRANCH': repo.branch or '', 'KAS_REPO_TAG': repo.tag or '', 'KAS_REPO_REFSPEC': repo.refspec or '', } logging.info('%s$ %s', repo.path, self.command) retcode = subprocess.call(self.command, shell=True, cwd=repo.path, env=env) if retcode != 0: raise CommandExecError([self.command], retcode) __KAS_PLUGINS__ = [ForAllRepos] siemens-kas-41ad961/kas/plugins/lock.py000066400000000000000000000217741520561422700200440ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas lock`` command. When this command is executed a locking spec is created which only contains the exact commit of each repository. This is used to pin the commit of floating branches and tags, while still keeping an easy update path. The lockfile is created next to the first file on the kas cmdline. For details on the locking support, see :class:`kas.includehandler.IncludeHandler`. .. note :: * all referenced repositories are checked out to resolve cross-repo configs * all branches are resolved before patches are applied **Updating lockfiles** When updating lockfiles, kas attempts to update the repository revisions in the lockfile that defines the revision. If a repository is exclusively locked in an external lockfile, this lock is not updated (we cannot modify an external repository). However, if the revision is also defined in a local lockfile, it is updated in the local lockfile. The algorithm for determining where to pin the revision of a repository is as follows: #. Find all repositories that have a floating ref (i.e. no commit). Assign to `to-lock` list #. Iterate over all lockfiles (in include order) #. for each repository, check if it is locked in the current file #. if lock is up to date, remove from `to-lock` list #. else if lockfile is internal, update lockfile, remove from `to-lock` list #. else (repo is locked in external lockfile): mark repo external #. Remove all repos with `external` marks from `to-lock` list #. Add all remaining repos in `to-lock` list to topmost lockfile, create if needed **Examples** The lockfile is created as ``kas-project.lock.yml``. Call again to regenerate lockfile:: kas lock --update kas-project.yml The generated lockfile will automatically be used to pin the revisions:: kas build kas-project.yml Note, that the lockfiles should be checked-in into the VCS. """ import logging import os from dataclasses import dataclass from kas.context import get_context from kas.includehandler import ConfigFile from kas.plugins.checkout import Checkout from kas.plugins.dump import Dump, IoTarget, LOCKFILE_VERSION_MIN from kas.plugins.diff import Diff from kas.repos import Repo, RepoRefError __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2024' class Lock(Checkout): """ Implements a kas plugin to create and update kas project lockfiles. """ name = 'lock' helpmsg = ( 'Create and update kas project lockfiles.' ) @dataclass class RepoInfo(): repo: Repo ext_lock: bool @classmethod def setup_parser(cls, parser): super().setup_parser(parser) Dump.setup_parser_format_args(parser) def _print_log_diff(self, repo, old_commit): try: diff = repo.diff(old_commit, None) except NotImplementedError: return except RepoRefError as e: logging.warning(e) return Diff.formatting_diff_output( None, None, {'vcs': diff}, True, False, True, False) def _update_lockfile(self, lockfile, repos_to_lock, update_only, args): """ Update all locks in the given lockfile. If update_only, no new locks are added. """ output = IoTarget(target=lockfile, managed=True) lockfile_config = lockfile.config changed = False if 'overrides' not in lockfile_config: lockfile_config['overrides'] = {'repos': {}} if 'repos' not in lockfile_config['overrides']: lockfile_config['overrides']['repos'] = {} lock_header_vers = lockfile_config['header']['version'] if lock_header_vers < LOCKFILE_VERSION_MIN: logging.warning('Lockfile uses too-old header version (%s). ' 'Updating to version %d', lock_header_vers, LOCKFILE_VERSION_MIN) lockfile_config['header']['version'] = LOCKFILE_VERSION_MIN for k, v in lockfile_config['overrides']['repos'].items(): ri = repos_to_lock.get(k) if not ri: continue r = ri.repo if v['commit'] == r.revision or \ v['commit'] == r.revision_annotated_tag: logging.info('Lock of %s is up-to-date: %s', r.name, v['commit']) elif not lockfile.is_external: logging.info('Updating lock of %s: %s -> %s', r.name, v['commit'], r.revision) self._print_log_diff(r, v['commit']) v['commit'] = r.revision changed = True else: logging.warning( 'Repo %s is locked in remote lockfile %s. ' 'Not updating.', r.name, lockfile.filename) ri.ext_lock = True continue del repos_to_lock[k] if not update_only: for k, ri in repos_to_lock.items(): r = ri.repo logging.info('Adding lock of %s: %s', r.name, r.revision) lockfile_config['overrides']['repos'][k] = \ {'commit': r.revision} changed = True if not changed: return repos_to_lock logging.info('Updating lockfile %s', os.path.relpath(lockfile.filename, os.getcwd())) output = IoTarget(target=lockfile.filename, managed=True) format = "json" if lockfile.filename.suffix == '.json' else "yaml" Dump.dump_config(lockfile_config, output, format, args.indent, args.sort) return repos_to_lock def run(self, args): def _filter_enabled(repos): return [(k, r) for k, r in repos if not r.operations_disabled] args.skip += [ 'setup_dir', 'repos_apply_patches', 'setup_environ', 'write_bbconfig', ] super().run(args) ctx = get_context() repos_cfg = ctx.config.repo_dict.items() # when locking, only consider floating repos managed by kas # Important: process repos in the order they are defined in the # config file to update lockfile with highest precedence. repos_to_lock = dict([(k, self.RepoInfo(r, False)) for k, r in _filter_enabled(repos_cfg) if not r.commit]) if not repos_to_lock: logging.info('No floating repos found. Nothing to lock.') return # first update all locks we have without creating new ones lockfiles = ctx.config.get_lockfiles() for lock in lockfiles: repos_to_lock = self._update_lockfile(lock, repos_to_lock, True, args) # remove repos that are externally locked repos_to_lock = {k: v for k, v in repos_to_lock.items() if not v.ext_lock} # then add new locks for the remaining repos to the default lockfile if repos_to_lock: repo_to_lock_names = [r.repo.name for r in repos_to_lock.values()] logging.warning('The following repos are not covered by any ' 'lockfile. Adding to top lockfile: %s', ', '.join(repo_to_lock_names)) lockpath = ctx.config.handler.get_lock_filename() if lockfiles and lockfiles[0].filename == lockpath: lock = lockfiles[0] else: lock = ConfigFile(lockpath, is_external=False, is_lockfile=True) lock.config['header'] = {'version': LOCKFILE_VERSION_MIN} self._update_lockfile(lock, repos_to_lock, False, args) __KAS_PLUGINS__ = [Lock] siemens-kas-41ad961/kas/plugins/menu.py000066400000000000000000000445441520561422700200600ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2021 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # # Parts of this were based on kconfiglib, examples/menuconfig_example.py # # Copyright (c) 2011-2019, Ulf Magnusson # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # """ This plugin implements the ``kas menu`` command. When this command is executed, kas will open a configuration menu as described by a Kconfig file. It processes any pre-existing configuration file with saved settings, stores the final selections and invokes the build plugin if requested by the user. To make use of this plugin, a ``Kconfig`` file has to be provided. The menu can define these types of configuration variables that the plugin will translate into a kas configuration: - kas configuration files that will be included when building the generated configuration. Those are picked up from Kconfig string variables that have the name prefix ``KAS_INCLUDE_``. - bitbake targets that shall be built via the generated configuration. Those are picked up from Kconfig string variables that have the name prefix ``KAS_TARGET_``. - The ``build_system`` that will used. The static Kconfig string variable ``KAS_BUILD_SYSTEM`` defines this value which must be ``openembedded``, ``oe`` or ``isar`` is set. - bitbake configuration variables that will be added to the ``local_conf_header`` section of the generated configuration. All other active Kconfig string, integer or hex variables are treated as such. See https://www.kernel.org/doc/html/latest/kbuild/kconfig-language.html for a complete documentation of the Kconfig language. The menu plugin writes the selected configuration to a ``.config.yaml`` file in the kas work directory and also reads previous selection from such a file if it exists. The ``.config.yaml`` both contains the selected configuration in the ``menu_configuration`` key and also the effective settings that can be used to invoke ``kas build`` or other kas commands. """ import logging import os import pprint import yaml from kas import __version__, __file_version__ from kas.context import create_global_context from kas.config import CONFIG_YAML_FILE from kas.repos import Repo from kas.includehandler import ConfigFile, \ SOURCE_DIR_OVERRIDE_KEY, SOURCE_DIR_HOST_OVERRIDE_KEY from kas.plugins.build import Build from kas.kasusererror import KasUserError, MissingModuleError try: from kconfiglib import Kconfig, Symbol, Choice, KconfigError, \ expr_value, TYPE_TO_STR, MENU, COMMENT, STRING, BOOL, INT, HEX, UNKNOWN HAVE_KCONFIGLIB = True except ImportError: HAVE_KCONFIGLIB = False # will be reported in run() try: from snack import SnackScreen, EntryWindow, ButtonChoiceWindow, \ ButtonBar, Listbox, GridFormHelp HAVE_NEWT = True except ImportError: HAVE_NEWT = False # will be reported in run() __license__ = 'MIT' __copyright__ = \ 'Copyright (c) 2011-2019, Ulf Magnusson \n' \ 'Copyright (c) Siemens AG, 2021-2023' SOURCE_DIR_HOST_ENV_KEY = '_KAS_REPO_DIR_HOST' class VariableTypeError(KasUserError): pass class KConfigLoadError(KasUserError): """ The KConfig file could not be found or is invalid """ pass def check_sym_is_string(sym): if sym.type != STRING: raise VariableTypeError(f'Variable {sym.name} must be of string type') def str_representer(dumper, data): style = '|' if len(data.splitlines()) > 1 else None return dumper.represent_scalar('tag:yaml.org,2002:str', data, style=style) class Args: pass class Menu: """ This class implements the menu plugin for kas. """ name = 'menu' helpmsg = ( 'Provides a configuration menu and triggers the build of the choices.' ) @classmethod def setup_parser(cls, parser): parser.add_argument('kconfig', help='Kconfig file', nargs='?', default='Kconfig') def load_config(self, filename): try: config = ConfigFile.load(filename) self.orig_config = config.config except FileNotFoundError: self.orig_config = {} return menu_configuration = self.orig_config.get('menu_configuration', {}) for symname in menu_configuration: sym = self.kconf.syms.get(symname) if not sym: logging.warning( 'Ignoring unknown configuration variable %s in %s', symname, filename) continue symvalue = menu_configuration[symname] if sym.type == BOOL: sym.set_value('y' if symvalue else 'n') elif sym.type == INT: sym.set_value(str(symvalue)) elif sym.type == HEX: sym.set_value(str(hex(symvalue))) else: # string sym.set_value(symvalue) def save_config(self, filename, top_repo_dir): kas_includes = [] kas_targets = [] kas_build_system = None kas_vars = {} menu_configuration = {} for symname in self.kconf.syms: if symname == 'MODULES': continue sym = self.kconf.syms[symname] symvalue = sym.str_value if expr_value(sym.direct_dep) < 2: continue if sym.visibility == 2: if sym.type == BOOL: menu_configuration[symname] = symvalue == 'y' elif sym.type == STRING: menu_configuration[symname] = symvalue elif sym.type == INT: menu_configuration[symname] = int(symvalue) elif sym.type == HEX: menu_configuration[symname] = int(symvalue, 16) else: raise VariableTypeError( 'Configuration variable {symname} uses unsupported ' 'type') if symname.startswith('KAS_INCLUDE_'): check_sym_is_string(sym) if symvalue != '': kas_includes.append(symvalue) elif symname.startswith('KAS_TARGET_'): check_sym_is_string(sym) if symvalue != '': kas_targets.append(symvalue) elif symname == 'KAS_BUILD_SYSTEM': check_sym_is_string(sym) if symvalue != '': kas_build_system = symvalue elif sym.type in (STRING, INT, HEX): kas_vars[symname] = symvalue config = { 'header': { 'version': __file_version__, 'includes': kas_includes }, 'menu_configuration': menu_configuration, SOURCE_DIR_OVERRIDE_KEY: top_repo_dir } if SOURCE_DIR_HOST_ENV_KEY in os.environ: config[SOURCE_DIR_HOST_OVERRIDE_KEY] = \ os.environ[SOURCE_DIR_HOST_ENV_KEY] if kas_build_system: config['build_system'] = kas_build_system if len(kas_targets) > 0: config['target'] = kas_targets if len(kas_vars) > 0: config['local_conf_header'] = { '__menu_config_vars': '\n'.join([ f'{key} = "{value}"' for key, value in kas_vars.items() ]) } logging.debug('Menu configuration:\n%s', pprint.pformat(config)) if config != self.orig_config: logging.info('Saving configuration as %s', filename) # format multi-line strings more nicely yaml.add_representer(str, str_representer) try: os.rename(filename, filename + '.old') except FileNotFoundError: pass with open(filename, 'w') as config_file: config_file.write( '#\n' f'# Automatically generated by kas {__version__}\n' '#\n') yaml.dump(config, config_file) def dump_kconf_warnings(self): if len(self.kconf.warnings) > 0: logging.warning("\n".join(self.kconf.warnings)) self.kconf.warnings = [] def run(self, args): if not HAVE_KCONFIGLIB: raise MissingModuleError('python3-kconfiglib', 'Menu plugin') if not HAVE_NEWT: raise MissingModuleError('python3-newt', 'Menu plugin') ctx = create_global_context(args) kconfig_file = os.path.abspath(args.kconfig) try: self.kconf = Kconfig(kconfig_file, warn_to_stderr=False) except (KconfigError, FileNotFoundError) as err: raise KConfigLoadError(str(err)) top_repo_path = Repo.get_root_path(os.path.dirname(kconfig_file)) config_filename = os.path.join(ctx.kas_work_dir, CONFIG_YAML_FILE) self.load_config(config_filename) self.dump_kconf_warnings() menu = Menuconfig(self.kconf) action = menu.show() if action == 'exit': return self.save_config(config_filename, top_repo_path) self.dump_kconf_warnings() if action == 'build': logging.debug('Starting build') build_args = Args() build_args.config = None build_args.target = None build_args.task = None build_args.extra_bitbake_args = [] build_args.skip = None build_args.provenance = False Build().run(build_args) class Menuconfig(): def __init__(self, kconf): self.kconf = kconf self.screen = None @staticmethod def value_str(sym): """ Returns the value part ("[*]", "(foo)" etc.) of a menu entry. """ if sym.type in (STRING, INT, HEX): return f"({sym.str_value})" # BOOL (TRISTATE not supported) # The choice mode is an upper bound on the visibility of choice # symbols, so we can check the choice symbols' own visibility to see # if the choice is in y mode if sym.choice and sym.visibility == 2: return "(*)" if sym.choice.selection is sym else "( )" tri_val_str = (" ", None, "*")[sym.tri_value] if len(sym.assignable) == 1: # Pinned to a single value return f"-{tri_val_str}-" if sym.type == BOOL: return f"[{tri_val_str}]" raise RuntimeError() @staticmethod def node_str(node, indent): """ Returns the complete menu entry text for a menu node, or "" for invisible menu nodes. Invisible menu nodes are those that lack a prompt or that do not have a satisfied prompt condition. Example return value: "[*] Bool symbol (BOOL)" The symbol name is printed in parentheses to the right of the prompt. This is so that symbols can easily be referred to in the configuration interface. """ if not node.prompt: return "" # Even for menu nodes for symbols and choices, it's wrong to check # Symbol.visibility / Choice.visibility here. The reason is that a # symbol (and a choice, in theory) can be defined in multiple # locations, giving it multiple menu nodes, which do not necessarily # all have the same prompt visibility. Symbol.visibility / # Choice.visibility is calculated as the OR of the visibility of all # the prompts. prompt, prompt_cond = node.prompt if not expr_value(prompt_cond): return "" if node.item == MENU: return f" {indent * ' '}{prompt} --->" if type(node.item) is Choice: return f" {indent * ' '}{prompt}" if node.item == COMMENT: return f" {indent * ' '}*** {prompt} ***" # Symbol sym = node.item if sym.type == UNKNOWN: return "" # {:3} sets the field width to three. Gives nice alignment for empty # string values. res = f"{Menuconfig.value_str(sym):3} {indent * ' '}{prompt}" # Append a sub-menu arrow if menuconfig and enabled if node.is_menuconfig: res += f" ---{'>' if sym.tri_value > 0 else '-'}" return res @staticmethod def menu_node_strings(node, indent): items = [] while node: string = Menuconfig.node_str(node, indent) if string: items.append((string, node)) if (node.list and node.item != MENU and (type(node.item) is Choice or not node.is_menuconfig)): items.extend(Menuconfig.menu_node_strings(node.list, indent + 2)) node = node.next return items def show_menu(self, title, top_node, is_submenu=False): selection = 0 while True: items = Menuconfig.menu_node_strings(top_node, 0) height = len(items) scroll = 0 if height > self.screen.height - 13: height = self.screen.height - 13 scroll = 1 buttons = [ ('Build', 'build', 'B'), ('Save & Exit', 'save', 'S'), (' Exit ', 'exit', 'E'), (' Help ', 'help', 'h') ] if is_submenu: buttons.insert(0, (' Return ', 'return', 'ESC')) buttonbar = ButtonBar(self.screen, buttons) if not is_submenu: buttonbar.hotkeys['ESC'] = 'exit' listbox = Listbox(height, scroll=scroll, returnExit=1) count = 0 for string, _ in items: listbox.append(string, count) if (selection == count): listbox.setCurrent(count) count = count + 1 grid = GridFormHelp(self.screen, title, None, 1, 2) grid.add(listbox, 0, 0, padding=(0, 0, 0, 1)) grid.add(buttonbar, 0, 1, growx=1) grid.addHotKey(' ') rc = grid.runOnce() action = buttonbar.buttonPressed(rc) if action and action != 'help': return action if count == 0: continue selection = listbox.current() _, selected_node = items[selection] sym = selected_node.item if action == 'help': prompt, _ = selected_node.prompt if hasattr(selected_node, 'help') and selected_node.help: help = selected_node.help else: help = 'No help available.' ButtonChoiceWindow( screen=self.screen, title=f"Help on '{prompt}'", text=help, width=60, buttons=[' Ok ']) continue show_submenu = False if type(sym) is Symbol: if rc == ' ': if sym.type == BOOL: sym.set_value('n' if sym.tri_value > 0 else 'y') else: if selected_node.is_menuconfig: show_submenu = True elif sym.type in (STRING, INT, HEX): action, values = EntryWindow( screen=self.screen, title=sym.name, text=f'Enter a {TYPE_TO_STR[sym.type]} value:', prompts=[('', sym.str_value)], buttons=[(' Ok ', 'Ok'), ('Cancel', '', 'ESC')]) if action == 'Ok': self.kconf.warnings = [] val = values[0] if sym.type == HEX and not val.startswith('0x'): val = '0x' + val sym.set_value(val) # only fetching triggers range check - how ugly... sym.str_value if len(self.kconf.warnings) > 0: ButtonChoiceWindow( screen=self.screen, title="Invalid entry", text="\n".join(self.kconf.warnings), width=60, buttons=[' Ok ']) self.kconf.warnings = [] elif selected_node.is_menuconfig and type(sym) is not Choice: show_submenu = True if show_submenu: submenu_title, _ = selected_node.prompt action = self.show_menu(submenu_title, selected_node.list, is_submenu=True) if action != 'return': return action def show(self): self.screen = SnackScreen() action = self.show_menu(self.kconf.mainmenu_text, self.kconf.top_node.list) self.screen.finish() return action __KAS_PLUGINS__ = [Menu] siemens-kas-41ad961/kas/plugins/shell.py000066400000000000000000000107241520561422700202140ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This plugin implements the ``kas shell`` command. When this command is executed, kas will checkout repositories, setup the build environment and then start a shell in the build environment. This can be used to manually run ``bitbake`` with custom command line options or to execute other commands such as ``runqemu``. The ``SHELL`` environment variable is inherited from the calling environment and controls which shell is started. For example, to start a shell in the build environment for the file ``kas-project.yml`` you could run:: kas shell kas-project.yml Or to invoke qemu to test an image which has been built:: kas shell kas-project.yml -c 'runqemu' """ import logging import os import subprocess from kas.context import create_global_context from kas.config import Config from kas.libcmds import Macro, Command, SetupHome, MakeInteractive from kas.libkas import setup_parser_common_args, setup_parser_config_arg from kas.libkas import setup_parser_keep_config_unchanged_arg from kas.libkas import setup_parser_preserve_env_arg from kas.libkas import run_handle_preserve_env_arg from kas.kasusererror import CommandExecError __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' SHELL_HISTORY_FILE = '.kas_shell_history' class Shell: """ Implements a kas plugin that opens a shell within the kas environment. """ name = 'shell' helpmsg = 'Run a shell in the build environment.' @classmethod def setup_parser(cls, parser): """ Setup the argument parser for the shell plugin """ setup_parser_common_args(parser) setup_parser_config_arg(parser) setup_parser_preserve_env_arg(parser) setup_parser_keep_config_unchanged_arg(parser) parser.add_argument('-c', '--command', help='Run command', default='') def run(self, args): """ Runs this kas plugin """ ctx = create_global_context(args) ctx.config = Config(ctx, args.config) run_handle_preserve_env_arg(ctx, os, args, SetupHome) macro = Macro() macro.add(MakeInteractive()) macro.add(ShellCommand(args.command)) macro.run(ctx, args.skip) @classmethod def get_managed_paths(cls): return [SHELL_HISTORY_FILE] class ShellCommand(Command): """ This class implements the command that starts a shell. """ def __init__(self, cmd): super().__init__() self.cmd = [] if cmd: self.cmd = cmd def __str__(self): return 'shell' def execute(self, ctx): logging.info("To start the default build, run: bitbake -c %s %s", ctx.config.get_bitbake_task(), ' '.join(ctx.config.get_bitbake_targets())) cmd = [ctx.environ.get('SHELL', '/bin/sh')] if self.cmd: cmd.append('-c') cmd.append(self.cmd) ctx.environ['HISTFILE'] = os.path.join(ctx.kas_work_dir, SHELL_HISTORY_FILE) ret = subprocess.call(cmd, env=ctx.environ, cwd=ctx.build_dir) if ret != 0: logging.error('Shell returned non-zero exit status') raise CommandExecError(cmd, ret, True) __KAS_PLUGINS__ = [Shell] siemens-kas-41ad961/kas/repos.py000066400000000000000000001123621520561422700165550ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2019 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ This module contains the Repo class. """ import re import os import linecache import logging import shutil from pathlib import Path from dataclasses import dataclass from datetime import datetime from urllib.parse import urlparse from tempfile import TemporaryDirectory from kas.configschema import CONFIGSCHEMA from kas.keyhandler import GPGKeyHandler, SSHKeyHandler from .context import get_context from .libkas import run_cmd_async, run_cmd from .kasusererror import KasUserError from functools import cached_property from git import GitCommandError, Repo as GitPythonRepo __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' class UnsupportedRepoTypeError(KasUserError, NotImplementedError): """ The requested repo type is unsupported / not implemented """ pass class RepoRefError(KasUserError): """ The requested repo reference is invalid, missing or could not be found """ pass class PatchFileNotFound(KasUserError, FileNotFoundError): """ The requested patch file was not found """ pass class PatchMappingError(KasUserError): """ The requested patch can not be related to a repo """ pass class PatchApplyError(KasUserError): """ The provided patch file could not be applied """ def __init__(self, msg, cmd=None, out=None, err=None): if cmd: msg += '\nvcs command: ' + ' '.join(cmd) if out and out.strip(): msg += f'\nvcs output:\n{out.strip()}' if err and err.strip(): msg += f'\nvcs error:\n{err.strip()}' super().__init__(msg) class RepoFetchError(KasUserError): """ An error occurred during repo fetching """ def __init__(self, repo, output): super().__init__(f'Fetching of repo: "{repo.name}" failed: {output}') class Repo: """ Represents a repository in the kas configuration. """ def __init__(self, name, url, path, commit, tag, branch, refspec, layers, patches, signers, disable_operations): self.name = name self.url = url self.path = path self.commit = commit self.tag = tag self.branch = branch self.refspec = refspec self.layers = layers self._patches = patches self.allowed_signers = signers self.operations_disabled = disable_operations if not self.url: self.resolve_local() @property def qualified_name(self): url = urlparse(self.url) return (f'{url.netloc}{url.path}' .replace('@', '.') .replace(':', '.') .replace('/', '.') .replace('*', '.')) @property def effective_url(self): mirrors = os.environ.get('KAS_PREMIRRORS', '') for mirror in mirrors.split('\n'): try: expr, subst = mirror.split() if re.match(expr, self.url): return re.sub(expr, subst, self.url) except ValueError: continue return self.url @cached_property def revision(self): if self.commit: (_, output) = run_cmd(self.get_commit_cmd(), cwd=self.path, fail=False) if output: return output.strip() return self.commit if self.tag: (_, output) = run_cmd(self.resolve_tag_cmd(), cwd=self.path, fail=False) if output: return output.strip() return self.tag branch = self.branch or self.refspec if branch: (_, output) = run_cmd(self.resolve_branch_cmd(), cwd=self.path, fail=False) if output: return output.strip() return branch return None @cached_property def revision_annotated_tag(self): if not self.tag: return None try: cmd = self.resolve_annotated_tag_cmd() except NotImplementedError: return None (retc, output) = run_cmd(cmd, cwd=self.path, fail=False) if retc or not output: return None return output.strip() @cached_property def dirty(self): if not self.url: return True (_, output) = run_cmd(self.is_dirty_cmd(), cwd=self.path, fail=False) return bool(output) @cached_property def signers_type(self): if self.allowed_signers is None: return 'gpg' signers = get_context().config.get_signers_config() try: ktypes = [signers[k].get('type', 'gpg') for k in self.allowed_signers] except KeyError as e: raise KasUserError(f'Repository {self.name}: ' f'Allowed signer "{e}" not found in config') if len(set(ktypes)) > 1: raise KasUserError(f'Repository {self.name}: ' 'Mixed signer types are not supported') return ktypes[0] @property def keyhandler(self): if not self.signed: return None return get_context().keyhandler[self.signers_type] def check_signature(self): self.keyhandler.prepare_validation(self) (ret, _, err) = run_cmd(self.is_signed_cmd(), cwd=self.path, fail=False, capture_stderr=True) logging.debug('Signature verification output (%d):\n%s', ret, err) if ret != 0: return (False, None) return self.keyhandler.validate_allowed_signer(self, err) @cached_property def signed(self): return self.allowed_signers is not None @staticmethod def get_type(): """ Repo type as defined in spdx-spec/v2.3/package-information """ raise NotImplementedError("Repo type not implemented") def contains_path(self, path): (ret, _) = run_cmd(self.contains_path_cmd(str(path)), cwd=self.path, fail=False) return ret == 0 def __str__(self): if self.commit and (self.tag or self.branch): refspec = f'{self.commit}({self.tag or self.branch})' else: refspec = self.commit or self.tag or self.branch or self.refspec return f'{self.url}:{refspec} ' \ f'{self.path} {self.layers}' __legacy_refspec_warned__ = [] __no_commit_warned__ = [] @staticmethod def factory(name, repo_config, repo_defaults, repo_fallback_path, repo_overrides={}): """ Returns a Repo instance depending on parameters. This factory function is referential transparent. """ default_patch_repo = repo_defaults.get('patches', {}).get('repo', None) patches_dict = repo_config.get('patches', {}) patches = [] for p in sorted(patches_dict): if not patches_dict[p]: continue this_patch = { 'id': p, 'repo': patches_dict[p].get('repo', default_patch_repo), 'path': patches_dict[p]['path'], } if this_patch['repo'] is None: raise PatchMappingError( f'No repo specified for patch entry "{p}" and no ' 'default repo specified.') patches.append(this_patch) url = repo_config.get('url', None) name = repo_config.get('name', name) repo_type = repo_config.get('type', 'git') commit = repo_config.get('commit', None) tag = repo_config.get('tag', repo_defaults.get('tag', None)) branch = repo_config.get('branch', repo_defaults.get('branch', None)) refspec = repo_config.get('refspec', repo_defaults.get('refspec', None)) if commit is None and tag is None and branch is None \ and refspec is None and url is not None: raise RepoRefError('No commit, tag or branch specified for ' f'repository "{name}". This is only allowed ' 'for local repositories.') if refspec is None: commit = repo_overrides.get('commit', commit) branch = repo_overrides.get('branch', branch) if commit and get_context().update: logging.warning(f'Update of "{name}" requested, but repo is ' 'pinned to a fixed commit. Not updating.') else: if name not in Repo.__legacy_refspec_warned__: logging.warning('Using deprecated refspec for repository "%s".' ' You should migrate to commit/tag/branch.', name) Repo.__legacy_refspec_warned__.append(name) if commit is not None or tag is not None or branch is not None: raise RepoRefError( 'Unsupported mixture of legacy refspec and ' f'commit/tag/branch for repository "{name}"') refspec = repo_overrides.get('commit', refspec) if not commit and name not in Repo.__no_commit_warned__: if tag: logging.warning('Using tag without commit for repository ' '"%s" is unsafe as tags are mutable.', name) Repo.__no_commit_warned__.append(name) elif branch: logging.warning('Using branch without commit for repository ' '"%s" is unsafe. Either add a commit or use ' 'a lock file.', name) Repo.__no_commit_warned__.append(name) path = repo_config.get('path', None) signed = repo_config.get('signed', False) signers = repo_config.get('allowed_signers', None) if signed else None if signed and not signers: raise KasUserError(f'Repository "{name}" is signed but no allowed ' 'signers specified.') disable_operations = False if path is None: if url is None: path = Repo.get_root_path(repo_fallback_path) logging.info('Using %s as root for repository %s', path, name) else: path = os.path.join(get_context().kas_work_dir, name) elif not os.path.isabs(path): # Relative pathes are assumed to start from work_dir path = os.path.join(get_context().kas_work_dir, path) layers_dict = repo_config.get('layers', {'': None}) layers = Repo._create_layers_from_dict(name, path, layers_dict) if url is None: # No version control operation on repository disable_operations = True if repo_type == 'git': if commit and not re.match(r'^[0-9a-f]{40}|[0-9a-f]{64}$', commit): logging.warning( f'{commit} is not a full-length hash for repo ' f'"{name}". This will be an error in future versions.') return GitRepo(name, url, path, commit, tag, branch, refspec, layers, patches, signers, disable_operations) if repo_type == 'hg': if not shutil.which('hg'): raise UnsupportedRepoTypeError( 'hg is required for Mercurial repositories') return MercurialRepo(name, url, path, commit, tag, branch, refspec, layers, patches, signers, disable_operations) raise UnsupportedRepoTypeError(f'Repo type "{repo_type}" ' 'not supported.') @staticmethod def get_root_path(path, fallback=True): """ Checks if path is under version control and returns its root path. If the repo is a submodule, the root path of the super-repository is returned. """ git_cmd = ['git', 'rev-parse', '--show-toplevel', '--show-superproject-working-tree'] (ret, output) = run_cmd(git_cmd, cwd=path, fail=False) if ret == 0: return sorted(output.strip().split('\n'))[0] (ret, output) = run_cmd(['hg', 'root'], cwd=path, fail=False) if ret == 0: return output.strip() return path if fallback else None @staticmethod def _create_layers_from_dict(repo_name, repo_path, layers_dict): layers = [] disabled_token = "disabled" legacy_disabled_tokens = ['excluded', 'n', 'no', '0', 'false', 0] default_prio = \ CONFIGSCHEMA['$defs']['layerPrio']['properties']['prio']['default'] for lname, prop in layers_dict.items(): if prop is None: layers.append(RepoLayer(name=lname, repo_path=Path(repo_path), repo_name=repo_name, priority=default_prio)) elif isinstance(prop, str) and prop == disabled_token: continue elif isinstance(prop, (str, int)) and \ prop in legacy_disabled_tokens: logging.warning('Use of deprecated value "%s" for repo ' '"%s", layer "%s". Replace with "disabled".', prop, repo_name, lname) continue elif isinstance(prop, dict): layers.append(RepoLayer(name=lname, repo_path=Path(repo_path), repo_name=repo_name, priority=prop.get('prio', default_prio))) else: raise NotImplementedError() return layers class RepoImpl(Repo): """ Provides a generic implementation for a Repo. """ async def fetch_async(self): """ Starts asynchronous repository fetch. """ if self.operations_disabled: return refdir = get_context().kas_repo_ref_dir sdir = os.path.join(refdir, self.qualified_name) if refdir else None # fetch to refdir if refdir and not os.path.exists(sdir): os.makedirs(refdir, exist_ok=True) with TemporaryDirectory(prefix=self.qualified_name + '.', dir=refdir) as tmpdir: (retc, _) = await run_cmd_async( self.clone_cmd(tmpdir, createref=True), cwd=get_context().kas_work_dir) logging.debug('Created repo ref for %s', self.qualified_name) try: os.rename(tmpdir, sdir) except OSError: logging.debug('repo %s already cloned by other instance', self.qualified_name) if not os.path.exists(self.path): logging.info('Cloning repository %s', self.name) os.makedirs(os.path.dirname(self.path), exist_ok=True) (retc, _) = await run_cmd_async( self.clone_cmd(sdir, createref=False), cwd=get_context().kas_work_dir) # Make sure the remote origin is set to the value # in the kas file to avoid surprises try: (retc, output) = await run_cmd_async( self.set_remote_url_cmd(), cwd=self.path) except NotImplementedError: logging.warning('Repo implementation does not support changing ' 'the remote url.') # take what came out of clone and stick to that forever if self.commit is None and self.tag is None and self.branch is None \ and self.refspec is None: return # check if we already have the commit. On update, check as well in case # the commit is fixed, hence the repo must not be updated anyways force_update = get_context().update if not force_update or (force_update and self.commit): # Do commit/tag/branch/refspec exist in the current repository? (retc, output) = await run_cmd_async(self.contains_refspec_cmd(), cwd=self.path, fail=False) if retc == 0: logging.info('Repository %s already contains %s as %s', self.name, self.commit or self.tag or self.branch or self.refspec, output.strip()) # if branch is specified, check if it contains the commit # also in our local clone depth = get_context().repo_clone_depth if self.branch and self.commit and not depth: (_, output) = await run_cmd_async( self.branch_contains_ref(), cwd=self.path, fail=False) if output.strip(): return else: return # Try to fetch if commit/tag/branch/refspec is missing or if --update # argument was passed (retc, _, err) = await run_cmd_async(self.fetch_cmd(), cwd=self.path, fail=False, capture_stderr=True) if retc: raise RepoFetchError(self, err) else: logging.info('Repository %s updated', self.name) def checkout(self): """ Checks out the correct revision of the repo. """ if self.operations_disabled \ or (self.commit is None and self.tag is None and self.branch is None and self.refspec is None): return if not get_context().force_checkout: # Check if repos is dirty if self.dirty: logging.warning('Repo %s is dirty - no checkout', self.name) return if self.tag and self.branch: raise RepoRefError( f'Both tag "{self.tag}" and branch "{self.branch}" ' f'cannot be specified for repository "{self.name}"') if self.tag: (retc, output) = run_cmd(self.resolve_tag_cmd(), cwd=self.path, fail=False) if retc: raise RepoRefError(f'Tag "{self.tag}" cannot be found ' f'in repository "{self.name}"') desired_ref = output.strip() if self.commit and desired_ref != self.commit: if self.revision_annotated_tag != self.commit: # Ensure provided commit and tag match raise RepoRefError(f'Provided tag "{self.tag}" ' f'("{desired_ref}") does not match ' f'provided commit "{self.commit}" in ' f'repository "{self.name}", aborting!') is_branch = False elif self.branch: (retc, output) = run_cmd(self.resolve_branch_cmd(), cwd=self.path, fail=False) if retc: raise RepoRefError( f'Branch "{self.branch}" cannot be found ' f'in repository "{self.name}"') # check if branch contains the requested commit. # skip check on shallow clones, as branch information is missing if self.commit and not get_context().repo_clone_depth: (_, output) = run_cmd(self.branch_contains_ref(), cwd=self.path, fail=False) if not output.strip(): raise RepoRefError( f'Branch "{self.branch}" in ' f'repository "{self.name}" does not contain ' f'commit "{self.commit}"') desired_ref = self.commit or output.strip() is_branch = True elif self.commit: desired_ref = self.commit is_branch = False else: desired_ref = self.refspec is_branch = False run_cmd(self.checkout_cmd(desired_ref, is_branch), cwd=self.path) logging.info(f'Repository {self.name} checked out to {desired_ref}') async def apply_patches_async(self): """ Applies patches to a repository asynchronously. """ if self.operations_disabled or not self._patches: return 0 if self.dirty: logging.warning(f'Repo {self.name} is dirty - no patching') return 0 (retc, _) = await run_cmd_async(self.prepare_patches_cmd(), cwd=self.path) my_patches = [] for patch in self._patches: other_repo = get_context().config.repo_dict.get(patch['repo'], None) if not other_repo: raise PatchMappingError('Could not find referenced repo. ' f'(missing repo: {patch["repo"]}, ' f'repo: {self.name}, ' f'patch entry: {patch["id"]})') path = os.path.join(other_repo.path, patch['path']) cmd = [] if os.path.isfile(path): my_patches.append((path, patch['id'])) elif os.path.isdir(path) \ and os.path.isfile(os.path.join(path, 'series')): with open(os.path.join(path, 'series')) as f: for line in f: if line.startswith('#'): continue p = os.path.join(path, line.split(' #')[0].rstrip()) if os.path.isfile(p): my_patches.append((p, patch['id'])) else: raise PatchFileNotFound(p) else: raise PatchFileNotFound( 'Could not find patch. ' f'(patch path: {path}, repo: {self.name}, patch ' f'entry: {patch["id"]})') for (path, patch_id) in my_patches: cmd = self.apply_patches_file_cmd(path) (retc, out, err) = await run_cmd_async( cmd, cwd=self.path, fail=False, capture_stderr=True) if retc: raise PatchApplyError( 'Could not apply patch. Please fix repos and patches:\n' f'patch path: {path}, repo: {self.name}, patch ' f'entry: {patch_id}', cmd, out, err) logging.info('Patch applied. ' '(patch path: %s, repo: %s, patch entry: %s)', path, self.name, patch_id) cmd = self.add_cmd() (retc, out, err) = await run_cmd_async( cmd, cwd=self.path, fail=False, capture_stderr=True) if retc: raise PatchApplyError('Could not add patched files: repo: ' f'{self.name}', cmd, out, err) timestamp = self.get_patch_timestamp(path) if not timestamp: dt = datetime.fromtimestamp(os.path.getmtime(path)) timestamp = dt.astimezone().strftime( "%a, %d %b %Y %H:%M:%S %z") env = get_context().environ.copy() msg = f'kas: {patch_id}\n\npatch {path} applied by kas' cmd = self.commit_cmd(env, 'kas ', msg, timestamp) (retc, out, err) = await run_cmd_async( cmd, cwd=self.path, env=env, fail=False, capture_stderr=True) if retc: raise PatchApplyError('Could not commit patch changes. repo: ' f'{self.name}', cmd, out, err) return 0 def resolve_local(self): (retc, output) = run_cmd(self.get_remote_url_cmd(), cwd=self.path, fail=False) if retc == 0: self.url = output.strip() (retc, output) = run_cmd(self.get_commit_cmd(), cwd=self.path, fail=False) if retc == 0: self.commit = output.strip() if self.url and self.commit: logging.debug('Repository %s resolved to %s @ %s', self.name, self.url, self.commit) class GitRepo(RepoImpl): """ Provides the git functionality for a Repo. """ @staticmethod def get_type(): return 'git' def remove_ref_prefix(self, ref): ref_prefix = 'refs/' return ref[ref.startswith(ref_prefix) and len(ref_prefix):] def add_cmd(self): return ['git', 'add', '-A'] def clone_cmd(self, srcdir, createref): cmd = ['git', 'clone', '-q'] depth = get_context().repo_clone_depth if depth: if self.refspec: logging.warning('Shallow cloning is not supported for legacy ' f'refspec on repository "{self.name}". ' 'Performing full clone.') else: if createref: # this is not a user-error, as the clone of the work repo # can still be shallow. logging.debug('Shallow cloning is not supported for ' f'reference repository of "{self.name}". ' 'Performing full clone.') else: cmd.extend(['--depth', str(depth)]) if self.branch: cmd.extend(['--branch', self.remove_ref_prefix(self.branch)]) if createref: cmd.extend(['--bare', '--', self.effective_url, srcdir]) elif srcdir: cmd.extend(['--reference', srcdir, '--', srcdir, self.path]) else: cmd.extend(['--', self.effective_url, self.path]) return cmd def commit_cmd(self, env, author, msg, date): env["GIT_COMMITTER_DATE"] = date return ['git', 'commit', '-a', '--author', author, '-m', msg, '--date', date] def contains_refspec_cmd(self): branch = self.branch or self.refspec if branch and branch.startswith('refs/'): branch = 'remotes/origin/' + self.remove_ref_prefix(branch) return ['git', 'cat-file', '-t', self.commit or self.tag or branch] def fetch_cmd(self): cmd = ['git', 'fetch', '-q'] depth = 0 if self.refspec else get_context().repo_clone_depth if depth: cmd.extend(['--depth', str(depth)]) if self.tag: cmd.extend(['origin', f'+{self.tag}:refs/tags/{self.tag}']) return cmd # only fetch this commit (branch information is lost) if depth and self.commit: cmd.extend(['origin', self.commit]) return cmd branch = self.branch or self.refspec if branch and (branch.startswith('refs/') or depth): branch = self.remove_ref_prefix(branch) cmd.extend(['origin', f'+{branch}:refs/remotes/origin/{branch}']) return cmd def is_dirty_cmd(self): return ['git', 'diff', '--stat'] def is_signed_cmd(self): if self.tag: return ['git', 'verify-tag', '--raw', self.tag] else: refspec = self.commit or self.revision return ['git', 'verify-commit', '--raw', refspec] def resolve_branch_cmd(self): refspec = self.remove_ref_prefix(self.branch or self.refspec) return ['git', 'rev-parse', '--verify', '-q', f'origin/{refspec}'] def resolve_tag_cmd(self): return ['git', 'rev-list', '-n', '1', self.remove_ref_prefix(self.tag)] def resolve_annotated_tag_cmd(self): return ['git', 'show-ref', '--tags', '--hash', self.tag] def branch_contains_ref(self): return ['git', 'branch', f'origin/{self.branch}', '-r', '--contains', self.commit] def checkout_cmd(self, desired_ref, is_branch): desired_ref = self.remove_ref_prefix(desired_ref) if re.match(r'^[0-9a-f]{40}|[0-9a-f]{64}$', desired_ref): desired_ref += '^{commit}' cmd = ['git', 'checkout', '-q', desired_ref] if is_branch: branch = self.remove_ref_prefix(self.branch or self.refspec) branch = branch[branch.startswith('heads/') and len('heads/'):] cmd.extend(['-B', branch]) if get_context().force_checkout: cmd.append('--force') return cmd def prepare_patches_cmd(self): refspec = self.commit \ or self.remove_ref_prefix(self.tag or self.branch or self.refspec) return ['git', 'checkout', '-q', '-B', f'patched-{refspec}'] def apply_patches_file_cmd(self, path): return ['git', 'apply', '--whitespace=nowarn', path] def set_remote_url_cmd(self): return ['git', 'remote', 'set-url', 'origin', self.effective_url] def get_remote_url_cmd(self): return ['git', 'remote', 'get-url', 'origin'] def get_commit_cmd(self): rev = self.commit or 'HEAD' return ['git', 'rev-parse', '--verify', rev] def get_patch_timestamp(self, path): date = linecache.getline(path, 3) linecache.clearcache() if date and date.startswith("Date: "): return date.replace("Date: ", "").strip() def contains_path_cmd(self, path): return ['git', 'ls-files', '--error-unmatch', path] def diff(self, commit1, commit2): if commit1 is None: commit1 = 'HEAD' if commit2 is None: commit2 = 'HEAD' git_repo = GitPythonRepo(self.path) shallow_file = os.path.join(git_repo.git_dir, 'shallow') if os.path.isfile(shallow_file): git_repo.git.fetch(unshallow=True) try: commits = list(git_repo.iter_commits( f'{commit1}..{commit2}')) except GitCommandError as e: valid1 = git_repo.is_valid_object(commit1) valid2 = git_repo.is_valid_object(commit2) if not valid1 and not valid2: error_msg = f'Commits ({commit1} and {commit2}) are not valid' elif not valid1: error_msg = f'First commit ({commit1}) is not valid' elif not valid2: error_msg = f'Second commit ({commit2}) is not valid' else: error_msg = str(e) raise RepoRefError( 'Could not compute diff for repository ' f'"{self.name}": {error_msg}') diff_json = {self.name: []} for commit in commits: diff_json[self.name].append({ 'commit': commit.hexsha, 'author': commit.author.name, 'email': commit.author.email, 'commit_date': commit.committed_datetime. strftime("%Y-%m-%d %H:%M:%S"), 'message': commit.message }) return diff_json class MercurialRepo(RepoImpl): """ Provides the hg functionality for a Repo. """ @staticmethod def get_type(): return 'hg' def add_cmd(self): return ['hg', 'add'] def clone_cmd(self, srcdir, createref): # Mercurial does not support repo references (object caches) if createref: return ['true'] return ['hg', 'clone', self.effective_url, self.path] def commit_cmd(self, env, author, msg, date): return ['hg', 'commit', '--user', author, '-m', msg, '--date', date] def contains_refspec_cmd(self): return ['hg', 'log', '-r', self.commit or self.tag or self.branch or self.refspec] def fetch_cmd(self): return ['hg', 'pull'] def is_dirty_cmd(self): return ['hg', 'status', '--modified', '--added', '--removed', '--deleted'] def is_signed_cmd(self): raise NotImplementedError() def resolve_branch_cmd(self): if self.branch: return ['hg', 'identify', '--id', '-r', f'limit(heads(branch({self.branch})))'] else: return ['hg', 'identify', '--id', '-r', self.refspec] def resolve_tag_cmd(self): refspec = self.tag or self.refspec return ['hg', 'identify', '--id', '-r', f'tag({refspec})'] def resolve_annotated_tag_cmd(self): raise NotImplementedError("Mercurial does not support annotated tags") def branch_contains_ref(self): return ['hg', 'log', '-r', self.commit, '-b', self.branch] def checkout_cmd(self, desired_ref, is_branch): cmd = ['hg', 'checkout', desired_ref] if get_context().force_checkout: cmd.append('--clean') return cmd def prepare_patches_cmd(self): refspec = (self.commit or self.tag or self.branch or self.refspec) # strip revision part from refspec as not allowed in branch names refspec = refspec.split(':')[-1] return ['hg', 'branch', '-f', f'patched-{refspec}'] def apply_patches_file_cmd(self, path): return ['hg', 'import', '--no-commit', path] def set_remote_url_cmd(self): raise NotImplementedError() def get_remote_url_cmd(self): return ['hg', 'paths', 'default'] def get_commit_cmd(self): rev = self.commit or '.' return ['hg', 'log', '-r', rev, '--template', '{node}\n'] def get_patch_timestamp(self, path): date = None if linecache.getline(path, 3).startswith("# Date "): date = linecache.getline(path, 4) linecache.clearcache() if date and date.startswith("# "): return date.replace("# ", "").strip() def contains_path_cmd(self, path): return ['hg', 'files', path] def diff(self, commit1, commit2): raise NotImplementedError("Unsupported diff for MercurialRepo") @dataclass class RepoLayer: """ Standalone definition of a single bitbake layer. """ name: str priority: int repo_name: str repo_path: Path @property def path(self): return self.repo_path / self.name def __lt__(self, other): if isinstance(other, RepoLayer): # The priority is negated because a higher priority means the # element appears earlier in the list. return (-self.priority, self.repo_name, self.name) < \ (-other.priority, other.repo_name, other.name) return NotImplemented class SignatureValidator: """ Handles key loading and signature validation of repos based on the configuration. """ @staticmethod def import_keys(ctx): handler_cfg = { 'gpg': (GPGKeyHandler, Path(ctx.kas_work_dir) / '.kas_gnupg'), 'ssh': (SSHKeyHandler, Path(ctx.kas_work_dir) / '.kas_ssh-handler'), } for name, (handler_cls, dir) in handler_cfg.items(): signers_cfg = ctx.config.get_signers_config(name) if not signers_cfg: continue dir.mkdir(exist_ok=True) dir.chmod(0o700) ctx.managed_paths.add(dir) ctx.keyhandler[name] = handler_cls(dir, signers_cfg, ctx.config) for keyhandler in ctx.keyhandler.values(): ctx.environ.update(keyhandler.env) @staticmethod def ensure_valid_if_signed(ctx, repo): if not repo.signed: return valid, keyid = repo.check_signature() keyhandler = ctx.keyhandler[repo.signers_type] info = keyhandler.get_key_repr(keyid) if keyid else 'No info' if valid: logging.info(f'Repository {repo.name} signature valid: {info}') return elif keyid: raise RepoRefError(f'Repository {repo.name} is not signed ' f'with a trusted key: {info}') raise RepoRefError(f'Repository {repo.name} is not signed ' 'with a trusted key.') siemens-kas-41ad961/kas/schema-kas.json000066400000000000000000000550131520561422700177610ustar00rootroot00000000000000{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://github.com/siemens/kas/blob/master/kas/schema-kas.json", "title": "kas configuration", "description": "kas, a setup tool for bitbake based projects", "type": "object", "$defs": { "layerPrio": { "type": "object", "additionalProperties": false, "properties": { "prio": { "description": "Include order priority, higher means earlier. The include priority is global, hence it is applied across all repositories.", "type": "integer", "minimum": -99, "maximum": 99, "default": 0 } } } }, "required": [ "header" ], "additionalProperties": false, "properties": { "header": { "description": "Header of every kas configuration file. It contains information about the context of the file.", "type": "object", "required": [ "version" ], "additionalProperties": false, "properties": { "version": { "description": "Version of the configuration file format.", "anyOf": [ { "const": "0.10" }, { "type": "integer", "minimum": 1, "maximum": 22 } ] }, "includes": { "description": "List of configuration files to include (parsed depth-first). They are merged in order they are stated. So a latter one could overwrite settings from previous files. The current file can overwrite settings from every included file.", "type": "array", "items": { "anyOf": [ { "description": "Path to a kas configuration file, relative to the repository root of the current file.", "type": "string" }, { "description": "If files from other repositories should be included, choose this (dict) representation.", "type": "object", "required": [ "repo", "file" ], "additionalProperties": false, "properties": { "repo": { "description": "Repository ``key`` the configuration file is located in.", "type": "string" }, "file": { "description": "Path to the configuration file relative to the repository.", "type": "string" } } } ] } } } }, "build_system": { "description": "Defines the bitbake-based build system.", "enum": [ "openembedded", "oe", "isar" ] }, "defaults": { "description": "Default values applied to all configuration nodes.", "type": "object", "additionalProperties": false, "properties": { "repos": { "description": "Default values for some repository properties.", "type": "object", "additionalProperties": false, "properties": { "branch": { "description": "Default ``branch`` property applied to all repositories that do not override this.", "type": "string" }, "tag": { "description": "Default ``tag`` property applied to all repositories that do not override this.", "type": "string" }, "refspec": { "description": "Deprecated: Use 'branch' / 'tag' instead.", "type": "string" }, "patches": { "description": "Default patches to apply to all repositories.", "type": "object", "additionalProperties": false, "properties": { "repo": { "type": "string" } } } } } } }, "overrides": { "description": "Overrides for specific configuration nodes. By that, only items that already exist are overridden. Note, that all entries below this key are reserved for auto-generation using kas plugins. Do not manually add entries.", "type": "object", "additionalProperties": false, "properties": { "repos": { "type": "object", "additionalProperties": { "type": "object", "additionalProperties": false, "properties": { "branch": { "description": "Branch in which to find the overridden commit, can be Null", "type": ["string", "null"] }, "commit": { "type": "string" } } } } } }, "machine": { "description": "Value of the ``MACHINE`` variable that is written into the ``local.conf``. Can be overwritten by the ``KAS_MACHINE`` environment variable.", "default": "qemux86-64", "type": "string" }, "distro": { "description": "Value of the ``DISTRO`` variable that is written into the ``local.conf``. Can be overwritten by the ``KAS_DISTRO`` environment variable.", "default": "nodistro", "type": "string" }, "env": { "description": "Environment variables to forward and their default values (set to nulltype to only forward if set). These variables are made available to bitbake via ``BB_ENV_PASSTHROUGH_ADDITIONS`` (``BB_ENV_EXTRAWHITE`` in older Bitbake versions) and can be overwritten by the variables of the environment in which kas is started.", "type": "object", "additionalProperties": { "type": ["string", "null"] } }, "target": { "description": "Single target or a list of targets to build by bitbake. Can be overwritten by the ``KAS_TARGET`` environment variable. Space is used as a delimiter if multiple targets should be specified via the environment variable. For targets prefixed with ``multiconfig:`` or ``mc:``, corresponding entries are added to the ``BBMULTICONFIG`` in ``local.conf``.", "default": "core-image-minimal", "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "task": { "description": "Task to build by bitbake. Can be overwritten by the ``KAS_TASK`` environment variable.", "default": "build", "type": "string" }, "repos": { "description": "Definitions of all available repos and layers. The layers are appended to the ``bblayers.conf`` sorted by the layer priority first (descending), then repository name and then by the layer name (ascending).", "type": "object", "additionalProperties": { "anyOf": [ { "description": "Definition of a repository and the layers, that should be part of the build. If the value is ``None``, the repository, where the current configuration file is located is defined as ```` and added as a layer to the build. It is recommended that the ```` is related to the containing repository/layer to ease cross-project referencing.", "type": "object", "additionalProperties": false, "properties": { "name": { "description": "Defines under which name the repository is stored. If its missing the ```` will be used.", "type": "string" }, "url": { "description": "URL of the repository. If this is missing, no version control operations are performed.", "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "type": { "description": "Type of version control repository.", "default": "git", "enum": ["git", "hg"] }, "commit": { "description": "Full-length commit ID (all-lowercase, no branch names, no symbolic refs, no tags) that should be used. If ``url`` was specified but no ``commit``, ``branch`` or ``tag``, the revision you get depends on the defaults of the version control system used.", "type": "string" }, "branch": { "description": "Upstream branch that should be tracked. If ``commit`` was specified, kas checks that the branch contains the commit. If no ``commit`` was specified, the head of the upstream branch is checked out. The nothing (``null``) value is used to remove a possible default value.", "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "tag": { "description": "Tag that should be checked out. If a ``commit`` was specified, kas checks that the tag points to this commit. This must not be combined with ``branch``. The nothing (``null``) value is used to remove a possible default value.", "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "refspec": { "description": "Deprecated. Use 'commit' / 'branch' / 'tag' instead.", "type": "string" }, "signed": { "description": "Whether the commit / tag must be signed. If enabled, 'allowed_signers' must be defined. If both tag and commit are specified, only the signature of the tag is checked. Git only.", "type": "boolean", "default": false }, "allowed_signers": { "description": "List of signer-ids to verify the signature against. Must be non-empty if 'signed=true'. Git only.", "type": "array", "items": { "type": "string" } }, "path": { "description": "Path where the repository is stored.", "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "layers": { "description": "List of layers to append to the ``bblayers.conf``. If this is missing or ``None`` or an empty dictionary, the path to the repo itself is added as a layer. Additionally, ``.`` is a valid value if the repo itself should be added as a layer.", "type": "object", "additionalProperties": { "anyOf": [ { "type": "null" }, { "description": "Exclude this layer from bblayers.conf.", "const": "disabled" }, { "description": "Deprecated, use ``disabled`` instead.", "enum": ["excluded", "n", "no", "0", "false", 0, false] }, { "$ref": "#/$defs/layerPrio" } ] } }, "patches": { "description": "Patches to apply to the repository.", "type": "object", "additionalProperties": { "anyOf": [ { "type": "object", "additionalProperties": false, "required": [ "path" ], "properties": { "repo": { "type": "string" }, "path": { "type": "string" } } }, { "type": "null" } ] } } } }, { "type": "null" } ] } }, "bblayers_conf_header": { "description": "Header to prepend to the ``bblayers.conf`` before any layers are included.", "type": "object", "additionalProperties": { "type": "string" } }, "local_conf_header": { "description": "Header to prepend to the ``local.conf``.", "type": "object", "additionalProperties": { "type": "string" } }, "proxy_config": { "description": "Deprecated. Proxy variables are automatically forwarded.", "type": "object", "additionalProperties": false, "properties": { "http_proxy": { "type": "string" }, "https_proxy": { "type": "string" }, "ftp_proxy": { "type": "string" }, "no_proxy": { "type": "string" } } }, "menu_configuration": { "description": "Menu configuration, auto-generated by menu plugin.", "type": "object", "additionalProperties": { "anyOf": [ { "type": "boolean" }, { "type": "string" }, { "type": "integer" } ] } }, "artifacts": { "description": "Artifacts which are expected to be present after executing the build (id / path pairs).", "type": "object", "additionalProperties": { "type": ["string"] } }, "signers": { "description": "Specification of the repository signature verification.", "type": "object", "additionalProperties": { "type": "object", "description": "Pairs of name and location of a public key or certificate. The name is used to reference the entry in the repository configuration.", "additionalProperties": false, "oneOf": [ {"required": ["path"]}, {"required": ["gpg_keyserver"]} ], "allOf": [ { "if": {"required": ["path"]}, "then": {"required": ["repo"]} }, { "if": {"required": ["gpg_keyserver"]}, "then": {"required": ["fingerprint"]} }, { "if": {"properties": {"type": {"const": "ssh"}}, "required": ["type"]}, "then": {"required": ["path", "repo"]} } ], "properties": { "type": { "description": "Type of the cryptographic material.", "enum": ["gpg", "ssh"], "default": "gpg" }, "repo": { "description": "Repo which provides the public key or certificate file." }, "path": { "description": "Path to the public key or certificate file, relative to the repo. Must be used together with 'repo'.", "type": "string" }, "fingerprint": { "description": "Fingerprint of the key or certificate (gpg: long-form, 40 hex digits, ssh: SHA256:...). If not provided, it is read from the file.", "anyOf": [ { "type": "string", "description": "GnuPG fingerprint of the key (long-form, 40 hex digits).", "minLength": 40, "maxLength": 40 }, { "type": "string", "description": "SSH fingerprint of the key (SHA256:...).", "minLength": 50, "maxLength": 50 } ] }, "gpg_keyserver": { "description": "GnuPG keyserver to use for key retrieval (if no local path is provided). The 'fingerprint' must be provided. GPG only.", "type": "string" } } } }, "_source_dir": { "description": "Path to the top repo at time of invoking the plugin (auto-generated by the menu plugin). It must not be set manually and might only be defined in the top-level ``.config.yaml`` file.", "type": "string" }, "_source_dir_host": { "description": "Source directory of the config file on the host (auto-generated by kas menu plugin, when using kas-container).", "type": "string" }, "buildtools": { "type": "object", "required": [ "version", "sha256sum" ], "properties": { "version": { "description": "Yocto Project version, as 5.0 (or even 5.0.8) for Scarthgap.", "type": "string" }, "sha256sum": { "description": "The installer's checksum, to perform integrity validation of the fetched artifact.", "type": "string" }, "base_url": { "default": "https://downloads.yoctoproject.org/releases/yocto", "description": "Base URL to fetch downloads from.", "type": "string" }, "filename": { "description": "Alternative name for the buildtools archive (.sh) to be downloaded.", "type": "string" } } } } } siemens-kas-41ad961/pyproject.toml000066400000000000000000000055271520561422700172150ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2021-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. [build-system] build-backend = "setuptools.build_meta" requires = [ "setuptools>=40.5", "wheel" ] [project] name = "kas" description = "Setup tool for bitbake based projects" readme = "README.rst" keywords = [ "OpenEmbedded bitbake development" ] license = { text = "MIT" } maintainers = [ { name = "Jan Kiszka", email = "jan.kiszka@siemens.com" }, ] requires-python = ">=3.9" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Build Tools", ] dynamic = [ "version" ] dependencies = [ "distro>=1,<2", "gitpython>=3.1,<4", "jsonschema>=3.2,<5", "pyyaml>=3,<7", ] optional-dependencies.tui = [ "kconfiglib>=14.1,<15" ] optional-dependencies.verify = [ "python-gnupg>=0.4.0,<1" ] optional-dependencies.test = [ "kas[tui]", "kas[verify]", "pytest>=6,<9", ] urls.Documentation = "https://kas.readthedocs.io/" urls.Homepage = "https://github.com/siemens/kas" urls.Repository = "https://github.com/siemens/kas.git" scripts.kas = "kas.kas:main" [tool.poetry] version = "1.0" [tool.setuptools] package-data = { "kas" = [ "*.json" ] } script-files = [ "kas-container" ] [tool.setuptools.packages.find] namespaces = false [tool.pytest.ini_options] markers = [ "online: tests requiring internet access", "dirsfromenv: parameterize test work dirs and build dirs", ] filterwarnings = [ "error", ] siemens-kas-41ad961/run-kas000077500000000000000000000023671520561422700156060ustar00rootroot00000000000000#!/usr/bin/env python3 # # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from kas import kas __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017' kas.main() siemens-kas-41ad961/scripts/000077500000000000000000000000001520561422700157575ustar00rootroot00000000000000siemens-kas-41ad961/scripts/build-container.sh000077500000000000000000000103521520561422700213760ustar00rootroot00000000000000#!/bin/sh # # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2024 # # Authors: # Jan Kiszka # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # This needs to be aligned with .github/actions/docker-init/action.yml BUILDKIT="moby/buildkit:v0.16.0" usage() { DEFAULT_DEBIAN_TAG=$(grep -m 1 'ARG DEBIAN_TAG=' "$(dirname "$0")/../Dockerfile" | sed 's/.*DEBIAN_TAG=\(.*\)-\(.*\)/\1--\2/') printf "%b" "Usage: $0 [OPTIONS]\n" printf "%b" "\nOptional arguments:\n" printf "%b" "--arch\t\tBuild for specified architecture, rather than the native one\n" printf "%b" "--clean\t\tRemove local images (ghcr.io/siemens/kas/TARGET:TAG) before\n" \ "\t\tstarting the build and do not use image cache\n" printf "%b" "--debian-tag\tUse specified tag for Debian base image\n" \ "\t\t(default=$DEFAULT_DEBIAN_TAG)\n" printf "%b" "--git-refspec\tUse specified revision/branch of kas repository (default=HEAD)\n" printf "%b" "--tag\t\tTag container with specified name (default=next)\n" printf "%b" "--target\tBuild specified target(s) (default=\"kas kas-isar\")\n" } build_image() { IMAGE_NAME="ghcr.io/siemens/kas/$1:$TAG" OLD_IMAGE_ID=$(docker images -q "$IMAGE_NAME" 2>/dev/null) PLATFORM_OPT= if [ -n "$ARCH" ]; then PLATFORM_OPT="--platform linux/$ARCH" fi NOCHACHE_OPT= if [ "$CLEAN" = y ]; then NOCHACHE_OPT="--no-cache" fi # shellcheck disable=SC2086 if ! docker buildx build --build-arg SOURCE_DATE_EPOCH="$(git log -1 --pretty=%ct)" \ --output type=docker,rewrite-timestamp=true \ --tag "$IMAGE_NAME" --build-arg DEBIAN_TAG="$DEBIAN_TAG" \ --target "$1" $PLATFORM_OPT $NOCHACHE_OPT .; then echo "Build failed!" return 1 fi if [ -n "$OLD_IMAGE_ID" ]; then if [ "$(docker images -q "$IMAGE_NAME")" = "$OLD_IMAGE_ID" ]; then echo "Reproduced identical image $IMAGE_NAME $OLD_IMAGE_ID" else echo "Deleting old image $OLD_IMAGE_ID" docker rmi "$OLD_IMAGE_ID" fi fi return 0 } ARCH= CLEAN= DEBIAN_TAG= GIT_REFSPEC=HEAD TARGETS= TAG=next while [ $# -gt 0 ]; do case "$1" in --arch) shift ARCH="$1" ;; --clean) CLEAN=y ;; --debian-tag) shift DEBIAN_TAG="$1" ;; --git-refspec) shift GIT_REFSPEC="$1" ;; --tag) shift TAG="$1" ;; --target) shift TARGETS="$TARGETS $1" ;; *) usage exit 1 esac shift done TARGETS="${TARGETS:-kas kas-isar}" if [ -z "$DEBIAN_TAG" ]; then DEBIAN_RELEASE=$(grep -m 1 'ARG DEBIAN_TAG=' "$(dirname "$0")/../Dockerfile" | sed 's/.*DEBIAN_TAG=\(.*\)-.*/\1/') DEBIAN_TAG=$(podman image search --list-tags debian --limit 1000000000 | \ grep "$DEBIAN_RELEASE-.*-slim" | sort -r | head -1 | sed 's/.*[ ]\+//') fi if [ "$CLEAN" = y ]; then for TARGET in $TARGETS; do docker rmi "ghcr.io/siemens/kas/$TARGET:$TAG" 2>/dev/null done fi if ! docker buildx inspect | grep -q "Driver Options:.*$BUILDKIT"; then BUILDX_INSTANCE=$(docker buildx create --driver-opt image="$BUILDKIT") docker buildx use "$BUILDX_INSTANCE" fi KAS_CLONE=$(mktemp -d --tmpdir kas-tmp.XXXXXXXXXX) git clone . "$KAS_CLONE" cd "$KAS_CLONE" || exit 1 git checkout -q "$GIT_REFSPEC" RESULT=0 for TARGET in $TARGETS; do if ! build_image "$TARGET"; then RESULT=1 break fi done cd - >/dev/null || exit 1 rm -rf "$KAS_CLONE" exit $RESULT siemens-kas-41ad961/scripts/checkcode.sh000077500000000000000000000013321520561422700202250ustar00rootroot00000000000000#!/bin/sh ERROR=0 if [ $# != 1 ]; then SRCDIR=$(dirname "$0")/.. else SRCDIR=$1 fi echo "Checking with pycodestyle" pycodestyle --ignore=W503,W606 "$SRCDIR"/*.py "$SRCDIR"/*/*.py || ERROR=$((ERROR + 1)) echo "Checking with flake8" flake8 "$SRCDIR" || ERROR=$((ERROR + 2)) echo "Checking with doc8" doc8 "$SRCDIR"/docs --ignore-path "$SRCDIR"/docs/_build --ignore D000 || ERROR=$((ERROR + 4)) echo "Checking with shellcheck" shellcheck "$SRCDIR"/kas-container \ "$SRCDIR"/scripts/release.sh \ "$SRCDIR"/scripts/checkcode.sh \ "$SRCDIR"/scripts/build-container.sh \ "$SRCDIR"/scripts/reproduce-container.sh \ "$SRCDIR"/container-entrypoint || ERROR=$((ERROR + 8)) exit $ERROR siemens-kas-41ad961/scripts/kas-container-usage-to-rst.sh000077500000000000000000000035031520561422700234050ustar00rootroot00000000000000#!/bin/sh # # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2024 # # Authors: # Felix Moessbauer # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # Extract the usage information of kas-container and convert it to rst # to be included in the documentation. cat - | \ sed 's/^Usage:/|SYNOPSIS|\n----------\n/g' | \ sed -e 's/^\s*kas-container /| kas-container /g' | \ # unwrap long lines perl -0pe 's/\n\s\s+/ /g' | \ sed 's/^Positional arguments:/|KAS-COMMANDS|\n--------------/g' | \ # each commands starts with a new line sed -r 's/^(build|checkout|diff|dump|lock|shell|for-all-repos|clean|cleansstate|cleanall|purge|menu)\t\t*(.*)$/:\1: \2/g' | \ sed 's/^Optional arguments:/|OPTIONS|\n---------/g' | \ sed '/^You can force/d' | \ cat siemens-kas-41ad961/scripts/lower-bound.py000077500000000000000000000004141520561422700205700ustar00rootroot00000000000000#!/usr/bin/env python3 # takes a reverse-sorted, line separated list and # returns the first element that is equal or smaller # than the first argument import sys for line in sys.stdin: if line.rstrip() <= sys.argv[1]: print(line.rstrip()) break siemens-kas-41ad961/scripts/release.sh000077500000000000000000000040011520561422700177310ustar00rootroot00000000000000#!/bin/bash OLD_VERSION=$(git describe --abbrev=0) NEW_VERSION=$1 usage() { echo "$0: NEW_VERSION" echo "" echo "example:" echo " $0 0.16.0" } if [ -z "$NEW_VERSION" ] ; then usage exit 1 fi echo "$NEW_VERSION" > newchangelog git shortlog "$OLD_VERSION".. >> newchangelog cat CHANGELOG.md >> newchangelog $EDITOR newchangelog echo -n "All fine, ready to release? [y/N]" read -r a a=$(echo "$a" | tr '[:upper:]' '[:lower:]') if [ "$a" != "y" ]; then echo "no not happy, let's stop doing the release" exit 1 fi mv newchangelog CHANGELOG.md sed -i "s,\(__version__ =\).*,\1 \'$NEW_VERSION\'," kas/__version__.py sed -i "s,^\(KAS_CONTAINER_SCRIPT_VERSION=\).*,\1\"$NEW_VERSION\"," kas-container git add CHANGELOG.md git add kas/__version__.py git add kas-container git commit -m "Release $NEW_VERSION" git tag -s -m "Release $NEW_VERSION" "$NEW_VERSION" git push --follow-tags python3 -m build twine upload -s -r pypi "dist/{kas-$NEW_VERSION.tar.gz,kas-$NEW_VERSION-py3-none-any.whl}" authors=$(git shortlog -s "$OLD_VERSION".."$NEW_VERSION" | cut -c8- | paste -s -d, - | sed -e 's/,/, /g') highlights=$(sed -e "/$OLD_VERSION$/,\$d" CHANGELOG.md) prolog=$PWD/release-email.txt echo \ "Hi all, !!! SET CONTAINER SHAs BEFORE SENDING !!! A new release $NEW_VERSION is available. A big thanks to all contributors: $authors Highlights in $highlights Thanks, Jan https://github.com/siemens/kas/releases/tag/$NEW_VERSION ($(git rev-parse "$NEW_VERSION")) https://github.com/orgs/siemens/packages/container/package/kas%2Fkas (ghcr.io/siemens/kas/kas:$NEW_VERSION@sha256:FILLME) https://github.com/orgs/siemens/packages/container/package/kas%2Fkas-isar (ghcr.io/siemens/kas/kas-isar:$NEW_VERSION@sha256:FILLME) "> "$prolog" git shortlog "$OLD_VERSION..$NEW_VERSION" >> "$prolog" thunderbird -compose "subject=[ANNOUNCE] Release $NEW_VERSION,to=kas-devel@googlegroups.com,message=$prolog" echo "Do not forget to update also the docs (https://readthedocs.org/projects/kas/versions/)! Done? [y/Y]" read -r siemens-kas-41ad961/scripts/reproduce-container.sh000077500000000000000000000047711520561422700222770ustar00rootroot00000000000000#!/bin/sh # # kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2024 # # Authors: # Jan Kiszka # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. if [ -z "$1" ] || [ "$1" = "--help" ]; then echo "Usage: $0 kas[-isar]: [architecture]" exit 0 fi TARGET=$(echo "$1" | sed 's/:.*//') TAG=$(echo "$1" | sed 's/.*://') ARCH=$2 ARCH_OPT= PLATFORM_OPT= if [ -n "$ARCH" ]; then ARCH_OPT="--arch $ARCH" PLATFORM_OPT="--platform linux/$ARCH" fi # shellcheck disable=SC2086 docker pull $PLATFORM_OPT "ghcr.io/siemens/kas/$TARGET:$TAG" DEBIAN_TAG=$(docker image inspect --format '{{json .Config.Env}}' \ "ghcr.io/siemens/kas/$TARGET:$TAG" | sed 's/.*DEBIAN_BASE_IMAGE_TAG=\([^"]\+\).*/\1/') if [ -z "$DEBIAN_TAG" ]; then echo "Cannot determine base image of ghcr.io/siemens/kas/$TARGET:$TAG" exit 1 fi GIT_REFSPEC="${TAG%%-debian-*}" if [ "$GIT_REFSPEC" = "latest" ]; then GIT_REFSPEC=master fi # shellcheck disable=SC2086 "$(dirname "$0")/build-container.sh" $ARCH_OPT --target "$TARGET" \ --tag repro-test --git-refspec "$GIT_REFSPEC" \ --debian-tag "$DEBIAN_TAG" --clean || exit 1 echo "" docker images --digests | grep "^REPOSITORY\|^ghcr.io/siemens/kas/${TARGET}[ ]*\($TAG\|repro-test\)" printf "%b" "\nReproduction test " if [ "$(docker images -q "ghcr.io/siemens/kas/$1")" = "$(docker images -q "ghcr.io/siemens/kas/$TARGET:repro-test")" ]; then printf "%b" "SUCCEEDED\n" docker rmi "ghcr.io/siemens/kas/$TARGET:repro-test" >/dev/null else printf "%b" "FAILED\n" fi siemens-kas-41ad961/setup.py000066400000000000000000000026761520561422700160150ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """ Setup script for kas, a setup tool for bitbake based projects """ import sys from setuptools import setup sys.path.append('.') __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2025' def get_version(): from kas import __version__ return __version__ setup( version=get_version(), ) siemens-kas-41ad961/tests/000077500000000000000000000000001520561422700154325ustar00rootroot00000000000000siemens-kas-41ad961/tests/conftest.py000066400000000000000000000150361520561422700176360ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2019-2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import signal import pytest import os import subprocess import shutil from pathlib import Path from kas import kas ENVVARS_KAS = [ 'KAS_WORK_DIR', 'KAS_BUILD_DIR', 'KAS_REPO_REF_DIR', 'KAS_DISTRO', 'KAS_MACHINE', 'KAS_TARGET', 'KAS_TASK', 'KAS_PREMIRRORS', 'KAS_CLONE_DEPTH', 'KAS_CONTAINER_SCRIPT_VERSION', 'SSH_PRIVATE_KEY', 'SSH_PRIVATE_KEY_FILE', 'SSH_AUTH_SOCK', 'CI_SERVER_HOST', 'CI_JOB_TOKEN', 'CI_PROJECT_DIR', 'GITLAB_CI', 'GITHUB_ACTIONS', 'REMOTE_CONTAINERS' ] ENVVARS_TOOLS = [ 'EMAIL' ] class KasEnvConfig: def __init__(self, work_dir: str, build_dir: str): self.work_dir = work_dir self.build_dir = build_dir KAS_OPERATING_DIRS = [ KasEnvConfig(None, None), KasEnvConfig("kas-work", None), KasEnvConfig("kas-work", "kas-build"), ] KAS_OPERATING_IDS = ["noenv", "work", "workbuild"] # only fan-out offline tests def pytest_generate_tests(metafunc): if "monkeykas" not in metafunc.fixturenames: return if next(metafunc.definition.iter_markers("dirsfromenv"), None): metafunc.parametrize("monkeykas", KAS_OPERATING_DIRS, ids=KAS_OPERATING_IDS, indirect=True) else: metafunc.parametrize("monkeykas", [KAS_OPERATING_DIRS[0]], ids=[KAS_OPERATING_IDS[0]], indirect=True) @pytest.fixture(autouse=True) def prepare_kas_for_tests(monkeypatch): """ Patch-out some global logic kas modifies. The strategy is as following: kas implements the problematic parts as dedicated functions which we monkeypatch here. """ def _register_signal_handlers(loop): loop.add_signal_handler(signal.SIGTERM, kas.interruption) # do not handle signal.SIGINT def _set_global_loglevel(level): # do not set global log level as this is handled by pytest pass monkeypatch.setattr(kas, "register_signal_handlers", _register_signal_handlers) monkeypatch.setattr(kas, "set_global_loglevel", _set_global_loglevel) @pytest.fixture() def monkeykas(request, monkeypatch, tmpdir): def get_kas_work_dir(): work_dir = os.environ.get('KAS_WORK_DIR', '.') return Path(work_dir).absolute() def get_kas_build_dir(): build_dir = os.environ.get('KAS_BUILD_DIR') if (not build_dir): return get_kas_work_dir() / 'build' return Path(build_dir).absolute() def move_to_workdir(orig_path): kas_wd = get_kas_work_dir() _orig = Path(orig_path) if not _orig.parent.samefile(kas_wd): shutil.move(_orig, kas_wd / _orig.basename()) monkeypatch.get_kwd = get_kas_work_dir monkeypatch.get_kbd = get_kas_build_dir monkeypatch.move_to_kwd = move_to_workdir _work_dir = request.param.work_dir if _work_dir: workdir = tmpdir / _work_dir workdir.mkdir() monkeypatch.setenv('KAS_WORK_DIR', str(workdir)) _build_dir = request.param.build_dir if _build_dir: builddir = workdir / _build_dir builddir.mkdir() monkeypatch.setenv('KAS_BUILD_DIR', str(builddir)) for var in ENVVARS_KAS + ENVVARS_TOOLS: monkeypatch.delenv(var, raising=False) # Set HOME to a temporary directory homedir = tmpdir / '_home' homedir.mkdir() monkeypatch.setenv('HOME', str(homedir)) # remove all VSCode devcontainers related variables for var in os.environ.keys(): if var.startswith('REMOTE_CONTAINERS_'): monkeypatch.delenv(var) # remove all git related variables for var in os.environ.keys(): if var.startswith('GIT_'): monkeypatch.delenv(var) # provide minimal git environment monkeypatch.setenv('GIT_AUTHOR_NAME', 'kas') monkeypatch.setenv('GIT_AUTHOR_EMAIL', 'kas@example.com') monkeypatch.setenv('GIT_COMMITTER_NAME', 'kas') monkeypatch.setenv('GIT_COMMITTER_EMAIL', 'kas@example.com') yield monkeypatch class MercurialRepo: """ Create a new Mercurial repository with a single commit. """ def __init__(self, tmpdir, name, branch=None): """ Creates a new Mercurial repository with a single commit. The content resembles the Makefile from the hello world example. """ self.repo = tmpdir / name self.repo.mkdir() subprocess.check_call(['hg', 'init'], cwd=self.repo) if branch: subprocess.check_call(['hg', 'branch', branch], cwd=self.repo) with open(self.repo / 'Makefile', 'w') as f: f.write('all:\n\techo hello\n') subprocess.check_call(['hg', 'add', 'Makefile'], cwd=self.repo) subprocess.check_call(['hg', '--config', 'ui.username=kas', 'commit', '-m', 'initial commit', '--date', '2024-11-10 08:00'], cwd=self.repo) def __enter__(self): return self.repo def get_commit(self): return subprocess.check_output( ['hg', 'log', '-r', '.', '--template', '{rev}:{node}'], cwd=self.repo).decode('utf-8').strip() def __exit__(self, exc_type, exc_value, traceback): self.repo.remove() @pytest.fixture def mercurial(): def make_hg_repo(tmpdir, name, branch=None): return MercurialRepo(tmpdir, name, branch) return make_hg_repo siemens-kas-41ad961/tests/test_attestation.py000066400000000000000000000043201520561422700214010ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2026 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import pytest from kas.attestation import Provenance @pytest.mark.parametrize('url,expected', [ # plain https ('https://example.com/siemens/kas.git', 'https://example.com/siemens/kas.git'), # https with user:password ('https://user:password@example.com/siemens/kas.git', 'https://example.com/siemens/kas.git'), # https with user only ('https://user@example.com/siemens/kas.git', 'https://example.com/siemens/kas.git'), # https with port and credentials ('https://user:password@example.com:8443/siemens/kas.git', 'https://example.com:8443/siemens/kas.git'), # http with credentials ('http://token:x-oauth-basic@example.com/repo.git', 'http://example.com/repo.git'), # ssh without credentials ('ssh://git@example.com/siemens/kas.git', 'ssh://example.com/siemens/kas.git'), # plain URL without credentials ('https://example.com/path?query=1#frag', 'https://example.com/path?query=1#frag'), # empty string ('', ''), ]) def test_strip_credentials(url, expected): assert Provenance._strip_credentials(url) == expected siemens-kas-41ad961/tests/test_build.py000066400000000000000000000055431520561422700201510ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import shutil import pytest import json from kas import kas from kas.kasusererror import ArtifactNotFoundError BITBAKE_OPTIONS_SHA256 = "e35d535e81cfdc4ed304af8000284c36" \ "19d2c4c78392ddcefe9ca46b158235f8" @pytest.mark.dirsfromenv def test_artifact_node(monkeykas, tmpdir): tdir = str(tmpdir / 'test_build') shutil.copytree('tests/test_build', tdir) monkeykas.chdir(tdir) kas.kas(['build', 'artifact-named.yml']) kas.kas(['build', 'artifact-glob.yml']) kas.kas(['build', 'artifact-invalid.yml']) @pytest.mark.online def test_provenance(monkeykas, tmpdir): tdir = str(tmpdir / 'test_build') shutil.copytree('tests/test_build', tdir) monkeykas.chdir(tdir) kas_bd = monkeykas.get_kbd() with pytest.raises(ArtifactNotFoundError): kas.kas(['build', '--provenance', 'mode=min', 'artifact-invalid.yml']) provenance_path = kas_bd / 'attestation/kas-build.provenance.json' kas.kas(['build', '--provenance', 'mode=min', 'provenance.yml']) with open(provenance_path, 'r') as f: prov = json.load(f) assert prov['subject'][0]['name'] == 'bitbake.options' assert 'env' not in \ prov['predicate']['buildDefinition']['internalParameters'] with monkeykas.context() as mp: mp.setenv('CAPTURE_THIS', 'OK Sir!') kas.kas(['build', '--provenance', 'mode=max', 'provenance.yml']) with open(provenance_path, 'r') as f: prov = json.load(f) params = prov['predicate']['buildDefinition']['internalParameters'] assert params['env']['CAPTURE_THIS'] == 'OK Sir!' assert prov['subject'][0]['name'] == 'bitbake.options' assert prov['subject'][0]['digest']['sha256'] == BITBAKE_OPTIONS_SHA256 siemens-kas-41ad961/tests/test_build/000077500000000000000000000000001520561422700175705ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_build/artifact-glob.yml000066400000000000000000000001141520561422700230250ustar00rootroot00000000000000header: version: 17 repos: this: artifacts: disk-file: bitbake.opt* siemens-kas-41ad961/tests/test_build/artifact-invalid.yml000066400000000000000000000001161520561422700235320ustar00rootroot00000000000000header: version: 17 repos: this: artifacts: disk-file: does-not-exist siemens-kas-41ad961/tests/test_build/artifact-named.yml000066400000000000000000000001171520561422700231710ustar00rootroot00000000000000header: version: 17 repos: this: artifacts: disk-file: bitbake.options siemens-kas-41ad961/tests/test_build/bitbake000077500000000000000000000000471520561422700211200ustar00rootroot00000000000000#!/bin/sh echo "$@" > bitbake.options siemens-kas-41ad961/tests/test_build/oe-init-build-env000066400000000000000000000000441520561422700227400ustar00rootroot00000000000000#!/bin/sh export PATH=$(pwd):$PATH siemens-kas-41ad961/tests/test_build/provenance.yml000066400000000000000000000003231520561422700224510ustar00rootroot00000000000000header: version: 17 env: CAPTURE_THIS: null repos: this: kas: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 artifacts: disk-file: bitbake.options siemens-kas-41ad961/tests/test_build_system.py000066400000000000000000000060461520561422700215540ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import shutil import pytest from kas import kas def test_build_system(monkeykas, tmpdir): tdir = str(tmpdir / 'test_build_system') shutil.copytree('tests/test_build_system', tdir) monkeykas.chdir(tdir) kas.kas(['shell', 'test-oe.yml', '-c', 'true']) with open('build-env', 'r') as f: assert f.readline().strip() == 'openembedded' kas.kas(['shell', 'test-isar.yml', '-c', 'true']) with open('build-env', 'r') as f: assert f.readline().strip() == 'isar' kas.kas(['shell', 'test-openembedded.yml', '-c', 'true']) with open('build-env', 'r') as f: assert f.readline().strip() == 'openembedded' @pytest.mark.dirsfromenv def test_gitconfig(monkeykas, tmpdir, capsys): tdir = str(tmpdir / 'test_gitconfig') shutil.copytree('tests/test_build_system', tdir) monkeykas.chdir(tdir) kas.kas(['shell', 'test-oe.yml', '-c', f'git config --get user.name > {tdir}/user.name']) with open(f'{tdir}/user.name', 'r') as f: assert f.readline().strip() == 'kas User' monkeykas.setenv('GITCONFIG_FILE', f'{tdir}/gitconfig') with open(f'{tdir}/gitconfig', 'w') as f: f.write('[user]\n') f.write('\temail = kas@kastest.io\n') f.write('[url "git@github.com:"]\n') f.write('\tinsteadOf = git://github\n') f.write('\tinsteadOf = git://github.io\n') kas.kas(['shell', 'test-oe.yml', '-c', f'git config --get user.email > {tdir}/user.email']) kas.kas(['shell', 'test-oe.yml', '-c', 'git config --get-all "url.git@github.com:.insteadof" ' f'> {tdir}/url']) # check if user is restored after patching with open(f'{tdir}/user.email', 'r') as f: assert f.readline().strip() == 'kas@kastest.io' # check the multi-key url rewrites with open(f'{tdir}/url', 'r') as f: assert f.readline().strip() == 'git://github' assert f.readline().strip() == 'git://github.io' siemens-kas-41ad961/tests/test_build_system/000077500000000000000000000000001520561422700211745ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_build_system/isar-init-build-env000077500000000000000000000000431520561422700247010ustar00rootroot00000000000000#!/bin/sh echo "isar" > build-env siemens-kas-41ad961/tests/test_build_system/oe-init-build-env000077500000000000000000000000531520561422700243470ustar00rootroot00000000000000#!/bin/sh echo "openembedded" > build-env siemens-kas-41ad961/tests/test_build_system/test-isar.yml000066400000000000000000000000721520561422700236310ustar00rootroot00000000000000header: version: 10 build_system: isar repos: this: siemens-kas-41ad961/tests/test_build_system/test-oe.yml000066400000000000000000000000701520561422700232740ustar00rootroot00000000000000header: version: 10 build_system: oe repos: this: siemens-kas-41ad961/tests/test_build_system/test-openembedded.yml000066400000000000000000000001021520561422700253000ustar00rootroot00000000000000header: version: 10 build_system: openembedded repos: this: siemens-kas-41ad961/tests/test_commands.py000066400000000000000000000502661520561422700206550ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Konsulko Group, 2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import glob import os import pathlib import shutil import json import yaml import subprocess import pytest from kas import kas from kas.libkas import TaskExecError, KasUserError, run_cmd from kas.attestation import file_digest_slow from kas.repos import RepoFetchError, RepoRefError @pytest.mark.dirsfromenv @pytest.mark.online def test_for_all_repos(monkeykas, tmpdir): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) kas.kas(['for-all-repos', 'test.yml', '''if [ -n "${KAS_REPO_URL}" ]; then git rev-parse HEAD \ >> %s/ref_${KAS_REPO_NAME}; fi''' % tdir]) with open('ref_kas_1.0', 'r') as f: assert f.readline().strip() \ == '907816a5c4094b59a36aec12226e71c461c05b77' with open('ref_kas_1.1', 'r') as f: assert f.readline().strip() \ == 'e9ca55a239caa1a2098e1d48773a29ea53c6cab2' @pytest.mark.online def test_for_all_repos_keep_config_unchanged(monkeykas, tmpdir): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) with pytest.raises(FileNotFoundError, match=r'.*/kas_1.[01]'): kas.kas(['for-all-repos', '--keep-config-unchanged', 'test.yml', 'pwd']) assert not os.path.exists('kas_1.0') assert not os.path.exists("kas_1.1") @pytest.mark.online def test_checkout(monkeykas, tmpdir): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) kas_bd = monkeykas.get_kbd() kas.kas(['checkout', 'test.yml']) # Ensure that local.conf and bblayers.conf are populated, check that no # build has been executed by ensuring that no tmp, sstate-cache or # downloads directories are present. assert (kas_bd / 'conf/local.conf').exists() assert (kas_bd / 'conf/bblayers.conf').exists() assert not glob.glob(f'{str(kas_bd)}/tmp*') assert not (kas_bd / 'downloads').exists() assert not (kas_bd / 'sstate-cache').exists() # test re-fetching (remove commit as nothing is fetched otherwise) with open('test.yml', 'r') as f: yml = yaml.safe_load(f) with open('test-no-commit.yml', 'w') as f: del yml['repos']['kas_1.1']['commit'] yml['repos']['kas_1.1']['branch'] = 'master' yaml.safe_dump(yml, f) kas.kas(['checkout', '--update', 'test-no-commit.yml']) with monkeykas.context() as mc: from kas.repos import GitRepo mc.setattr(GitRepo, "fetch_cmd", lambda _: ["false"]) with pytest.raises(RepoFetchError): kas.kas(['checkout', '--update', 'test-no-commit.yml']) @pytest.mark.parametrize( "file, error", [ ('test-invalid.yml', TaskExecError), ('test-invalid-tag.yml', RepoRefError), ], ) @pytest.mark.online def test_invalid_checkout(monkeykas, tmpdir, capsys, file, error): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) with pytest.raises(error): kas.kas(['checkout', file]) @pytest.mark.dirsfromenv @pytest.mark.online def test_checkout_with_ci_rewrite(monkeykas, tmpdir): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) project_dir = str(tmpdir.mkdir('build')) with monkeykas.context() as mp: mp.setenv('GITLAB_CI', 'true') mp.setenv('CI_SERVER_HOST', 'github.com') mp.setenv('CI_JOB_TOKEN', 'not-needed') mp.setenv('CI_PROJECT_DIR', project_dir) kas.kas(['checkout', 'test-url-rewrite.yml']) @pytest.mark.online def test_checkout_create_refs(monkeykas, tmpdir): tdir = str(tmpdir / 'test_commands') repo_cache = pathlib.Path(str(tmpdir.mkdir('repos'))) shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) monkeykas.setenv('KAS_REPO_REF_DIR', str(repo_cache)) kas_wd = monkeykas.get_kwd() kas.kas(['checkout', 'test.yml']) assert (repo_cache / 'github.com.siemens.kas.git').exists() assert (kas_wd / 'kas_1.0/.git/objects/info/alternates').exists() # check if refs are removed on purge kas.kas(['purge', 'test.yml']) assert not (repo_cache / 'github.com.siemens.kas.git').exists() @pytest.mark.online def test_checkout_shallow(monkeykas, tmpdir): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) with monkeykas.context() as mp: mp.setenv('KAS_CLONE_DEPTH', 'invalid') with pytest.raises(KasUserError): kas.kas(['checkout', 'test-shallow.yml']) with monkeykas.context() as mp: mp.setenv('KAS_CLONE_DEPTH', '1') kas.kas(['checkout', 'test-shallow.yml']) for repo in ['kas_1', 'kas_2', 'kas_3', 'kas_4', 'kas_5']: repo_path = monkeykas.get_kwd() / repo output = subprocess.check_output( ['git', 'rev-list', '--count', 'HEAD'], cwd=repo_path) count = int(output.decode('utf-8').strip()) if repo == 'kas_4': assert count >= 1 else: assert count == 1 @pytest.mark.online def test_shallow_updates(monkeykas, tmpdir): def _get_commit(repo): output = subprocess.check_output( ['git', 'rev-parse', '--verify', 'HEAD'], cwd=repo) return output.decode('utf-8').strip() tdir = tmpdir / 'test_commands' tdir.mkdir() shutil.copy('tests/test_commands/oe-init-build-env', tdir) monkeykas.chdir(tdir) monkeykas.setenv('KAS_CLONE_DEPTH', '1') kas_wd = monkeykas.get_kwd() # test non-pinned checkout of master branch base_yml = {'header': {'version': 15}, 'repos': { 'this': {}, 'kas': { 'url': 'https://github.com/siemens/kas.git', 'branch': 'master' }}} with open(tdir / 'kas.yml', 'w') as f: yaml.dump(base_yml, f) kas.kas(['checkout', 'kas.yml']) # switch branches, perform checkout again base_yml['repos']['kas']['branch'] = 'next' with open(tdir / 'kas.yml', 'w') as f: yaml.dump(base_yml, f) kas.kas(['checkout', 'kas.yml']) # pin commit on next branch commit = '5d1ab6e8ed3a12c7093c9041f104fb6a2db701a1' base_yml_lock = {'header': {'version': 15}, 'overrides': {'repos': {'kas': {'commit': commit}}}} with open(tdir / 'kas.lock.yml', 'w') as f: yaml.dump(base_yml_lock, f) kas.kas(['checkout', 'kas.yml']) assert _get_commit(kas_wd / 'kas') == commit # update to latest revision of next branch kas.kas(['checkout', '--update', 'kas.yml']) assert _get_commit(kas_wd / 'kas') != commit @pytest.mark.dirsfromenv @pytest.mark.online def test_repo_includes(monkeykas, tmpdir): tdir = tmpdir / 'test_commands' shutil.copytree('tests/test_repo_includes', tdir) monkeykas.chdir(tdir) monkeykas.move_to_kwd('subrepo') kas.kas(['checkout', 'test.yml']) @pytest.mark.online def test_dump(monkeykas, tmpdir, capsys): tdir = tmpdir / 'test_commands' shutil.copytree('tests/test_repo_includes', tdir) monkeykas.chdir(tdir) kas_bd = monkeykas.get_kbd() monkeykas.move_to_kwd('subrepo') formats = ['json', 'yaml'] resolve = ['', '--resolve-refs', '--resolve-env'] # test cross-product of these options (formats x resolve) for f, r in ((f, r) for f in formats for r in resolve): outfile = 'test_flat%s.%s' % (r, f) with monkeykas.context() as mp: if r == '--resolve-env': mp.setenv('TESTVAR_FOO', 'KAS') kas.kas(('dump --format %s %s test.yml' % (f, r)).split()) with open(outfile, 'w') as file: file.write(capsys.readouterr().out) with open(outfile, 'r') as cf: flatconf = json.load(cf) if f == 'json' else yaml.safe_load(cf) commit = flatconf['repos']['kas3'].get('commit', None) envvar = flatconf['env']['TESTVAR_FOO'] if r == '--resolve-refs': assert commit is not None else: assert commit is None if r == '--resolve-env': assert envvar == 'KAS' else: assert envvar == 'BAR' assert 'includes' not in flatconf['header'] # check if kas can read the generated file if f == 'yaml': shutil.rmtree(kas_bd, ignore_errors=True) kas.kas(('checkout %s' % outfile).split()) assert (kas_bd / 'conf/local.conf').exists() @pytest.mark.dirsfromenv @pytest.mark.online def test_diff(monkeykas, tmpdir, capsys): tdir = str(tmpdir / 'test_commands') shutil.copytree('tests/test_diff', tdir) monkeykas.chdir(tdir) diff = kas.kas(('diff diff1_folder/diff1.yml diff2_folder/diff2.yml') .split()) assert len(capsys.readouterr().out) > 0 diff = kas.kas(('diff diff4.yml:diff5.yml diff3.yml').split()) assert len(capsys.readouterr().out) > 0 diff = kas.kas(('diff diff3_cus_repopath.yml diff3.yml').split()) assert len(capsys.readouterr().out) > 0 diff = kas.kas(('diff --format json diff1_folder/diff1.yml ' 'diff2_folder/diff2.yml') .split()) outfile = 'diff-output.json' with open(outfile, 'w') as file: file.write(capsys.readouterr().out) with open(outfile, 'r') as cf: diff = json.load(cf) assert diff is not None assert diff["values_changed"] is not None assert diff["values_changed"]["target"]["old_value"] == "target1" assert diff["values_changed"]["target"]["new_value"] == "target2" assert diff["values_changed"]["machine"]["old_value"] == "machine1" assert diff["values_changed"]["machine"]["new_value"] == "machine2" assert diff["values_changed"]["distro"]["old_value"] == "distro1" assert diff["values_changed"]["distro"]["new_value"] == "distro2" assert len(diff["vcs"]["isar"]) == 59 @pytest.mark.dirsfromenv @pytest.mark.online def test_lockfile(monkeykas, tmpdir, capsys): tdir = tmpdir.mkdir('test_commands') shutil.rmtree(tdir, ignore_errors=True) shutil.copytree('tests/test_repo_includes', tdir) monkeykas.chdir(tdir) kas_wd = monkeykas.get_kwd() monkeykas.move_to_kwd('subrepo') # no lockfile yet, branches are floating kas.kas('dump test.yml'.split()) rawspec = yaml.safe_load(capsys.readouterr().out) assert rawspec['repos']['externalrepo']['refspec'] == 'master' with open(kas_wd / 'externalrepo/.git/refs/heads/master') as f: expected_commit = f.readline().strip() # create lockfile kas.kas('dump --lock --inplace test.yml'.split()) assert os.path.exists('test.lock.yml') # check if legacy dump -> lock redirection works with open('test.lock.yml', "rb") as f: hash_dump = file_digest_slow(f, 'sha256') os.remove('test.lock.yml') kas.kas('lock test.yml'.split()) with open('test.lock.yml', "rb") as f: hash_lock = file_digest_slow(f, 'sha256') assert hash_dump.hexdigest() == hash_lock.hexdigest() # lockfile is considered during import, expect pinned branches kas.kas('dump test.yml'.split()) lockspec = yaml.safe_load(capsys.readouterr().out) assert lockspec['overrides']['repos']['externalrepo']['commit'] \ == expected_commit # insert older commit into lockfile (kas post commit/branch introduction) test_commit = '226e92a7f30667326a63fd9812b8cc4a6184e398' # a commit message between test_commit and HEAD test_shortmsg = 'Release 4.8' lockspec['overrides']['repos']['externalrepo']['commit'] = test_commit with open('test.lock.yml', 'w') as f: yaml.safe_dump(lockspec, f) # check if repo is moved to specified commit kas.kas('dump test.yml'.split()) lockspec = yaml.safe_load(capsys.readouterr().out) assert lockspec['overrides']['repos']['externalrepo']['commit'] \ == test_commit # update lockfile, check if repo is pinned to other commit capsys.readouterr() kas.kas('dump --lock --inplace --update test.yml'.split()) assert test_shortmsg in capsys.readouterr().out with open('test.lock.yml', 'r') as f: lockspec = yaml.safe_load(f) assert lockspec['overrides']['repos']['externalrepo']['commit'] \ != test_commit # check if lockfile gets updated when repo is pinned to a nonexistent # commit nonexistent_commit = '0000000000000000000000000000000000000000' lockspec['overrides']['repos']['externalrepo']['commit'] = \ nonexistent_commit with open('test.lock.yml', 'w') as f: yaml.safe_dump(lockspec, f) kas.kas('dump --lock --inplace --update test.yml'.split()) with open('test.lock.yml', 'r') as f: lockspec = yaml.safe_load(f) assert lockspec['overrides']['repos']['externalrepo']['commit'] \ != nonexistent_commit @pytest.mark.dirsfromenv def test_root_resolve_novcs(monkeykas, tmpdir, capsys): tdir = str(tmpdir.mkdir('test_root_resolve_novcs')) shutil.rmtree(tdir, ignore_errors=True) shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) capsys.readouterr() kas.kas('dump --resolve-local test-local.yml'.split()) console = capsys.readouterr() assert 'not under version control' in console.err data = yaml.safe_load(console.out) assert data['repos']['local'] is None @pytest.mark.dirsfromenv def test_root_resolve_git(monkeykas, tmpdir, capsys): tdir = str(tmpdir.mkdir('test_root_resolve_git')) shutil.rmtree(tdir, ignore_errors=True) shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) upstream_url = 'http://github.com/siemens/kas.git' subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'branch', '-m', 'main']) subprocess.check_call(['git', 'add', 'oe-init-build-env', '*.yml']) subprocess.check_call(['git', 'commit', '-m', 'test']) subprocess.check_call(['git', 'remote', 'add', 'origin', upstream_url]) commit = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD']) capsys.readouterr() kas.kas('dump --resolve-local test-local.yml'.split()) console = capsys.readouterr() data = yaml.safe_load(console.out) assert data['repos']['local']['commit'] == commit.decode('utf-8').strip() assert data['repos']['local']['url'] == upstream_url # make repository dirty with open(f'{tdir}/new-file.txt', 'w') as f: f.write('test') kas.kas('dump --resolve-local test-local.yml'.split()) console = capsys.readouterr() data = yaml.safe_load(console.out) assert data['repos']['local']['commit'] == commit.decode('utf-8').strip() assert data['repos']['local']['url'] == upstream_url @pytest.mark.dirsfromenv def test_root_resolve_hg(monkeykas, tmpdir, capsys): tdir = str(tmpdir.mkdir('test_root_resolve_hg')) shutil.rmtree(tdir, ignore_errors=True) shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) upstream_url = 'http://github.com/siemens/kas' subprocess.check_call(['hg', 'init']) subprocess.check_call(['hg', 'add', 'test-local-hg.yml', 'oe-init-build-env']) subprocess.check_call(['hg', 'commit', '-m', 'test']) with open(f'{tdir}/.hg/hgrc', mode='w') as f: f.write(f'[paths]\ndefault = {upstream_url}') commit = subprocess.check_output(['hg', 'log', '-r', '.', '--template', '{node}\n']) capsys.readouterr() kas.kas('dump --resolve-local test-local-hg.yml'.split()) console = capsys.readouterr() data = yaml.safe_load(console.out) assert data['repos']['local']['commit'] == commit.decode('utf-8').strip() assert data['repos']['local']['url'] == upstream_url assert data['repos']['local']['type'] == 'hg' # add all files to repository subprocess.check_call(['hg', 'add']) subprocess.check_call(['hg', 'commit', '-m', 'test2']) kas.kas('dump --resolve-local test-local-hg.yml'.split()) def test_ff_merges(monkeykas, tmpdir): """ This tests check if kas correcly handles fast-forward merges. """ tdir = tmpdir / 'test_commands' rdirbare = tdir / 'upstream.git' rdirwork = tdir / 'upstream.clone' wdirkas = tdir / 'kas' for d in [tdir, rdirbare, rdirwork, wdirkas]: d.mkdir() with open('tests/test_commands/test-ff-merges.yml', 'r') as f: kas_input = yaml.safe_load(f) shutil.copy('tests/test_commands/oe-init-build-env', rdirwork) # create a bare upstream repository monkeykas.chdir(rdirbare) subprocess.check_call(['git', 'init', '--bare']) subprocess.check_call(['git', 'symbolic-ref', 'HEAD', 'refs/heads/main']) # create a repository with a branch (working copy) monkeykas.chdir(rdirwork) subprocess.check_call(['git', 'init']) subprocess.check_call(['git', 'branch', '-m', 'main']) subprocess.check_call(['git', 'add', 'oe-init-build-env']) subprocess.check_call(['git', 'commit', '-m', 'test']) # get head of main branch (before merge) c_main = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD']) subprocess.check_call(['git', 'checkout', '-b', 'feature']) subprocess.check_call(['touch', 'foo']) subprocess.check_call(['git', 'add', 'foo']) subprocess.check_call(['git', 'commit', '-m', 'test-feature']) # get head of feature branch c_feat = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD']) subprocess.check_call(['git', 'remote', 'add', 'origin', rdirbare]) subprocess.check_call(['git', 'push', 'origin', 'main', 'feature']) # perform initial kas checkout monkeykas.chdir(wdirkas) kas_input['repos']['upstream']['commit'] = c_main.decode('utf-8').strip() kas_input['repos']['upstream']['url'] = str(rdirbare) with open('kas.yml', 'w') as f: yaml.dump(kas_input, f) kas.kas(['checkout', 'kas.yml']) # ff merge feature into main monkeykas.chdir(rdirwork) # checkout main branch with kas subprocess.check_call(['git', 'checkout', 'main']) subprocess.check_call(['git', 'merge', '--ff-only', 'feature']) subprocess.check_call(['git', 'push', 'origin', 'main']) # bump commit on main branch in kas project, perform kas checkout monkeykas.chdir(wdirkas) # checkout main branch with kas again kas_input['repos']['upstream']['commit'] = c_feat.decode('utf-8').strip() with open('kas.yml', 'w') as f: yaml.dump(kas_input, f) kas.kas(['checkout', 'kas.yml']) def test_cmd_not_found(monkeykas, tmpdir): cmd = ['/usr/bin/kas-not-exists'] ret, _ = run_cmd(cmd, tmpdir, os.environ, fail=False) assert ret != 0 with pytest.raises(FileNotFoundError): run_cmd(cmd, tmpdir, os.environ, fail=True) @pytest.mark.dirsfromenv def test_build_dir_add_cachedir_tag(monkeykas, tmpdir): tdir = str(tmpdir.mkdir('test_build_dir_add_cachedir_tag')) shutil.rmtree(tdir, ignore_errors=True) shutil.copytree('tests/test_commands', tdir) monkeykas.chdir(tdir) kas.kas(['checkout', 'test-local.yml']) cachetag = monkeykas.get_kbd() / 'CACHEDIR.TAG' assert cachetag.exists() with cachetag.open() as f: assert f.readline().startswith('Signature:') siemens-kas-41ad961/tests/test_commands/000077500000000000000000000000001520561422700202725ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_commands/oe-init-build-env000077500000000000000000000000201520561422700234370ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_commands/test-ff-merges.yml000066400000000000000000000001161520561422700236430ustar00rootroot00000000000000header: version: 14 repos: upstream: path: upstream branch: main siemens-kas-41ad961/tests/test_commands/test-invalid-tag.yml000066400000000000000000000003021520561422700241640ustar00rootroot00000000000000header: version: 15 repos: kas_invalid: url: https://github.com/siemens/kas.git tag: '4.3' # non matching commit (tag: 4.2) commit: 23211cbd3518936d83169722d1eb40656344ba27 siemens-kas-41ad961/tests/test_commands/test-invalid.yml000066400000000000000000000001621520561422700234170ustar00rootroot00000000000000header: version: 14 repos: kas_invalid: url: https://example.com/kas.git branch: this-ref-is-invalid siemens-kas-41ad961/tests/test_commands/test-local-hg.yml000066400000000000000000000000641520561422700234600ustar00rootroot00000000000000header: version: 15 repos: local: type: hg siemens-kas-41ad961/tests/test_commands/test-local.yml000066400000000000000000000000471520561422700230650ustar00rootroot00000000000000header: version: 15 repos: local: siemens-kas-41ad961/tests/test_commands/test-shallow.yml000066400000000000000000000011571520561422700234470ustar00rootroot00000000000000header: version: 14 repos: this: kas_1: url: https://github.com/siemens/kas.git branch: master kas_2: url: https://github.com/siemens/kas.git tag: '4.3' commit: f650ebe2495a9cbe2fdf4a2c8becc7b3db470d55 kas_3: url: https://github.com/siemens/kas.git commit: e42a64a666082b77fbc2758b07191b662d17f792 kas_4: url: https://github.com/siemens/kas.git # keep legacy refspec here for testing purposes refspec: master kas_5: url: https://github.com/siemens/kas.git tag: '4.3' # commid id of tag object 4.3: commit: beb5b60c6823ec53300efb3b854a5a921b22bd3d siemens-kas-41ad961/tests/test_commands/test-url-rewrite.yml000066400000000000000000000002131520561422700242470ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: git@github.com:siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 siemens-kas-41ad961/tests/test_commands/test.yml000066400000000000000000000004001520561422700217660ustar00rootroot00000000000000header: version: 14 repos: this: kas_1.0: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 kas_1.1: url: https://github.com/siemens/kas.git commit: e9ca55a239caa1a2098e1d48773a29ea53c6cab2 siemens-kas-41ad961/tests/test_diff/000077500000000000000000000000001520561422700174015ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_diff/diff1_folder/000077500000000000000000000000001520561422700217255ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_diff/diff1_folder/diff1.yml000066400000000000000000000000661520561422700234430ustar00rootroot00000000000000header: version: 14 includes: - subdir/sub.yml siemens-kas-41ad961/tests/test_diff/diff1_folder/subdir/000077500000000000000000000000001520561422700232155ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_diff/diff1_folder/subdir/sub.yml000066400000000000000000000005721520561422700245350ustar00rootroot00000000000000header: version: 14 build_system: isar distro: distro1 target: target1 machine: machine1 defaults: repos: patches: repo: test repos: test: layers: .: isar: url: https://github.com/ilbers/isar tag: v0.10 local_conf_header: swu_hw_compat: | # Set hardware compatibility versions here SWU_HW_COMPAT = "sdc-compatible-machine-1.0" siemens-kas-41ad961/tests/test_diff/diff2_folder/000077500000000000000000000000001520561422700217265ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_diff/diff2_folder/diff2.yml000066400000000000000000000010121520561422700234350ustar00rootroot00000000000000header: version: 15 includes: - subdir/sub.yml distro: distro2 target: target2 machine: machine2 defaults: repos: patches: repo: meta-iot2050 repos: meta-iot2050: layers: .: isar: url: https://github.com/ilbers/isar tag: v0.10-rc1 local_conf_header: wic-swu: | IMAGE_CLASSES += "squashfs" IMAGE_TYPEDEP:wic += "squashfs" swu_hw_compat: | # Set hardware compatibility versions here SWU_HW_COMPAT = "sdc-compatible-machine-2.0" siemens-kas-41ad961/tests/test_diff/diff2_folder/subdir/000077500000000000000000000000001520561422700232165ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_diff/diff2_folder/subdir/sub.yml000066400000000000000000000005721520561422700245360ustar00rootroot00000000000000header: version: 14 build_system: isar distro: distro1 target: target1 machine: machine1 defaults: repos: patches: repo: test repos: test: layers: .: isar: url: https://github.com/ilbers/isar tag: v0.10 local_conf_header: swu_hw_compat: | # Set hardware compatibility versions here SWU_HW_COMPAT = "sdc-compatible-machine-1.0" siemens-kas-41ad961/tests/test_diff/diff3.yml000066400000000000000000000005761520561422700211270ustar00rootroot00000000000000header: version: 14 build_system: isar distro: distro3 target: target3 machine: machine3 defaults: repos: patches: repo: test repos: test: layers: .: isar: url: https://github.com/ilbers/isar tag: v0.10-rc1 local_conf_header: swu_hw_compat: | # Set hardware compatibility versions here SWU_HW_COMPAT = "sdc-compatible-machine-2.0" siemens-kas-41ad961/tests/test_diff/diff3_cus_repopath.yml000066400000000000000000000006141520561422700236740ustar00rootroot00000000000000header: version: 14 build_system: isar distro: distro3 target: target3 machine: machine3 defaults: repos: patches: repo: test repos: test: layers: .: isar: url: https://github.com/ilbers/isar tag: v0.10 path: isarcus local_conf_header: swu_hw_compat: | # Set hardware compatibility versions here SWU_HW_COMPAT = "sdc-compatible-machine-2.0" siemens-kas-41ad961/tests/test_diff/diff4.yml000066400000000000000000000003641520561422700211230ustar00rootroot00000000000000header: version: 14 build_system: isar distro: distro4 target: target4 machine: machine4 defaults: repos: patches: repo: test repos: test: layers: .: isar: url: https://github.com/ilbers/isar tag: v0.10 siemens-kas-41ad961/tests/test_diff/diff5.yml000066400000000000000000000003441520561422700211220ustar00rootroot00000000000000header: version: 14 build_system: isar distro: distro5 target: target5 machine: machine5 local_conf_header: swu_hw_compat: | # Set hardware compatibility versions here SWU_HW_COMPAT = "sdc-compatible-machine-2.0" siemens-kas-41ad961/tests/test_environment_variables.py000066400000000000000000000244251520561422700234460ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Peter Hatina, 2021 # Copyright (c) Siemens AG, 2021-2022 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import shutil import pathlib import subprocess import re import pytest import json import logging from kas import kas from kas.context import create_global_context from kas.kasusererror import ArgsCombinationError, EnvSetButNotFoundError from kas.libcmds import SetupHome from kas import __version__ @pytest.mark.dirsfromenv def test_build_dir_is_placed_inside_work_dir_by_default(monkeykas, tmpdir): conf_dir = str(tmpdir / 'test_env_variables') shutil.copytree('tests/test_environment_variables', conf_dir) monkeykas.chdir(conf_dir) kas.kas(['checkout', 'test.yml']) assert (monkeykas.get_kbd() / 'conf').exists() def test_build_dir_can_be_specified_by_environment_variable(monkeykas, tmpdir): conf_dir = str(tmpdir / 'test_env_variables') build_dir = str(tmpdir / 'test_build_dir') shutil.copytree('tests/test_environment_variables', conf_dir) monkeykas.chdir(conf_dir) monkeykas.setenv('KAS_BUILD_DIR', build_dir) kas.kas(['checkout', 'test.yml']) assert os.path.exists(os.path.join(build_dir, 'conf')) @pytest.mark.dirsfromenv def test_ssh_agent_setup(monkeykas, tmpdir, capsys): conf_dir = str(tmpdir / 'test_ssh_agent_setup') shutil.copytree('tests/test_environment_variables', conf_dir) monkeykas.chdir(conf_dir) SSH_AUTH_SOCK = '/tmp/ssh-KLTafE/agent.64708' with monkeykas.context() as mp: envfile = tmpdir / 'env' mp.setenv('SSH_AUTH_SOCK', SSH_AUTH_SOCK) kas.kas(['shell', '-c', f'env > {envfile}', 'test.yml']) env = _get_env_from_file(envfile) assert env['SSH_AUTH_SOCK'] == SSH_AUTH_SOCK with monkeykas.context() as mp: mp.setenv('SSH_AUTH_SOCK', SSH_AUTH_SOCK) mp.setenv('SSH_PRIVATE_KEY', 'id_rsa') with pytest.raises(ArgsCombinationError): kas.kas(['checkout', 'test.yml']) privkey_file = f'{tmpdir}/id_ecdsa_test' genkey_cmd = ['ssh-keygen', '-f', privkey_file, '-N', '', '-t', 'ecdsa'] subprocess.check_call(genkey_cmd) # ensure we also get the info messages log = kas.logging.getLogger() log.setLevel(kas.logging.INFO) # flush the captured output capsys.readouterr() with monkeykas.context() as mp: mp.setenv('SSH_PRIVATE_KEY_FILE', privkey_file) kas.kas(['checkout', 'test.yml']) out = capsys.readouterr().err assert 'adding SSH key from file' in out assert 'ERROR' not in out with monkeykas.context() as mp: privkey = pathlib.Path(privkey_file).read_text() mp.setenv('SSH_PRIVATE_KEY', privkey) kas.kas(['checkout', 'test.yml']) out = capsys.readouterr().err assert 'adding SSH key from env-var' in out assert 'ERROR' not in out def _get_env_from_file(filename): env = {} with filename.open() as f: for line in f.readlines(): key, val = line.split("=", 1) env[key] = val.strip() return env def _test_env_section_export(monkeykas, tmpdir, bb_env_var, bb_repo): conf_dir = tmpdir / 'test_env_variables' env_out = conf_dir / 'env_out' bb_env_out = conf_dir / 'bb_env_out' init_build_env = conf_dir / 'oe-init-build-env' shutil.copytree('tests/test_environment_variables', conf_dir) monkeykas.chdir(conf_dir) monkeykas.setenv('KAS_CLONE_DEPTH', '1') monkeykas.setenv('KAS_PREMIRRORS', 'https://git\\.openembedded\\.org/ ' 'https://github.com/openembedded/\n') # resolve after we chdir'd kas_wd = monkeykas.get_kwd() kas_bd = monkeykas.get_kbd() # Overwrite oe-init-build-env script # BB_ENV_* filter variables are only exported by # kas when they are already exported in the setup environment script script = """#!/bin/sh export %s="FOO" export PATH="%s/%s/bin:${PATH}" """ % (bb_env_var, str(kas_wd), bb_repo) init_build_env.write_text(script, encoding=None) init_build_env.chmod(0o775) # Before executing bitbake, first get the bitbake.conf kas.kas(['checkout', 'test_env.yml']) shutil.copy(kas_wd / bb_repo / 'conf' / 'bitbake.conf', kas_bd / 'conf' / 'bitbake.conf') kas.kas(['shell', '-c', 'env > %s' % env_out, 'test_env.yml']) kas.kas(['shell', '-c', 'bitbake -e > %s' % bb_env_out, 'test_env.yml']) # Check kas environment test_env = _get_env_from_file(env_out) # Variables with 'None' assigned should not be added to environment try: _ = test_env['TESTVAR_WHITELIST'] assert False except KeyError: assert True assert test_env['TESTVAR_DEFAULT_VAL'] == 'BAR' assert 'TESTVAR_WHITELIST' in test_env[bb_env_var] # Check bitbake's environment test_bb_env = {} with bb_env_out.open() as f: for line in f.readlines(): if re.match(r'^#', line): continue match = re.match(r'(^[a-zA-Z0-9_]+)=\"([a-zA-Z0-9_ ]+)\"', line) if match: key, val = match.group(1), match.group(2) test_bb_env[key] = val.strip() assert 'TESTVAR_WHITELIST' in test_bb_env[bb_env_var] assert test_bb_env["TESTVAR_DEFAULT_VAL"] == "BAR" # BB_ENV_EXTRAWHITE is deprecated but may still be used @pytest.mark.online def test_env_section_export_bb_extra_white(monkeykas, tmpdir): _test_env_section_export(monkeykas, tmpdir, 'BB_ENV_EXTRAWHITE', 'bitbake_old') @pytest.mark.online def test_env_section_export_bb_env_passthrough_additions(monkeykas, tmpdir): _test_env_section_export(monkeykas, tmpdir, 'BB_ENV_PASSTHROUGH_ADDITIONS', 'bitbake_new') def test_managed_env_detection(monkeykas): with monkeykas.context() as mp: mp.setenv('GITLAB_CI', 'true') ctx = create_global_context([]) me = ctx.managed_env assert bool(me) assert str(me) == 'GitLab CI' with monkeykas.context() as mp: mp.setenv('GITHUB_ACTIONS', 'true') ctx = create_global_context([]) me = ctx.managed_env assert bool(me) assert str(me) == 'GitHub Actions' with monkeykas.context() as mp: mp.setenv('REMOTE_CONTAINERS', 'true') mp.setenv('REMOTE_CONTAINERS_FOO', 'bar') ctx = create_global_context([]) me = ctx.managed_env assert bool(me) assert str(me) == 'VSCode Remote Containers' assert ctx.environ['REMOTE_CONTAINERS_FOO'] == 'bar' @pytest.mark.dirsfromenv def test_env_file_processing(monkeykas, tmpdir): ctx = create_global_context([]) rcfiles = [ ('NETRC_FILE', '.netrc'), ('NPMRC_FILE', '.npmrc'), ('REGISTRY_AUTH_FILE', '.docker/config.json'), ('AWS_CONFIG_FILE', '.aws/config'), ('AWS_WEB_IDENTITY_TOKEN_FILE', '.aws/web_identity_token') # no need to test gitconfig, as tested in test_gitconfig ] for name, file in rcfiles: with monkeykas.context() as mp: local_file = tmpdir / '.rcfile' with open(local_file, 'w') as f: if name in ['NETRC_FILE', 'NPMRC_FILE']: f.write('machine foo\nlogin bar\npassword baz\n') elif name in ['AWS_CONFIG_FILE', 'AWS_WEB_IDENTITY_TOKEN_FILE']: f.write('[profile foo]\nregion=bar\n') elif name == 'REGISTRY_AUTH_FILE': json.dump({'auths': {'example.com': {'auth': 'foo'}}}, f) else: f.write('foo: bar\n') mp.setenv(name, str(local_file)) if name in ['AWS_CONFIG_FILE', 'AWS_WEB_IDENTITY_TOKEN_FILE']: mp.setenv('AWS_SHARED_CREDENTIALS_FILE', str(local_file)) mp.setenv('AWS_ROLE_ARN', 'arn:foo') if name == 'AWS_WEB_IDENTITY_TOKEN_FILE': mp.setenv('AWS_CONFIG_FILE', str(local_file)) elif name == 'REGISTRY_AUTH_FILE': mp.setenv('CI_JOB_TOKEN', 'foo') mp.setenv('CI_REGISTRY_USER', 'reg-user') mp.setenv('CI_REGISTRY', 'example.com') sh = SetupHome() sh.execute(ctx) assert (pathlib.Path(sh.tmpdirname) / pathlib.Path(file)).exists() def test_env_set_but_not_existing(monkeykas): with monkeykas.context() as mp: mp.setenv('NETRC_FILE', '/path/does/not/exist') ctx = create_global_context([]) with pytest.raises(EnvSetButNotFoundError): SetupHome().execute(ctx) def test_kas_container_version(monkeykas, caplog): caplog.set_level(logging.WARNING) with monkeykas.context() as mp: # not a kas-container call create_global_context([]) assert 'versions do not match' not in caplog.text caplog.clear() with monkeykas.context() as mp: # kas-container call from matching script mp.setenv('KAS_CONTAINER_SCRIPT_VERSION', __version__) create_global_context([]) assert 'versions do not match' not in caplog.text caplog.clear() with monkeykas.context() as mp: # kas-container call from older/newer script mp.setenv('KAS_CONTAINER_SCRIPT_VERSION', '0.0') create_global_context([]) assert 'versions do not match' in caplog.text caplog.clear() siemens-kas-41ad961/tests/test_environment_variables/000077500000000000000000000000001520561422700230655ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_environment_variables/meta-dummy/000077500000000000000000000000001520561422700251445ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_environment_variables/meta-dummy/classes/000077500000000000000000000000001520561422700266015ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_environment_variables/meta-dummy/classes/base.bbclass000066400000000000000000000000001520561422700310340ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_environment_variables/meta-dummy/conf/000077500000000000000000000000001520561422700260715ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_environment_variables/meta-dummy/conf/layer.conf000066400000000000000000000001171520561422700300530ustar00rootroot00000000000000# We have a conf and classes directory, add to BBPATH BBPATH .= ":${LAYERDIR}" siemens-kas-41ad961/tests/test_environment_variables/oe-init-build-env000077500000000000000000000000201520561422700262320ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_environment_variables/test.yml000066400000000000000000000000461520561422700245670ustar00rootroot00000000000000header: version: 10 repos: this: siemens-kas-41ad961/tests/test_environment_variables/test_env.yml000066400000000000000000000007621520561422700254440ustar00rootroot00000000000000header: version: 14 env: TESTVAR_DEFAULT_VAL: "BAR" TESTVAR_WHITELIST: repos: this: layers: meta-dummy: # Testing new BB_ENV_PASSTHROUGH_ADDITIONS bitbake_new: url: https://git.openembedded.org/bitbake commit: 87104b6a167188921da157c7dba45938849fb22a layers: .: excluded # Testing deprecated BB_ENV_WHITELIST bitbake_old: url: https://git.openembedded.org/bitbake commit: efaafc9ec2e8c0475e3fb27e877a1c0a5532a0e5 layers: .: excluded siemens-kas-41ad961/tests/test_includehandler.py000066400000000000000000000452211520561422700220300ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2017-2018 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import io import textwrap import contextlib import pytest from kas import includehandler, context from kas.includehandler import ConfigFile @pytest.fixture(autouse=True) def fixed_version(monkeypatch): monkeypatch.setattr(includehandler, '__file_version__', 5) monkeypatch.setattr(includehandler, '__compatible_file_version__', 4) @pytest.fixture(autouse=True) def with_kas_context(): context.create_global_context(None) yield context.__context__ = None class MockFileIO(io.StringIO): def close(self): self.seek(0) def mock_file(indented_content): return MockFileIO(textwrap.dedent(indented_content)) @contextlib.contextmanager def patch_open(component, string='', dictionary=None): dictionary = dictionary or {} old_attr = getattr(component, 'open', None) component.open = lambda f, *a, **k: mock_file(dictionary.get(f, string)) yield if old_attr: component.open = old_attr else: del component.open class TestLoadConfig: def test_err_invalid_ext(self): # Test for invalid file extension: exception = includehandler.LoadConfigException with pytest.raises(exception): ConfigFile.load('x.xyz') def util_exception_content(self, testvector): for string, exception in testvector: with patch_open(includehandler, string=string): with pytest.raises(exception): ConfigFile.load('x.yml') def test_err_header_missing(self): exception = includehandler.LoadConfigException testvector = [ ('', exception), ('a', exception), ('1', exception), ('a:', exception) ] self.util_exception_content(testvector) def test_err_header_invalid_type(self): exception = includehandler.LoadConfigException testvector = [ ('header:', exception), ('header: 1', exception), ('header: a', exception), ('header: []', exception), ] self.util_exception_content(testvector) def test_err_version_missing(self): exception = includehandler.LoadConfigException testvector = [ ('header: {}', exception), ('header: {a: 1}', exception), ] self.util_exception_content(testvector) def test_err_version_invalid_format(self): exception = includehandler.LoadConfigException testvector = [ ('header: {version: "0.5"}', exception), ('header: {version: "x"}', exception), ('header: {version: 3}', exception), ('header: {version: 6}', exception), ] self.util_exception_content(testvector) def test_err_parse_yaml(self): exception = includehandler.LoadConfigException testvector = [ # misaligned column ('header:\n version: 17\n repo:', exception), ] self.util_exception_content(testvector) def test_header_valid(self): testvector = [ 'header: {version: 4}', 'header: {version: 5}', ] for string in testvector: with patch_open(includehandler, string=string): ConfigFile.load('x.yml') def test_compat_version(self, monkeypatch): monkeypatch.setattr(includehandler, '__compatible_file_version__', 1) with patch_open(includehandler, string='header: {version: "0.10"}'): ConfigFile.load('x.yml') def test_source_dir_rejected_in_non_main_file(self): string = 'header: {version: 5}\n_source_dir: /some/path' with patch_open(includehandler, string=string): with pytest.raises(includehandler.LoadConfigException): ConfigFile.load('x.yml', is_main_file=False) def test_source_dir_allowed_in_main_file(self): string = 'header: {version: 5}\n_source_dir: /some/path' with patch_open(includehandler, string=string): cf = ConfigFile.load('x.yml', is_main_file=True) assert cf.src_dir == '/some/path' def test_signers_require_path_or_keyserver(self): exception = includehandler.LoadConfigException testvector = [ # signer with neither path nor gpg_keyserver ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' type: gpg\n', exception), ] self.util_exception_content(testvector) def test_signers_path_requires_repo(self): exception = includehandler.LoadConfigException testvector = [ # path without repo ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' path: key.asc\n', exception), ] self.util_exception_content(testvector) def test_signers_keyserver_requires_fingerprint(self): exception = includehandler.LoadConfigException testvector = [ # gpg_keyserver without fingerprint ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' gpg_keyserver: keyserver.ubuntu.com\n', exception), ] self.util_exception_content(testvector) def test_signers_ssh_requires_path_and_repo(self): exception = includehandler.LoadConfigException testvector = [ # ssh type with keyserver but no path ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' type: ssh\n' ' gpg_keyserver: keyserver.ubuntu.com\n' ' fingerprint: 2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9\n', exception), # ssh type with path but no repo ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' type: ssh\n' ' path: key.pub\n', exception), ] self.util_exception_content(testvector) def test_signers_valid(self): testvector = [ # path with repo ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' repo: this\n' ' path: key.asc\n'), # keyserver with fingerprint ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' gpg_keyserver: keyserver.ubuntu.com\n' ' fingerprint: 2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9\n'), # ssh type with path and repo ('header: {version: 5}\n' 'signers:\n' ' mysigner:\n' ' type: ssh\n' ' repo: this\n' ' path: key.pub\n'), ] for string in testvector: with patch_open(includehandler, string=string): ConfigFile.load('x.yml') class TestIncludes: header = ''' header: version: 5 {}''' def util_include_content(self, testvector, monkeypatch): # disable schema validation for these tests: monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {}) for test in testvector: with patch_open(includehandler, dictionary=test['fdict']): ginc = includehandler.IncludeHandler(['x.yml']) config, missing = ginc.get_config(repos=test['rdict']) # Remove header, because we dont want to compare it: config.pop('header') assert test['conf'] == config assert test['rmiss'] == missing if 'cfiles' in test: assert len(ginc.config_files) == len(test['cfiles']) for i, cf in enumerate(ginc.config_files): assert test['cfiles'][i][0] == str(cf.filename) assert test['cfiles'][i][1] == cf.is_lockfile assert test['cfiles'][i][2] == cf.is_external def test_valid_includes_none(self, monkeypatch): header = self.__class__.header testvector = [ { 'fdict': { 'x.yml': header.format('') }, 'rdict': { }, 'conf': { }, 'rmiss': [ ], 'cfiles': [ ( 'x.yml', False, False ) ] }, ] self.util_include_content(testvector, monkeypatch) def test_valid_includes_some(self, monkeypatch): header = self.__class__.header testvector = [ # Include one file from the same repo: { 'fdict': { 'x.yml': header.format(' includes: ["y.yml"]'), os.path.abspath('y.yml'): header.format('\nv:') }, 'rdict': { }, 'conf': { 'v': None }, 'rmiss': [ ], 'cfiles': [ ( os.path.abspath('y.yml'), False, False ), ( 'x.yml', False, False ) ] }, # Include one file from another not available repo: { 'fdict': { 'x.yml': header.format( ' includes: [{repo: rep, file: y.yml}]'), }, 'rdict': { }, 'conf': { }, 'rmiss': [ 'rep', ], 'cfiles': [ ( 'x.yml', False, False ) ] }, # Include one file from the same repo and one from another # not available repo: { 'fdict': { 'x.yml': header.format(' includes: ["y.yml", ' '{repo: rep, file: y.yml}]'), os.path.abspath('y.yml'): header.format('\nv:') }, 'rdict': { }, 'conf': { 'v': None }, 'rmiss': [ 'rep', ], 'cfiles': [ ( os.path.abspath('y.yml'), False, False ), ( 'x.yml', False, False ) ] }, # Include one file from another available repo: { 'fdict': { 'x.yml': header.format( ' includes: [{repo: rep, file: y.yml}]'), '/rep/y.yml': header.format('\nv:') }, 'rdict': { 'rep': '/rep' }, 'conf': { 'v': None }, 'rmiss': [ ], 'cfiles': [ ( '/rep/y.yml', False, True ), ( 'x.yml', False, False ) ] }, # Include two files from another repo in sub-directories: { 'fdict': { 'x.yml': header.format( ' includes: [{repo: rep, file: dir1/y.yml}]'), '/rep/dir1/y.yml': header.format( ' includes: ["dir2/z.yml"]'), '/rep/dir2/z.yml': header.format('\nv:') }, 'rdict': { 'rep': '/rep' }, 'conf': { 'v': None }, 'rmiss': [ ], 'cfiles': [ ( '/rep/dir2/z.yml', False, True ), ( '/rep/dir1/y.yml', False, True ), ( 'x.yml', False, False ) ] }, ] self.util_include_content(testvector, monkeypatch) def test_valid_overwriting(self, monkeypatch): header = self.__class__.header testvector = [ { 'fdict': { 'x.yml': header.format(''' includes: ["y.yml"] v: x'''), os.path.abspath('y.yml'): header.format(''' v: y''') }, 'rdict': { }, 'conf': { 'v': 'x' }, 'rmiss': [ ] }, { 'fdict': { 'x.yml': header.format(''' includes: ["y.yml"] v: {v: x}'''), os.path.abspath('y.yml'): header.format(''' v: {v: y}''') }, 'rdict': { }, 'conf': { 'v': {'v': 'x'} }, 'rmiss': [ ] }, { 'fdict': { 'x.yml': header.format(''' includes: ["y.yml"] v1: v2: [] v3: - a: c'''), os.path.abspath('y.yml'): header.format(''' v1: a v2: [a] v3: - a: b - d: c}]''') }, 'rdict': { }, 'conf': { 'v1': None, 'v2': [], 'v3': [{'a': 'c'}] }, 'rmiss': [ ] }, ] self.util_include_content(testvector, monkeypatch) def test_valid_merging(self, monkeypatch): header = self.__class__.header testvector = [ { 'fdict': { 'x.yml': header.format(''' includes: ["y.yml"] v1: x v3: a: b b: e: c: d'''), os.path.abspath('y.yml'): header.format(''' v2: y v3: d: e b: c: e: f''') }, 'rdict': { }, 'conf': { 'v1': 'x', 'v2': 'y', 'v3': { 'a': 'b', 'b': {'c': None, 'e': None}, 'c': 'd', 'd': 'e', 'e': 'f'} }, 'rmiss': [ ] }, ] self.util_include_content(testvector, monkeypatch) def test_valid_ordering(self, monkeypatch): # disable schema validation for this test: monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {}) header = self.__class__.header data = {'x.yml': header.format(''' includes: ["y.yml", "z.yml"] v: {v1: x, v2: x}'''), os.path.abspath('y.yml'): header.format(''' includes: ["z.yml"] v: {v2: y, v3: y, v5: y}'''), os.path.abspath('z.yml'): header.format(''' v: {v3: z, v4: z}''')} with patch_open(includehandler, dictionary=data): ginc = includehandler.IncludeHandler(['x.yml']) config, _ = ginc.get_config() keys = list(config['v'].keys()) index = {keys[i]: i for i in range(len(keys))} # Check for vars in z.yml: assert index['v3'] < index['v1'] assert index['v3'] < index['v2'] assert index['v3'] < index['v5'] assert index['v4'] < index['v1'] assert index['v4'] < index['v2'] assert index['v4'] < index['v5'] # Check for vars in y.yml: assert index['v2'] < index['v1'] assert index['v3'] < index['v1'] assert index['v5'] < index['v1'] def test_err_string_include_traversal(self, monkeypatch): monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {}) header = self.__class__.header fdict = { 'x.yml': header.format( ' includes: [{repo: rep, file: y.yml}]'), '/top-repo/repos/rep/y.yml': header.format( ' includes: ["../../other-repo/secret.yml"]'), } with patch_open(includehandler, dictionary=fdict): ginc = includehandler.IncludeHandler(['x.yml']) with pytest.raises(includehandler.IncludeException): ginc.get_config(repos={'rep': '/top-repo/repos/rep'}) def test_err_repo_include_traversal(self, monkeypatch): monkeypatch.setattr(includehandler, 'CONFIGSCHEMA', {}) header = self.__class__.header fdict = { 'x.yml': header.format( ' includes: [{repo: rep, ' 'file: "../../other-repo/secret.yml"}]'), } with patch_open(includehandler, dictionary=fdict): ginc = includehandler.IncludeHandler(['x.yml']) with pytest.raises(includehandler.IncludeException): ginc.get_config(repos={'rep': '/top-repo/repos/rep'}) siemens-kas-41ad961/tests/test_layers.py000066400000000000000000000067301520561422700203500ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2020 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import shutil from kas import kas import pytest LAYERBASE = '${TOPDIR}/..' class DoKas: def __init__(self, monkeykas, tmpdir): self.tmpdir = tmpdir self.monkeykas = monkeykas def run(self, filename): tdir = str(self.tmpdir / 'test_layers') shutil.copytree('tests/test_layers', tdir) self.monkeykas.chdir(tdir) self.monkeykas.setenv('KAS_CLONE_DEPTH', '1') kas.kas(['shell', filename, '-c', 'true']) # extract layer path from bblayers, keep order with open(self.monkeykas.get_kbd() / 'conf/bblayers.conf', 'r') as f: return [x.strip(' \\"\n').replace(LAYERBASE, '') for x in f.readlines() if x.lstrip().startswith(LAYERBASE)] @pytest.fixture def dokas(monkeykas, tmpdir): return DoKas(monkeykas, tmpdir) @pytest.mark.online def test_layers_default(dokas): layers = dokas.run('test.yml') assert len([l_ for l_ in layers if l_ == '/kas']) == 1 @pytest.mark.online def test_layers_include(dokas): layers = dokas.run('test.yml') assert len([l_ for l_ in layers if '/kas1/meta-' in l_]) == 2 @pytest.mark.online def test_layers_exclude(dokas): layers = dokas.run('test.yml') assert not any([l_ for l_ in layers if '/kas2' in l_]) @pytest.mark.online def test_layers_strip_dot(dokas): layers = dokas.run('test.yml') assert any([l_ for l_ in layers if l_ == '/kas3']) assert any([l_ for l_ in layers if l_ == '/kas3/meta-bar']) @pytest.mark.online def test_layers_order(dokas): layers = dokas.run('test.yml') # layers of a repo are sorted alphabetically assert layers[1] == '/kas1/meta-bar' assert layers[2] == '/kas1/meta-foo' # repos are sorted alphabetically (aa-kas from kas4 is last) assert layers[-1] == '/aa-kas/meta' @pytest.mark.online def test_layers_prio(dokas, monkeykas): layers = dokas.run('test-layer-prio.yml') # layers are sorted by global priority # highest prio (10) assert layers[0] == '/02-kas/meta-foo' # no prio, sorted alphabetically by repo name, layer name assert layers[1] == '/01-kas/aa-test' assert layers[2] == '/01-kas/zz-test' # default prio as not explicitly specified, sorted by repo name assert layers[3] == '' # lower than default prio (-10) assert layers[4] == '/01-kas' # even lower (-20) assert layers[5] == '/02-kas/meta-bar' siemens-kas-41ad961/tests/test_layers/000077500000000000000000000000001520561422700177705ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_layers/oe-init-build-env000077500000000000000000000000201520561422700231350ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_layers/test-layer-prio.yml000066400000000000000000000005171520561422700235560ustar00rootroot00000000000000header: version: 14 repos: this: 01-kas: url: https://github.com/siemens/kas.git branch: master layers: .: prio: -10 zz-test: aa-test: 02-kas: url: https://github.com/siemens/kas.git branch: master layers: meta-bar: prio: -20 meta-foo: prio: 10 siemens-kas-41ad961/tests/test_layers/test.yml000066400000000000000000000010671520561422700214760ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/siemens/kas.git branch: master kas1: url: https://github.com/siemens/kas.git branch: master layers: meta-foo: meta-bar: kas2: url: https://github.com/siemens/kas.git branch: master layers: .: excluded kas3: url: https://github.com/siemens/kas.git branch: master layers: .: meta-bar: kas4: url: https://github.com/siemens/kas.git path: aa-kas name: zz-last branch: master layers: meta: siemens-kas-41ad961/tests/test_menu.py000066400000000000000000000075001520561422700200110ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2021 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import shutil import snack import pytest from kas import kas @pytest.fixture(autouse=True) def patch_kas(monkeykas): INPUTS = iter([' ', None, ' ', None]) ACTIONS = iter([None, 'build', None, 'build']) SELECTIONS = iter([0, 3]) def mock_runOnce(unused1): return next(INPUTS) def mock_buttonPressed(unused1, unused2): return next(ACTIONS) def mock_current(unused1): return next(SELECTIONS) monkeykas.setattr(snack.GridFormHelp, 'runOnce', mock_runOnce) monkeykas.setattr(snack.ButtonBar, 'buttonPressed', mock_buttonPressed) monkeykas.setattr(snack.Listbox, 'current', mock_current) def file_contains(filename, expected): with open(filename) as file: for line in file.readlines(): if line == expected: return True return False def check_bitbake_options(expected, build_dir): with open(build_dir / 'bitbake.options') as file: return file.readline() == expected def test_menu(monkeykas, tmpdir): tdir = str(tmpdir / 'test_menu') shutil.copytree('tests/test_menu', tdir) monkeykas.chdir(tdir) kas_wd = monkeykas.get_kwd() kas_bd = monkeykas.get_kbd() local_conf = kas_bd / 'conf/local.conf' # select opt1 & build kas.kas(['menu']) assert file_contains(local_conf, 'OPT1 = "1"\n') assert file_contains(kas_wd / '.config.yaml', 'build_system: openembedded\n') assert check_bitbake_options('-c build target1\n', kas_bd) # rebuild test kas.kas(['build']) assert file_contains(local_conf, 'OPT1 = "1"\n') assert check_bitbake_options('-c build target1\n', kas_bd) # select alternative target & build kas.kas(['menu']) assert file_contains(local_conf, 'OPT1 = "1"\n') assert check_bitbake_options('-c build target2\n', kas_bd) @pytest.mark.dirsfromenv def test_menu_inc_workdir(monkeykas, tmpdir): tdir = str(tmpdir / 'test_menu_inc') kas_workdir = str(tmpdir / 'test_menu_inc' / 'out') shutil.copytree('tests/test_menu', tdir) monkeykas.chdir(tdir) os.mkdir(kas_workdir) monkeykas.setenv('KAS_WORK_DIR', kas_workdir) kas.kas(['menu']) default_config = os.path.join(kas_workdir, '.config.yaml') assert os.path.exists(default_config) # check if purge removes all files (including the generated .config.yml) kas.kas(['purge']) assert not os.path.exists(default_config) @pytest.mark.dirsfromenv def test_menu_implicit_workdir(monkeykas, tmpdir): tdir = str(tmpdir / 'test_menu_iwd') kas_workdir = str(tmpdir / 'test_menu_iwd_out') shutil.copytree('tests/test_menu', tdir) os.mkdir(kas_workdir) monkeykas.chdir(kas_workdir) kas.kas(['menu', tdir + '/Kconfig']) siemens-kas-41ad961/tests/test_menu/000077500000000000000000000000001520561422700174355ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_menu/Kconfig000066400000000000000000000006541520561422700207450ustar00rootroot00000000000000config KAS_INCLUDE_MAIN string default "test.yaml" config KAS_BUILD_SYSTEM string default "openembedded" config BOOL_OPT1 bool "ppt1" config KAS_INCLUDE_OPT1 string default "opt1.yaml" depends on BOOL_OPT1 choice prompt "choice" default CHOICE1 config CHOICE1 bool "choice1" config CHOICE2 bool "choice2" endchoice config KAS_TARGET_CHOICE string default "target1" if CHOICE1 default "target2" if CHOICE2 siemens-kas-41ad961/tests/test_menu/bitbake000077500000000000000000000000471520561422700207650ustar00rootroot00000000000000#!/bin/sh echo "$@" > bitbake.options siemens-kas-41ad961/tests/test_menu/oe-init-build-env000077500000000000000000000000441520561422700226100ustar00rootroot00000000000000#!/bin/sh export PATH=$(pwd):$PATH siemens-kas-41ad961/tests/test_menu/opt1.yaml000066400000000000000000000001041520561422700211770ustar00rootroot00000000000000header: version: 11 local_conf_header: opt1: | OPT1 = "1" siemens-kas-41ad961/tests/test_menu/test.yaml000066400000000000000000000000461520561422700213000ustar00rootroot00000000000000header: version: 11 repos: this: siemens-kas-41ad961/tests/test_patch.py000066400000000000000000000121331520561422700201420ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2019 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import stat import shutil import pytest import subprocess from kas import kas from kas.repos import PatchApplyError, PatchFileNotFound, PatchMappingError def git_get_commit(path): output = subprocess.check_output( ['git', 'rev-parse', 'HEAD'], cwd=path) return output.decode('utf-8').strip() def mercurial_get_commit(path): output = subprocess.check_output( ['hg', 'log', '-r', '.', '--template', '{node}\n'], cwd=path) return output.decode('utf-8').strip() @pytest.mark.online def test_patch(monkeykas, tmpdir, mercurial): tdir = tmpdir / 'test_patch' shutil.copytree('tests/test_patch', tdir) monkeykas.chdir(tdir) kas_wd = monkeykas.get_kwd() repo = mercurial(tmpdir, 'example') commit = repo.get_commit() subprocess.check_call( ['sed', '-i', f's/82e55d328c8c/{commit}/', 'test.yml']) kas.kas(['shell', 'test.yml', '-c', 'true']) for f in [kas_wd / 'kas/tests/test_patch/hello.sh', kas_wd / 'hello/hello.sh']: assert os.stat(f)[stat.ST_MODE] & stat.S_IXUSR kas_head_ref = git_get_commit(kas_wd / "kas") kas_branch_head_ref = git_get_commit(kas_wd / "kas-branch") hello_head_ref = mercurial_get_commit(kas_wd / "hello") hello_branch_head_ref = mercurial_get_commit(kas_wd / "hello-branch") kas.kas(['shell', 'test.yml', '-c', 'true']) assert git_get_commit(kas_wd / "kas") == kas_head_ref assert git_get_commit(kas_wd / "kas-branch") == kas_branch_head_ref assert mercurial_get_commit(kas_wd / "hello") == hello_head_ref assert mercurial_get_commit(kas_wd / "hello-branch") == \ hello_branch_head_ref @pytest.mark.online def test_patch_update(monkeykas, tmpdir, mercurial): """ Test that patches are applied correctly after switching a repo from a branch to a commit hash and vice-versa with both git and mercurial repositories. """ tdir = str(tmpdir / 'test_patch_update') shutil.copytree('tests/test_patch', tdir) monkeykas.chdir(tdir) kas_wd = monkeykas.get_kwd() repo = mercurial(tmpdir, 'example') commit = repo.get_commit() for file, c in [('test.yml', '82e55d328c8c'), ('test2.yml', '0a04b987be5a')]: subprocess.check_call( ['sed', '-i', f's/{c}/{commit}/', file]) kas.kas(['shell', 'test.yml', '-c', 'true']) kas.kas(['shell', 'test2.yml', '-c', 'true']) for f in [kas_wd / 'kas/tests/test_patch/hello.sh', kas_wd / 'hello/hello.sh']: assert os.stat(f)[stat.ST_MODE] & stat.S_IXUSR @pytest.mark.online def test_invalid_patch(monkeykas, tmpdir): """ Test on common errors when applying patches """ tdir = str(tmpdir / 'test_patch_invalid') shutil.copytree('tests/test_patch', tdir) monkeykas.chdir(tdir) with pytest.raises(PatchFileNotFound): kas.kas(['shell', 'test-invalid.yml', '-c', 'true']) with pytest.raises(PatchMappingError): kas.kas(['shell', 'test-invalid2.yml', '-c', 'true']) with pytest.raises(PatchApplyError): kas.kas(['shell', 'test-invalid3.yml', '-c', 'true']) @pytest.mark.online def test_patch_dirty_repo(monkeykas, tmpdir, mercurial): """ Test that kas will not apply patches to a dirty repository """ tdir = str(tmpdir / 'test_patch_dirty') shutil.copytree('tests/test_patch', tdir) monkeykas.chdir(tdir) repo = mercurial(tmpdir, 'example') commit = repo.get_commit() subprocess.check_call( ['sed', '-i', f's/82e55d328c8c/{commit}/', 'test.yml']) kas.kas(['checkout', '--skip', 'repos_apply_patches', 'test.yml']) kas_branch_dir = monkeykas.get_kwd() / 'kas-branch' with open(kas_branch_dir / 'README.rst', 'a') as f: f.write('echo "dirty"') kas.kas(['checkout', 'test.yml']) # the dirty repo must not be patched assert not (kas_branch_dir / 'tests/test_patch/hello.sh').exists() # the clean repo must be patched assert (monkeykas.get_kwd() / 'hello-branch/hello.sh').exists() siemens-kas-41ad961/tests/test_patch/000077500000000000000000000000001520561422700175705ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/oe-init-build-env000077500000000000000000000000201520561422700227350ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_patch/patches/000077500000000000000000000000001520561422700212175ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/hello-branch/000077500000000000000000000000001520561422700235555ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/hello-branch/003.patch000077700000000000000000000000001520561422700310722../hello/quilt/003.patchustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/hello/000077500000000000000000000000001520561422700223225ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/hello/001.patch000066400000000000000000000001631520561422700236430ustar00rootroot00000000000000diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,1 +1,4 @@ all: hello + +clean: + rm -f hello siemens-kas-41ad961/tests/test_patch/patches/hello/quilt/000077500000000000000000000000001520561422700234605ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/hello/quilt/001.patch000066400000000000000000000001631520561422700250010ustar00rootroot00000000000000diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,4 +1,1 @@ all: hello - -clean: - rm -f hello siemens-kas-41ad961/tests/test_patch/patches/hello/quilt/002.patch000066400000000000000000000001631520561422700250020ustar00rootroot00000000000000diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,1 +1,4 @@ all: hello + +clean: + rm -f hello siemens-kas-41ad961/tests/test_patch/patches/hello/quilt/003.patch000066400000000000000000000005341520561422700250050ustar00rootroot00000000000000# HG changeset patch # User kas # Date 1568977454 -7200 # Fri Sep 20 13:04:14 2019 +0200 # Node ID b8fc0927f880cc0ef6b79ba6ee470f7504d68466 # Parent 66ff34c19e817cf518ebbd53bef4a992d669e68e msg diff --git a/hello.sh b/hello.sh new file mode 100755 --- /dev/null +++ b/hello.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo Hello World! siemens-kas-41ad961/tests/test_patch/patches/hello/quilt/series000066400000000000000000000002221520561422700246710ustar00rootroot00000000000000# test comment # this is a revert of the other 001 we applied without quilt 001.patch # first patch 002.patch 003.patch # introduce an executable siemens-kas-41ad961/tests/test_patch/patches/kas-branch/000077500000000000000000000000001520561422700232305ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/kas-branch/003.patch000077700000000000000000000000001520561422700302202../kas/quilt/003.patchustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/kas/000077500000000000000000000000001520561422700217755ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/kas/001.patch000066400000000000000000000011641520561422700233200ustar00rootroot00000000000000diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b594e6..f1222ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,3 @@ -1.0 -- isar: Take qemu-user-static from buster and adjust binfmt setup - 0.20.1 - kas-docker: Restore KAS_PREMIRRORS support diff --git a/kas/__version__.py b/kas/__version__.py index 0ac2731..020cb6f 100644 --- a/kas/__version__.py +++ b/kas/__version__.py @@ -25,7 +25,7 @@ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' -__version__ = '1.0' +__version__ = '0.20.1' # Please update docs/format-changelog.rst when changing the file version. __file_version__ = 8 siemens-kas-41ad961/tests/test_patch/patches/kas/quilt/000077500000000000000000000000001520561422700231335ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_patch/patches/kas/quilt/001.patch000066400000000000000000000011641520561422700244560ustar00rootroot00000000000000diff --git a/CHANGELOG.md b/CHANGELOG.md index f1222ab..4b594e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +1.0 +- isar: Take qemu-user-static from buster and adjust binfmt setup + 0.20.1 - kas-docker: Restore KAS_PREMIRRORS support diff --git a/kas/__version__.py b/kas/__version__.py index 020cb6f..0ac2731 100644 --- a/kas/__version__.py +++ b/kas/__version__.py @@ -25,7 +25,7 @@ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' -__version__ = '0.20.1' +__version__ = '1.0' # Please update docs/format-changelog.rst when changing the file version. __file_version__ = 8 siemens-kas-41ad961/tests/test_patch/patches/kas/quilt/002.patch000066400000000000000000000011641520561422700244570ustar00rootroot00000000000000diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b594e6..f1222ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,3 @@ -1.0 -- isar: Take qemu-user-static from buster and adjust binfmt setup - 0.20.1 - kas-docker: Restore KAS_PREMIRRORS support diff --git a/kas/__version__.py b/kas/__version__.py index 0ac2731..020cb6f 100644 --- a/kas/__version__.py +++ b/kas/__version__.py @@ -25,7 +25,7 @@ __license__ = 'MIT' __copyright__ = 'Copyright (c) Siemens AG, 2017-2018' -__version__ = '1.0' +__version__ = '0.20.1' # Please update docs/format-changelog.rst when changing the file version. __file_version__ = 8 siemens-kas-41ad961/tests/test_patch/patches/kas/quilt/003.patch000066400000000000000000000007551520561422700244650ustar00rootroot00000000000000From b401ea2143d6e214a74f100c1d86b57ee3aaed26 Mon Sep 17 00:00:00 2001 From: kas Date: Fri, 20 Sep 2019 10:33:07 +0200 Subject: [PATCH] test --- tests/test_patch/hello.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 tests/test_patch/hello.sh diff --git a/tests/test_patch/hello.sh b/tests/test_patch/hello.sh new file mode 100755 index 0000000..c100654 --- /dev/null +++ b/tests/test_patch/hello.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo Hello World! -- 2.21.0 siemens-kas-41ad961/tests/test_patch/patches/kas/quilt/series000066400000000000000000000002221520561422700243440ustar00rootroot00000000000000# test comment # this is a revert of the other 001 we applied without quilt 001.patch # first patch 002.patch 003.patch # introduce an executable siemens-kas-41ad961/tests/test_patch/test-invalid.yml000066400000000000000000000003551520561422700227210ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 patches: plain: repo: this path: patches/kas/001-non-existent.patch siemens-kas-41ad961/tests/test_patch/test-invalid2.yml000066400000000000000000000003371520561422700230030ustar00rootroot00000000000000header: version: 14 repos: kas: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 patches: plain: repo: non-existent path: patches/kas/001.patch siemens-kas-41ad961/tests/test_patch/test-invalid3.yml000066400000000000000000000003401520561422700227760ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/ilbers/isar.git commit: 47aaeedecd0ea6f754da36be1d10717b04eb8275 patches: plain: repo: this path: patches/kas/001.patch siemens-kas-41ad961/tests/test_patch/test.yml000066400000000000000000000015011520561422700212670ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 patches: plain: repo: this path: patches/kas/001.patch quilt: repo: this path: patches/kas/quilt/ kas-branch: url: https://github.com/siemens/kas.git branch: master patches: plain: repo: this path: patches/kas-branch/003.patch hello: url: ../example commit: 82e55d328c8c type: hg patches: plain: repo: this path: patches/hello/001.patch quilt: repo: this path: patches/hello/quilt/ hello-branch: url: ../example branch: default type: hg patches: plain: repo: this path: patches/hello-branch/003.patch siemens-kas-41ad961/tests/test_patch/test2.yml000066400000000000000000000015011520561422700213510ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/siemens/kas.git branch: master patches: plain: repo: this path: patches/kas-branch/003.patch kas-branch: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 patches: plain: repo: this path: patches/kas/001.patch quilt: repo: this path: patches/kas/quilt/ hello: url: ../example branch: default type: hg patches: plain: repo: this path: patches/hello/001.patch quilt: repo: this path: patches/hello/quilt/ hello-branch: url: ../example commit: 0a04b987be5a type: hg patches: plain: repo: this path: patches/hello-branch/003.patch siemens-kas-41ad961/tests/test_refspec.py000066400000000000000000000273531520561422700205040ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2019 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import pytest import shutil import yaml import subprocess import os from pathlib import Path from kas import kas from kas.repos import RepoRefError, Repo from kas.kasusererror import CommandExecError def run_cmd(cmd, cwd=None, fail=True): """ Run a command and return the return code and output. Replacement for kas internal run_cmd function which cannot be used outside of kas as there is no event loop. """ kas_wd = Path(os.environ.get('KAS_WORK_DIR', '.')) _cwd = Path(cwd or '.') if not _cwd.is_absolute(): _cwd = kas_wd / _cwd try: output = subprocess.check_output(cmd, cwd=_cwd) return (0, output.decode('utf-8')) except subprocess.CalledProcessError as e: if fail: raise e return (e.returncode, e.output.decode('utf-8')) @pytest.mark.online def test_refspec_switch(monkeykas, tmpdir): """ Test that the local git clone is correctly updated when switching between a commit hash refspec and a branch refspec. """ tdir = str(tmpdir / 'test_refspec_switch') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) kas.kas(['shell', 'test.yml', '-c', 'true']) (rc, output) = run_cmd(['git', 'symbolic-ref', '-q', 'HEAD'], cwd='kas', fail=False) assert rc != 0 assert output.strip() == '' (rc, output) = run_cmd(['git', 'rev-parse', 'HEAD'], cwd='kas', fail=False) assert rc == 0 assert output.strip() == '907816a5c4094b59a36aec12226e71c461c05b77' (rc, output) = run_cmd(['git', 'symbolic-ref', '-q', 'HEAD'], cwd='kas2', fail=False) assert rc == 0 assert output.strip() == 'refs/heads/master' (rc, output) = run_cmd(['git', 'tag', '--points-at', 'HEAD'], cwd='kas3', fail=False) assert rc == 0 assert output.strip() == '3.0.1' kas.kas(['shell', 'test2.yml', '-c', 'true']) (rc, output) = run_cmd(['git', 'symbolic-ref', '-q', 'HEAD'], cwd='kas', fail=False) assert rc == 0 assert output.strip() == 'refs/heads/master' (rc, output) = run_cmd(['git', 'symbolic-ref', '-q', 'HEAD'], cwd='kas2', fail=False) assert rc != 0 assert output.strip() == '' (rc, output) = run_cmd(['git', 'rev-parse', 'HEAD'], cwd='kas2', fail=False) assert rc == 0 assert output.strip() == '907816a5c4094b59a36aec12226e71c461c05b77' (rc, output) = run_cmd(['git', 'symbolic-ref', '-q', 'HEAD'], cwd='kas3', fail=False) assert rc == 0 assert output.strip() == 'refs/heads/master' (rc, output) = run_cmd(['git', 'tag', '--points-at', 'HEAD'], cwd='kas4', fail=False) assert rc == 0 assert output.strip() == '2.6.3' @pytest.mark.online def test_refspec_absolute(monkeykas, tmpdir): """ Test that the local git clone works when a absolute refspec is given. """ tdir = str(tmpdir / 'test_refspec_absolute') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) kas.kas(['shell', 'test3.yml', '-c', 'true']) (rc, output) = run_cmd(['git', 'symbolic-ref', '-q', 'HEAD'], cwd='kas_abs', fail=False) assert rc == 0 assert output.strip() == 'refs/heads/master' (rc, output_kas_abs) = run_cmd(['git', 'rev-parse', 'HEAD'], cwd='kas_abs', fail=False) assert rc == 0 (rc, output_kas_rel) = run_cmd(['git', 'rev-parse', 'HEAD'], cwd='kas_rel', fail=False) assert rc == 0 assert output_kas_abs.strip() == output_kas_rel.strip() (rc, output) = run_cmd(['git', 'tag', '--points-at', 'HEAD'], cwd='kas_tag_abs', fail=False) assert rc == 0 assert output.strip() == '3.0.1' def test_url_no_refspec(monkeykas, tmpdir): """ Test that a repository with url but no refspec raises an error. """ tdir = str(tmpdir / 'test_url_no_refspec') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) with pytest.raises(RepoRefError): kas.kas(['shell', 'test4.yml', '-c', 'true']) def test_commit_refspec_mix(monkeykas, tmpdir): """ Test that mixing legacy refspec with commit/branch raises errors. """ tdir = str(tmpdir / 'test_commit_refspec_mix') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) with pytest.raises(RepoRefError): kas.kas(['shell', 'test5.yml', '-c', 'true']) with pytest.raises(RepoRefError): kas.kas(['shell', 'test6.yml', '-c', 'true']) with pytest.raises(RepoRefError): kas.kas(['shell', 'test7.yml', '-c', 'true']) @pytest.mark.online def test_tag_commit_do_not_match(monkeykas, tmpdir): """ Test that giving tag and commit that do not match raises an error. """ tdir = str(tmpdir / 'test_tag_commit_do_not_match') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) with pytest.raises(RepoRefError): kas.kas(['shell', 'test8.yml', '-c', 'true']) @pytest.mark.online def test_unsafe_tag_warning(capsys, monkeykas, tmpdir): """ Test that using tag without commit issues a warning, but only once. """ tdir = str(tmpdir / 'test_unsafe_tag_warning') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) # needs to be reset in case other tests ran before Repo.__no_commit_warned__ = [] kas.kas(['shell', 'test2.yml', '-c', 'true']) assert capsys.readouterr().err.count( 'Using tag without commit for repository "kas4" is unsafe as tags ' 'are mutable.') == 1 @pytest.mark.online def test_unsafe_branch_warning(capsys, monkeykas, tmpdir): """ Test that using branch without commit issues a warning, but only once. """ tdir = str(tmpdir / 'test_unsafe_branch_warning') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) # needs to be reset in case other tests ran before Repo.__no_commit_warned__ = [] kas.kas(['shell', 'test2.yml', '-c', 'true']) assert capsys.readouterr().err.count( 'Using branch without commit for repository "kas3" is unsafe. Either ' 'add a commit or use a lock file.') == 1 @pytest.mark.online def test_tag_branch_same_name(capsys, monkeykas, tmpdir): """ Test that kas uses the tag if a branch has the same name as the tag. """ tdir = str(tmpdir / 'test_tag_branch_same_name') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) # Checkout the repositories kas.kas(['shell', 'test.yml', '-c', 'true']) # In kas3: create a branch named "3.0.1" on master HEAD # A tag named "3.0.1" already exists on an old commit from 2022 (rc, output) = run_cmd(['git', 'switch', 'master'], cwd='kas3', fail=False) assert rc == 0 (rc, output) = run_cmd(['git', 'branch', '3.0.1'], cwd='kas3', fail=False) assert rc == 0 # In kas4: create a tag named "master" on existing 2.6.3 tag (rc, output) = run_cmd(['git', 'checkout', '2.6.3'], cwd='kas4', fail=False) assert rc == 0 (rc, output) = run_cmd(['git', 'tag', 'master'], cwd='kas4', fail=False) assert rc == 0 # Checkout the repositories again kas.kas(['shell', 'test.yml', '-c', 'true']) # Check the commit hashes (rc, output) = run_cmd(['git', 'rev-parse', 'HEAD'], cwd='kas3', fail=False) assert rc == 0 assert output.strip() == '229310958b17dc2b505b789c1cc1d0e2fddccc44' (rc, output) = run_cmd(['git', 'rev-parse', 'HEAD'], cwd='kas4', fail=False) assert rc == 0 (rc, output2) = run_cmd(['git', 'rev-parse', 'refs/heads/master'], cwd='kas4', fail=False) assert rc == 0 assert output.strip() == output2.strip() @pytest.mark.online def test_refspec_warning(capsys, monkeykas, tmpdir): """ Test that using legacy refspec issues a warning, but only once. """ tdir = str(tmpdir / 'test_refspec_warning') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) # needs to be reset in case other tests ran before Repo.__legacy_refspec_warned__ = [] kas.kas(['shell', 'test2.yml', '-c', 'true']) assert capsys.readouterr().err.count( 'Using deprecated refspec for repository "kas2".') == 1 @pytest.mark.online def test_branch_and_tag(monkeykas, tmpdir, mercurial): """ Test if error is raised when branch and tag are set. """ tdir = str(tmpdir / 'test_branch_and_tag') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) with mercurial(tmpdir, 'evolve', branch='stable'): with pytest.raises(RepoRefError): kas.kas(['checkout', 'test9.yml']) with pytest.raises(RepoRefError): kas.kas(['checkout', 'test10.yml']) with pytest.raises(RepoRefError): kas.kas(['checkout', 'test11.yml']) @pytest.mark.online def test_commit_expand(monkeykas, tmpdir, capsys): """ Test if an abbreviated commit hash is expanded to the full hash. """ tdir = str(tmpdir / 'test_commit_expand') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) kas.kas(['dump', '--resolve-refs', 'test12.yml']) rawspec = yaml.safe_load(capsys.readouterr().out) assert rawspec['repos']['kas']['commit'] == \ 'abd109469d17b7ff4d958b5aa5ab5f5511cc4d43' @pytest.mark.online def test_sha_like_branch_tag(monkeykas, tmpdir, capsys): """ Test if branches or tags that look like shas are rejected. """ tdir = str(tmpdir / 'test_sha_like_branch_tag') shutil.copytree('tests/test_refspec', tdir) monkeykas.chdir(tdir) # Checkout the repositories kas.kas(['checkout', 'test13.yml']) def test_non_commit(variant, commit): (rc, output) = run_cmd(['git', variant, commit], cwd='kas', fail=False) assert rc == 0 with open(f'{tdir}/sha-test.yml', 'w') as f: f.write('header:\n') f.write(' version: 15\n') f.write('repos:\n') f.write(' this:\n') f.write(' kas-2:\n') f.write(f' url: file://{tdir}/kas\n') f.write(f' commit: {commit}\n') with pytest.raises(CommandExecError): kas.kas(['checkout', 'sha-test.yml']) test_non_commit('branch', '123456789abcdef0123456789abcdef012345678') test_non_commit('tag', '123456789abcdef0123456789abcdef012345679') siemens-kas-41ad961/tests/test_refspec/000077500000000000000000000000001520561422700201205ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_refspec/oe-init-build-env000077500000000000000000000000201520561422700232650ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_refspec/test.yml000066400000000000000000000006301520561422700216210ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 kas2: url: https://github.com/siemens/kas.git branch: master kas3: url: https://github.com/siemens/kas.git tag: 3.0.1 commit: 229310958b17dc2b505b789c1cc1d0e2fddccc44 kas4: url: https://github.com/siemens/kas.git branch: master siemens-kas-41ad961/tests/test_refspec/test10.yml000066400000000000000000000001741520561422700217650ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git tag: 3.0.1 branch: master siemens-kas-41ad961/tests/test_refspec/test11.yml000066400000000000000000000003051520561422700217620ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git # this commit is valid, but not a branch. Reject! branch: abd109469d17b7ff4d958b5aa5ab5f5511cc4d43 siemens-kas-41ad961/tests/test_refspec/test12.yml000066400000000000000000000001601520561422700217620ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git commit: abd109469 siemens-kas-41ad961/tests/test_refspec/test13.yml000066400000000000000000000002171520561422700217660ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git commit: 907816a5c4094b59a36aec12226e71c461c05b77 siemens-kas-41ad961/tests/test_refspec/test2.yml000066400000000000000000000006301520561422700217030ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git branch: master kas2: url: https://github.com/siemens/kas.git # keep legacy refspec here for testing purposes refspec: 907816a5c4094b59a36aec12226e71c461c05b77 kas3: url: https://github.com/siemens/kas.git branch: master kas4: url: https://github.com/siemens/kas.git tag: 2.6.3 siemens-kas-41ad961/tests/test_refspec/test3.yml000066400000000000000000000004341520561422700217060ustar00rootroot00000000000000header: version: 15 repos: this: kas_abs: url: https://github.com/siemens/kas.git branch: refs/heads/master kas_rel: url: https://github.com/siemens/kas.git branch: master kas_tag_abs: url: https://github.com/siemens/kas.git tag: refs/tags/3.0.1 siemens-kas-41ad961/tests/test_refspec/test4.yml000066400000000000000000000001311520561422700217010ustar00rootroot00000000000000header: version: 8 repos: this: kas: url: https://github.com/siemens/kas.git siemens-kas-41ad961/tests/test_refspec/test5.yml000066400000000000000000000002431520561422700217060ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/siemens/kas.git commit: dc44638cd87c4d0045ea2ca441e682f3525d8b91 refspec: master siemens-kas-41ad961/tests/test_refspec/test6.yml000066400000000000000000000002431520561422700217070ustar00rootroot00000000000000header: version: 14 repos: this: kas: url: https://github.com/siemens/kas.git refspec: dc44638cd87c4d0045ea2ca441e682f3525d8b91 branch: master siemens-kas-41ad961/tests/test_refspec/test7.yml000066400000000000000000000002371520561422700217130ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git refspec: dc44638cd87c4d0045ea2ca441e682f3525d8b91 tag: 3.0.1 siemens-kas-41ad961/tests/test_refspec/test8.yml000066400000000000000000000001761520561422700217160ustar00rootroot00000000000000header: version: 15 repos: this: kas: url: https://github.com/siemens/kas.git tag: 3.0.1 commit: de4dcafe siemens-kas-41ad961/tests/test_refspec/test9.yml000066400000000000000000000002041520561422700217070ustar00rootroot00000000000000header: version: 15 repos: this: evolve: url: ../evolve type: hg branch: stable commit: '6634:991cbf0f66f2' siemens-kas-41ad961/tests/test_repo_includes/000077500000000000000000000000001520561422700213245ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_repo_includes/oe-init-build-env000077500000000000000000000000201520561422700244710ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_repo_includes/subrepo/000077500000000000000000000000001520561422700230035ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_repo_includes/subrepo/test.yml000066400000000000000000000002751520561422700245110ustar00rootroot00000000000000header: version: 11 includes: - repo: externalrepo file: tests/test_layers/test.yml repos: externalrepo: url: https://github.com/siemens/kas.git path: externalrepo siemens-kas-41ad961/tests/test_repo_includes/test.yml000066400000000000000000000003441520561422700230270ustar00rootroot00000000000000header: version: 11 includes: - repo: subrepo file: test.yml env: TESTVAR_FOO: "BAR" repos: this: subrepo: path: subrepo name: sub-repo externalrepo: refspec: master name: external-repo siemens-kas-41ad961/tests/test_signature_verify.py000066400000000000000000000123761520561422700224410ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens, 2025 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import pytest import shutil import yaml import gnupg from pathlib import Path from subprocess import check_output from kas import kas from kas.repos import RepoRefError from kas.kasusererror import KasUserError from kas.keyhandler import KeyImportError @pytest.mark.online def test_signed_gpg_invalid(monkeykas, tmpdir): tdir = tmpdir / 'test_signature_verify' shutil.copytree('tests/test_signature_verify', tdir) monkeykas.chdir(tdir) with pytest.raises(RepoRefError): kas.kas(['checkout', 'test-gpg-wrong-key.yml']) @pytest.mark.online def test_signed_gpg_remove_key(monkeykas, tmpdir): tdir = tmpdir / 'test_signature_verify' shutil.copytree('tests/test_signature_verify', tdir) monkeykas.chdir(tdir) # successfully validate signature kas.kas(['checkout', 'test-gpg-key-retention.yml']) # change key in config and try again with open('test-gpg-key-retention.yml', 'r') as f: data = yaml.safe_load(f) # use OEs signing key data['signers']['jan-kiszka']['fingerprint'] = \ '2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9' with open('test-gpg-key-retention.yml', 'w') as f: yaml.dump(data, f) with pytest.raises(RepoRefError): kas.kas(['checkout', 'test-gpg-key-retention.yml']) # remove key from config and try again (without deleting the keystore) with open('test-gpg-key-retention.yml', 'r') as f: data = yaml.safe_load(f) del data['signers'] with open('test-gpg-key-retention.yml', 'w') as f: yaml.dump(data, f) with pytest.raises(KasUserError): kas.kas(['checkout', 'test-gpg-key-retention.yml']) @pytest.mark.dirsfromenv @pytest.mark.online def test_signed_gpg_local_key(monkeykas, tmpdir): tdir = tmpdir / 'test_signature_verify' shutil.copytree('tests/test_signature_verify', tdir) monkeykas.chdir(tdir) # we don't want to store keys in this repo, hence download a key, # export it and pass to kas KEY_FINGERPRINT = 'CA5F8C00F5FBC85466016C808AD4AC6F7AE5E714' gnupghome = tdir / 'test_gnupg' gnupghome.mkdir() gnupghome.chmod(0o700) gpg = gnupg.GPG(gnupghome=str(gnupghome)) gpg.recv_keys('keyserver.ubuntu.com', KEY_FINGERPRINT) gpg.export_keys(KEY_FINGERPRINT, armor=True, output=str(tdir / 'jan-kiszka.asc')) with pytest.raises(KeyImportError): kas.kas(['checkout', 'test-gpg-local-key.yml']) # remove key definition with wrong fingerprint with open('test-gpg-local-key.yml', 'r') as f: data = yaml.safe_load(f) del data['signers']['wrong-fp'] with open('test-gpg-local-key.yml', 'w') as f: yaml.dump(data, f) kas.kas(['checkout', 'test-gpg-local-key.yml']) @pytest.mark.dirsfromenv def test_signed_ssh_key(monkeykas, tmpdir): tdir = tmpdir / 'test_signature_verify' shutil.copytree('tests/test_signature_verify', tdir) monkeykas.chdir(tdir) kas_wd = monkeykas.get_kwd() repodir = Path(kas_wd / 'testrepo') repodir.mkdir() # create a new ssh key sshdir = Path(tdir / 'ssh') sshdir.mkdir() sshdir.chmod(0o700) monkeykas.chdir(repodir) check_output(['ssh-keygen', '-t', 'rsa', '-N', '', '-C', 'Comment Space', '-f', str(sshdir / 'testkey')]) # create a git repository with a single commit check_output(['git', 'init']) check_output(['git', 'config', 'user.name', 'kas']) check_output(['git', 'config', 'user.email', 'kas@example.com']) check_output(['git', 'config', 'gpg.format', 'ssh']) check_output(['git', 'config', 'user.signingkey', str(sshdir / 'testkey')]) check_output(['git', 'commit', '-S', '-m', 'initial commit', '--allow-empty']) check_output(['git', 'tag', '-s', 'signed-tag', '-m', 'signed tag']) monkeykas.chdir(tdir) kas.kas(['checkout', 'test-ssh-key.yml']) # now replace the ssh key and check if signature verification fails (sshdir / 'testkey').unlink() (sshdir / 'testkey.pub').unlink() check_output(['ssh-keygen', '-t', 'rsa', '-N', '', '-f', str(sshdir / 'testkey')]) with pytest.raises(RepoRefError): kas.kas(['checkout', 'test-ssh-key.yml']) siemens-kas-41ad961/tests/test_signature_verify/000077500000000000000000000000001520561422700220565ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_signature_verify/oe-init-build-env000077500000000000000000000000201520561422700252230ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_signature_verify/test-gpg-key-retention.yml000066400000000000000000000006021520561422700271240ustar00rootroot00000000000000header: version: 19 signers: jan-kiszka: fingerprint: CA5F8C00F5FBC85466016C808AD4AC6F7AE5E714 gpg_keyserver: keyserver.ubuntu.com repos: this: kas: url: https://github.com/siemens/kas.git tag: '4.3' commit: f650ebe2495a9cbe2fdf4a2c8becc7b3db470d55 signed: true # use the wrong key to test the error message allowed_signers: - jan-kiszka siemens-kas-41ad961/tests/test_signature_verify/test-gpg-local-key.yml000066400000000000000000000006041520561422700262110ustar00rootroot00000000000000header: version: 19 signers: jan-kiszka: repo: this path: jan-kiszka.asc wrong-fp: repo: this path: jan-kiszka.asc fingerprint: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA repos: this: kas: url: https://github.com/siemens/kas.git tag: '4.3' commit: f650ebe2495a9cbe2fdf4a2c8becc7b3db470d55 signed: true allowed_signers: - jan-kiszka siemens-kas-41ad961/tests/test_signature_verify/test-gpg-wrong-key.yml000066400000000000000000000006261520561422700262570ustar00rootroot00000000000000header: version: 19 signers: YoctoBuildandRelease: fingerprint: 2AFB13F28FBBB0D1B9DAF63087EB3D32FB631AD9 gpg_keyserver: keyserver.ubuntu.com repos: this: kas: url: https://github.com/siemens/kas.git tag: '4.3' commit: f650ebe2495a9cbe2fdf4a2c8becc7b3db470d55 signed: true # use the wrong key to test the error message allowed_signers: - YoctoBuildandRelease siemens-kas-41ad961/tests/test_signature_verify/test-ssh-key.yml000066400000000000000000000003301520561422700251350ustar00rootroot00000000000000header: version: 19 signers: testkey: type: ssh repo: this path: ssh/testkey.pub repos: this: local: path: testrepo tag: signed-tag signed: true allowed_signers: - testkey siemens-kas-41ad961/tests/test_transitive_includes.py000066400000000000000000000111531520561422700231220ustar00rootroot00000000000000# kas - setup tool for bitbake based projects # # Copyright (c) Siemens AG, 2024 # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os from pathlib import Path import shutil import subprocess import yaml import pytest from kas import kas from kas.includehandler import IncludeException def test_transitive_includes(monkeykas, tmpdir, capsys): """ Check if a reference to a repo that itself is specified in a remote repo is correctly resolved. """ repos = ['main', 'foo', 'bar'] dirs = {} for r in repos: dirs[r] = Path(tmpdir / r) os.mkdir(dirs[r]) # create main repo config with open('tests/test_transitive_includes/main.yml', 'rb') as fds: config_top = yaml.safe_load(fds) config_top['repos']['foo']['url'] = str(dirs['foo']) with open(dirs['main'] / 'main.yml', 'w') as config_file: yaml.dump(config_top, config_file) shutil.copy('tests/test_transitive_includes/oe-init-build-env', dirs['main']) # create foo repo and config with open('tests/test_transitive_includes/foo.yml', 'rb') as fds: config_foo = yaml.safe_load(fds) config_foo['repos']['bar']['url'] = str(dirs['bar']) with open(dirs['foo'] / 'foo.yml', 'w') as config_file: yaml.dump(config_foo, config_file) subprocess.check_call(['git', 'init'], cwd=dirs['foo']) subprocess.check_call(['git', 'config', 'user.email', 'test'], cwd=dirs['foo']) subprocess.check_call(['git', 'config', 'user.name', 'test'], cwd=dirs['foo']) subprocess.check_call(['git', 'checkout', '-b', 'kas'], cwd=dirs['foo']) subprocess.check_call(['git', 'add', 'foo.yml'], cwd=dirs['foo']) subprocess.check_call(['git', 'commit', '-m', 'init'], cwd=dirs['foo']) # create bar repo shutil.copy('tests/test_transitive_includes/bar.yml', dirs['bar']) subprocess.check_call(['git', 'init'], cwd=dirs['bar']) subprocess.check_call(['git', 'config', 'user.email', 'test'], cwd=dirs['bar']) subprocess.check_call(['git', 'config', 'user.name', 'test'], cwd=dirs['bar']) subprocess.check_call(['git', 'checkout', '-b', 'kas'], cwd=dirs['bar']) subprocess.check_call(['git', 'add', 'bar.yml'], cwd=dirs['bar']) subprocess.check_call(['git', 'commit', '-m', 'init'], cwd=dirs['bar']) monkeykas.chdir(dirs['main']) kas.kas(['dump', 'main.yml']) config_final = yaml.safe_load(capsys.readouterr().out) assert config_final['env']['KAS_REPO_BAR'] == "1" @pytest.mark.dirsfromenv def test_include_from_submodule(monkeykas, tmpdir): tmpdir.mkdir('main') tmpdir.mkdir('sub') for s, t in [('sub_repo.yml', 'sub'), ('super_repo.yml', 'main'), ('oe-init-build-env', 'main')]: shutil.copy(f'tests/test_transitive_includes/{s}', tmpdir / t) monkeykas.chdir(tmpdir / 'sub') subprocess.check_call(['git', 'init', '-b', 'main']) subprocess.check_call(['git', 'add', 'sub_repo.yml']) subprocess.check_call(['git', 'commit', '-m', 'init']) monkeykas.chdir(tmpdir / 'main') subprocess.check_call(['git', 'init', '-b', 'main']) # clone a repo as non-submodule subprocess.check_call(['git', 'clone', '../sub', 'sub']) with pytest.raises(IncludeException): kas.kas(['checkout', 'super_repo.yml:sub/sub_repo.yml']) # now add the repo as submodule shutil.rmtree('sub') subprocess.check_call(['git', '-c', 'protocol.file.allow=always', 'submodule', 'add', '../sub', 'sub']) kas.kas(['-l', 'debug', 'checkout', 'super_repo.yml:sub/sub_repo.yml']) siemens-kas-41ad961/tests/test_transitive_includes/000077500000000000000000000000001520561422700225475ustar00rootroot00000000000000siemens-kas-41ad961/tests/test_transitive_includes/bar.yml000066400000000000000000000000601520561422700240320ustar00rootroot00000000000000header: version: 14 env: KAS_REPO_BAR: "1" siemens-kas-41ad961/tests/test_transitive_includes/foo.yml000066400000000000000000000001251520561422700240530ustar00rootroot00000000000000header: version: 14 repos: bar: url: <> branch: kas siemens-kas-41ad961/tests/test_transitive_includes/main.yml000066400000000000000000000002611520561422700242150ustar00rootroot00000000000000header: version: 14 includes: - repo: foo file: foo.yml - repo: bar file: bar.yml repos: this: foo: url: <> branch: kas siemens-kas-41ad961/tests/test_transitive_includes/oe-init-build-env000077500000000000000000000000201520561422700257140ustar00rootroot00000000000000#!/bin/sh true siemens-kas-41ad961/tests/test_transitive_includes/sub_repo.yml000066400000000000000000000000261520561422700251060ustar00rootroot00000000000000header: version: 14 siemens-kas-41ad961/tests/test_transitive_includes/super_repo.yml000066400000000000000000000000461520561422700254550ustar00rootroot00000000000000header: version: 14 repos: this: