pax_global_header00006660000000000000000000000064144716452070014524gustar00rootroot0000000000000052 comment=75101fb857583b54649943f7d43a0669571cb3ba firebuild-0.8.2/000077500000000000000000000000001447164520700135005ustar00rootroot00000000000000firebuild-0.8.2/.github/000077500000000000000000000000001447164520700150405ustar00rootroot00000000000000firebuild-0.8.2/.github/workflows/000077500000000000000000000000001447164520700170755ustar00rootroot00000000000000firebuild-0.8.2/.github/workflows/build.yml000066400000000000000000000277201447164520700207270ustar00rootroot00000000000000name: CI on: push: branches: - master pull_request: branches: - master env: BUILD_DEPS: cmake libconfig++-dev libxxhash-dev libjemalloc-dev libtsl-hopscotch-map-dev pkg-config python3-jinja2 TEST_DEPS: bc bats clang node-d3 graphviz moreutils fakeroot jobs: build-on-ubuntu-lts: needs: style-checks runs-on: ubuntu-22.04 timeout-minutes: 15 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 with: path: ~/.cache/debs key: build-on-ubuntu-lts-debs-${{ needs.style-checks.outputs.week }} # TODO: This and saving the debs can be dropped and and the debs can be saved directly like in docker # when https://github.com/actions/cache/issues/324 gets fixed - name: restore-cached-debs run: | [ -d ~/.cache/debs ] && sudo cp ~/.cache/debs/* /var/cache/apt/archives/ || mkdir -p ~/.cache/debs - uses: firebuild/firebuild-action@v3 with: key: build-on-ubuntu-lts - name: install-deps run: | sudo eatmydata apt-get -y install $BUILD_DEPS $TEST_DEPS doxygen lcov - name: build-out-of-tree run: | mkdir build cd build firebuild cmake -DCMAKE_BUILD_TYPE=Release .. firebuild make -j2 - name: doc run: | cd build firebuild doxygen - name: build-in-tree run: | firebuild cmake -DWITH_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Debug . firebuild make -j2 all check-bins - name: test run: | make check - name: coverage # tests don't run with out of tree builds at the moment run: | git clean -dxf firebuild cmake -DCOVERAGE=1 -DCMAKE_BUILD_TYPE=Debug . firebuild make all check-bins make -j2 check coverage-info [ $(echo "$(make coverage-info | grep '^[\.0-9]*$') >= 75" | bc) = 1 ] - name: save-cached-debs run: | rm -f ~/.cache/debs/* cp /var/cache/apt/archives/*deb ~/.cache/debs/ test-with-valgrind: needs: style-checks runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 with: path: ~/.cache/debs key: test-with-valgrind-debs-${{ needs.style-checks.outputs.week }} # TODO: This and saving the debs can be dropped and and the debs can be saved directly like in docker # when https://github.com/actions/cache/issues/324 gets fixed - name: restore-cached-debs run: | [ -d ~/.cache/debs ] && sudo cp ~/.cache/debs/* /var/cache/apt/archives/ || mkdir -p ~/.cache/debs - uses: firebuild/firebuild-action@v3 with: key: test-with-valgrind - name: install-deps run: | sudo eatmydata apt-get -y install $BUILD_DEPS $TEST_DEPS valgrind - name: build-in-tree run: | firebuild cmake -DWITH_JEMALLOC=OFF -DCMAKE_BUILD_TYPE=Debug . firebuild make -j2 all check-bins - name: test run: | firebuild make -j3 close_fds_exec test/close_fds_exec make valgrind-check - name: save-cached-debs run: | rm -f ~/.cache/debs/* cp /var/cache/apt/archives/*deb ~/.cache/debs/ # build on various Debian derivative releases in docker build-in-docker: strategy: matrix: container: ["ubuntu:rolling", "ubuntu:devel", "i386/debian"] needs: style-checks runs-on: ubuntu-latest container: ${{ matrix.container }} timeout-minutes: 15 steps: - name: disable-apt-docker-clean run: | # don't clean apt archive cache to make deb caching work rm -f /etc/apt/apt.conf.d/docker-clean - name: apt update run: | sed -i 's|/archive.ubuntu.com|/azure.archive.ubuntu.com|' /etc/apt/sources.list || true apt-get -qq update - name: install-deps run: | # configure tzdata in advance to prevent hanging at the prompt TZ=Europe/Budapest ln -snf /usr/share/zoneinfo/$TZ /etc/localtime echo $TZ > /etc/timezone apt-get install -y eatmydata eatmydata apt-get -y install $BUILD_DEPS $TEST_DEPS g++ gcc git # use checkout@v1 that works in i386 containers without node - uses: actions/checkout@v1 with: fetch-depth: 0 - name: build-out-of-tree run: | # avoid git error about repository ownership git config --global --add safe.directory $PWD mkdir build cd build cmake -DCMAKE_BUILD_TYPE=Release .. make -j2 # Version should be set in the source before tagging a release, # then the version should be unset right in the next commit. BUILT_VERSION=$(src/firebuild/firebuild --version | head -n1 | cut -d" " -f2) GIT_VERSION=$(git describe --tags | sed s/^v//) dpkg --compare-versions $BUILT_VERSION ge $GIT_VERSION - name: build-in-tree run: | cmake -DCMAKE_BUILD_TYPE=Debug . make -j2 all check-bins make -j2 check clang-build: needs: style-checks runs-on: ubuntu-22.04 timeout-minutes: 15 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 with: path: ~/.cache/debs key: clang-build-debs-${{ needs.style-checks.outputs.week }} - name: restore-cached-debs run: | [ -d ~/.cache/debs ] && sudo cp ~/.cache/debs/* /var/cache/apt/archives/ || mkdir -p ~/.cache/debs - uses: firebuild/firebuild-action@v3 with: key: clang-build - name: install-deps run: | sudo eatmydata apt-get -y install clang-tools $BUILD_DEPS $TEST_DEPS valgrind - name: build run: | env CC=clang CXX=clang++ LD=ld.lld cmake . firebuild make -j2 all check-bins - name: test run: | make check test/close_fds_exec make valgrind-check - name: clean run: make clean - name: scan-build run: | # work around static analyzer report about emmintrin.h with XXH_INLINE_ALL scan-build cmake -DENABLE_XXH_INLINE_ALL=OFF . scan-build --status-bugs make -j2 - name: save-cached-debs run: | rm -f ~/.cache/debs/* cp /var/cache/apt/archives/*deb ~/.cache/debs/ rebuild-self: needs: style-checks runs-on: ubuntu-22.04 timeout-minutes: 15 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 with: path: ~/.cache/debs key: rebuild-self-debs-${{ needs.style-checks.outputs.week }} - name: restore-cached-debs run: | [ -d ~/.cache/debs ] && sudo cp ~/.cache/debs/* /var/cache/apt/archives/ || mkdir -p ~/.cache/debs - uses: firebuild/firebuild-action@v3 with: key: rebuild-self - name: install-deps run: | sudo eatmydata apt-get -y install $BUILD_DEPS - name: rebuild self run: | tools/rebuild-self build # rebuild again, to test shortcutting cd build-first-build/test ./run-firebuild make -C ../../build clean time ./run-firebuild make -j2 -C ../../build all du -sh test_cache_dir - name: rebuild self with -j8 run: | cd build-first-build/test rm -r test_cache_dir ./run-firebuild make -C ../../build clean time ./run-firebuild make -j8 -C ../../build all ./run-firebuild make -C ../../build clean time ./run-firebuild make -j8 -C ../../build all - name: save-cached-debs run: | rm -f ~/.cache/debs/* cp /var/cache/apt/archives/*deb ~/.cache/debs/ build-other-projects: needs: style-checks runs-on: ubuntu-22.04 timeout-minutes: 15 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 with: path: ~/.cache/debs key: build-other-projects-debs-${{ needs.style-checks.outputs.week }} - name: restore-cached-debs run: | [ -d ~/.cache/debs ] && sudo cp ~/.cache/debs/* /var/cache/apt/archives/ || mkdir -p ~/.cache/debs - uses: firebuild/firebuild-action@v3 with: key: build-other-projects - name: install-deps run: | sudo sed -i 's/# deb-src/deb-src/' /etc/apt/sources.list # work around https://bugs.launchpad.net/ubuntu/+source/apt/+bug/1979244 echo 'APT::Get::Always-Include-Phased-Updates "true";' | sudo tee /etc/apt/apt.conf.d/99-firebuild-phased-updates sudo apt-get update sudo eatmydata apt-get -y install devscripts $BUILD_DEPS $TEST_DEPS sudo eatmydata apt-get build-dep vte2.91 - name: build-in-tree run: | firebuild cmake -DWITH_JEMALLOC=OFF -DSANITIZE=ON . firebuild make -j2 all check-bins make -j2 check - name: build-vte run: | apt-get source vte2.91 cd vte2.91-* meson build cd ../test ./run-firebuild ninja -j8 -C ../vte2.91-*/build ninja -C ../vte2.91-*/build clean ./run-firebuild ninja -j8 -C ../vte2.91-*/build - name: save-cached-debs run: | rm -f ~/.cache/debs/* cp /var/cache/apt/archives/*deb ~/.cache/debs/ build-deb: strategy: matrix: os: [ubuntu-20.04, ubuntu-22.04] needs: style-checks runs-on: ${{ matrix.os }} timeout-minutes: 15 steps: - uses: actions/checkout@v3 - uses: actions/cache@v3 with: path: ~/.cache/debs key: build-deb-debs-${{ matrix.os }}-${{ needs.style-checks.outputs.week }} - name: restore-cached-debs run: | [ -d ~/.cache/debs ] && sudo cp ~/.cache/debs/* /var/cache/apt/archives/ || mkdir -p ~/.cache/debs - uses: firebuild/firebuild-action@v3 with: key: build-deb-${{ matrix.os }} - name: install-deps run: | ! [ ${{ matrix.os }} == "ubuntu-20.04" ] || sudo add-apt-repository -y -n ppa:firebuild/build-deps sudo apt update sudo eatmydata apt-get build-dep . - name: deb run: | # skip dh_buildinfo step printf '\noverride_dh_buildinfo:\n' >> debian/rules # don't intercept bats and ldd to avoid breaking firebuild's own tests firebuild -o 'processes.dont_intercept += "bats"' -o 'processes.dont_intercept += "ldd"' dpkg-buildpackage -j2 -us -uc echo debconf firebuild/license-accepted select true | sudo debconf-set-selections sudo apt-get install -y --allow-downgrades ../*.deb firebuild -- ls - name: save-cached-debs run: | rm -f ~/.cache/debs/* cp /var/cache/apt/archives/*deb ~/.cache/debs/ build-on-macos: needs: style-checks runs-on: macos-12 timeout-minutes: 30 steps: - uses: actions/checkout@v3 - uses: hendrikmuhs/ccache-action@v1 with: key: build-on-macos-12 - name: install-deps run: | brew update brew install bats-core coreutils docbook-xsl jemalloc jinja2-cli libconfig xxhash (cd .. && git clone https://github.com/Tessil/hopscotch-map.git && cd hopscotch-map/ && cmake . && sudo make install) - name: build-in-tree run: | export PATH=/usr/local/opt/ccache/libexec:$(ls -d /usr/local/Cellar/jinja2-cli/0.8.2/libexec/bin):$PATH cmake -DCMAKE_CXX_FLAGS="-I/usr/local/include" -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/lib" -DCMAKE_BUILD_TYPE=Debug . export XML_CATALOG_FILES=/usr/local/etc/xml/catalog make -j2 - name: test run: | # this fails yet export XML_CATALOG_FILES=/usr/local/etc/xml/catalog make check || true style-checks: runs-on: ubuntu-22.04 outputs: week: ${{ steps.week-of-year.outputs.week }} # map step output to job output timeout-minutes: 2 steps: - uses: actions/checkout@v3 - name: install-deps run: | # TODO(rbalint) use packaged cpplint when it becomes available https://bugs.debian.org/960847 pip3 install cpplint==1.5.4 - name: style-check run: env PATH=$HOME/.local/bin:$PATH cpplint --recursive src test # this will be used by other jobs to invalidate cache after a week - name: week of year id: week-of-year run: echo "week=$(/bin/date -u "+%V")" >> $GITHUB_OUTPUT firebuild-0.8.2/.github/workflows/codeql.yml000066400000000000000000000015311447164520700210670ustar00rootroot00000000000000name: "CodeQL" on: schedule: - cron: '25 2 * * 5' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'cpp', 'python' ] steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: install-deps run: | sudo apt update sudo apt-get build-dep . - name: Autobuild uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" firebuild-0.8.2/.gitignore000066400000000000000000000005671447164520700155000ustar00rootroot00000000000000*~ *.gcno *.lock *.DS_Store *.swp *.out *.o *.so CMakeCache.txt CMakeDoxyfile.in CMakeDoxygenDefaults.cmake CMakeFiles/ CTestTestfile.cmake Doxyfile Makefile build/ cmake_install.cmake doc/html/ *_generated.h fbb.c fbb.h gcov/ gcovhtml/ src/fbbcomm.cc src/firebuild/fbbfp.cc src/firebuild/fbbstore.cc src/firebuild/firebuild test/fbbtest.cc test/test_orphan test/test_random firebuild-0.8.2/CMakeLists.txt000066400000000000000000000112641447164520700162440ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.13) project(Firebuild C CXX) include(CMakeOptions.txt) if (NOT DEFINED FIREBUILD_VERSION) # Set in tagged commits only set(FIREBUILD_VERSION "0.8.2") endif() if (NOT DEFINED FIREBUILD_VERSION) execute_process(COMMAND git describe OUTPUT_VARIABLE FIREBUILD_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) set(FIREBUILD_VERSION "Git ${FIREBUILD_VERSION}") endif() include(CheckIPOSupported) check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_SUPPORT_ERROR) if(NOT IPO_SUPPORTED) message(STATUS "IPO / LTO not supported: <${IPO_SUPPORT_ERROR}>") endif() include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) include(GNUInstallDirs) set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules) # TODO set to install dir set(DATADIR "${CMAKE_INSTALL_FULL_DATAROOTDIR}/firebuild" CACHE STRING "Firebuild's architecture-independent files' location") set(TENTATIVE_C_CXX_FLAGS -std=c++20 -W -Wall -Wextra -Werror -Wpointer-arith -Warray-bounds -Wcast-align -Wformat -Wformat-security -fstrict-overflow -Wstrict-overflow=4 -Wunreachable-code -Warray-bounds -fvisibility=hidden -Woverflow # -Wlogical-op clang does not like that -Wredundant-decls -Wno-format-zero-length) foreach(FLAG ${TENTATIVE_C_CXX_FLAGS}) string(MAKE_C_IDENTIFIER "C_FLAG_VALID_${FLAG}" V) check_c_compiler_flag("${FLAG}" ${V}) if (${V}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG}") endif() string(MAKE_C_IDENTIFIER "CXX_FLAG_VALID_${FLAG}" V_CXX) check_cxx_compiler_flag(${FLAG} ${V_CXX}) if (${V_CXX}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG}") endif() endforeach() if (NOT APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weffc++") endif() # dynamic library handling if (APPLE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DLD_PRELOAD=\\\"DYLD_INSERT_LIBRARIES\\\" -DLD_LIBRARY_PATH=\\\"DYLD_LIBRARY_PATH\\\"") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLD_PRELOAD=\\\"DYLD_INSERT_LIBRARIES\\\" -DLD_LIBRARY_PATH=\\\"DYLD_LIBRARY_PATH\\\"") set(LIBFIREBUILD_SO "libfirebuild.dylib") set(CMAKE_TARGET_LD_LIBRARY_PATH "DYLD_LIBRARY_PATH") else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DLD_PRELOAD=\\\"LD_PRELOAD\\\" -DLD_LIBRARY_PATH=\\\"LD_LIBRARY_PATH\\\"") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLD_PRELOAD=\\\"LD_PRELOAD\\\" -DLD_LIBRARY_PATH=\\\"LD_LIBRARY_PATH\\\"") set(LIBFIREBUILD_SO "libfirebuild.so") set(CMAKE_TARGET_LD_LIBRARY_PATH "LD_LIBRARY_PATH") endif() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DLD_LIBRARY_PATH=\\\"${CMAKE_TARGET_LD_LIBRARY_PATH}\\\"") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLD_LIBRARY_PATH=\\\"${CMAKE_TARGET_LD_LIBRARY_PATH}\\\"") if (APPLE) # build the interceptor for multiple architectures set(FAT_ARCH_C_FLAGS "-arch arm64 -arch arm64e -arch x86_64") set(CMAKE_SHARED_LINKER_FLAGS ${FAT_ARCH_C_FLAGS}) # TODO build the supervisor, too, for multiple architectures endif() if(COVERAGE) # minimal (lines) coverage set(MIN_COVERAGE "65") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fprofile-arcs -ftest-coverage") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -fprofile-arcs -ftest-coverage") endif(COVERAGE) if (SANITIZE) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") endif() add_definitions(-DFIREBUILD_DATADIR="${DATADIR}" -DLIBFIREBUILD_SO="${LIBFIREBUILD_SO}") if (ENABLE_XXH_INLINE_ALL) add_definitions(-DXXH_INLINE_ALL) endif() string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) if (uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") add_definitions(-DFB_EXTRA_DEBUG) endif() add_subdirectory(src) enable_testing() add_subdirectory(test) add_subdirectory(man) # add a target to generate API documentation with Doxygen find_package(Doxygen) if(DOXYGEN_FOUND) configure_file(${CMAKE_SOURCE_DIR}/Doxyfile.in ${CMAKE_BINARY_DIR}/Doxyfile @ONLY) add_custom_target(doc ${DOXYGEN_EXECUTABLE} ${CMAKE_BINARY_DIR}/Doxyfile WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM) endif(DOXYGEN_FOUND) if(COVERAGE) add_custom_target(coverage-info tools/calculate-coverage WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Generating coverage report with lcov" VERBATIM) endif(COVERAGE) add_custom_target(style-check cpplint --quiet --recursive ${CMAKE_SOURCE_DIR}/src/ ${CMAKE_SOURCE_DIR}/test/) configure_file("${CMAKE_SOURCE_DIR}/etc/firebuild.conf" "${CMAKE_BINARY_DIR}/etc/firebuild.conf" COPYONLY) install(FILES "${CMAKE_BINARY_DIR}/etc/firebuild.conf" DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}") install(FILES data/build-report.html DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/firebuild") install(FILES data/firebuild-logo.svg DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/firebuild") firebuild-0.8.2/CMakeOptions.txt000066400000000000000000000011701447164520700165740ustar00rootroot00000000000000# Build options for use by CMake # Disabling this option may be useful for triaging performance issues # or result faster hash computation where the system's libxxhash library # uses HW acceleration. See: https://bugs.debian.org/977345 option(ENABLE_XXH_INLINE_ALL "Enable -DXXH_INLINE_ALL" ON) # Detect and link with jemalloc when it is available. # Disabling linking with jemalloc may help in triaging memory handling issues. option(WITH_JEMALLOC "Link with jemalloc" ON) # Enables ASan for the supervisor and UBSan for the supervisor and the interceptor option(SANITIZE "Enable Address and Undefined behaviour sanitizer" OFF) firebuild-0.8.2/ChangeLog000066400000000000000000000176021447164520700152600ustar00rootroot00000000000000================== 0.8.2 / 2023-08-24 ================== * Fix the build 32 bit architectures * supervisor: Handle inherited fd offset properly * ci: Upgrade docker job to run multiple releases on Ubuntu and add Debian * test: Fix building and testing test_cmd_clone only on Linux * tests: Suppress yes: standard output: Broken pipe error * tests: Allow running tests on installed firebuild * debian/tests: Add autopkgtest running build time tests on installed firebuild * supervisor: Support GLibc 2.38 passing "--" right after "sh -c" * debian: Depend on at least the glibc version firebuild is built with ================== 0.8.1 / 2023-08-15 ================== * artwork: Fix the circle in the logo to be really round * interceptor: Factor out some fd read write state handling from templates * Report error code when a call reported only once fails * interceptor: Treat successful copy_file_range() and sendfile() calls as reads and writes * report: Include file name and process potentially writing the file in parallel * report: List only directly and not transitively used filed in process details * report: Deduplicate used file strings * report: Register used files at the shortcut process when generating the report * report: Deduplicate process environments * supervisor: Run main loop until all interceptor connections and pipes are closed * supervisor: Run all cleanup and destructors when FB_EXTRA_DEBUG is defined * ci: Run coverage test on Debug build and increase required coverage * interceptor: Intercept __vfork for glibc < 2.34 * supervisor: Show maximum resident set with "-d time" * supervisor: Defer clearinf obsolete finalized ExecedProcess data * supervisor: Free whole finalized process subtrees when it is safe * interceptor: Fix calling handle_exit() * supervisor: Track the common ancestor of processes keeping files open for writing * supervisor: Track the process exec_point() for files opened for writing * interceptor: Perform seccomp() calls in not intercepted processes * Use global buffer size for paths * supervisor: Don't use alloca() in garbage collection loops 0.8.0 / 2023-04-19 ================== * portability: Fixes to build on Mac OS X (interception is not functional yet) * readme: Mention packages in the Arch Linux (AUR), Debian and Ubuntu archives * interceptor: Intercept seccomp() calls and always return with EINVAL error Fixes intercepting and caching man commands * supervisor: Add default quirk to ignore successful statfs() and fstatfs() calls Fixes shortcutting Doxygen * tests: Fix building all binaries for valgrind-check target * ci: Split running tests with valgrind to a separate job * supervisor: Fall back to pread() when mmap() fails when hashing a file Fixes shortcutting Java and javadoc again after their behavior changes * ci: Use firebuild to speed up firebuild builds on Linux * ci: Use latest Clang and scan-build versions * supervisor: Cache "sh" again by default and * supervisor: Skip caching command when parent sh -c "" can be cached * supervisor: Ignore missing configuration settings instead of crashing * supervisor: Rename system_locations configuration entry to read_only_locations * interceptor: Make relative or absolute unsuccessfully dlopen()-ed paths absolute * supervisor: Treat unsuccessfully dlopen()-ed files using absolute filename as missing Fixes shortcutting Intel Fortran Compiler (ifort). * supervisor: Fix saved bytes accounting * supervisor: Verify stored bytes accounting in Debug builds * supervisor: Fix invalid stored cache size * supervisor: Don't report missing possible debug files when debugging cache during GC * ci: Add CodeQL analysis * supervisor: Intercept ccache, but set it to run the original compiler This allows relying on firebuild's caching by default even when ccache is installed. * supervisor: Shortcut only ccache and not child gcc/g++/etc. command * tests: Don't disable ccache in test helper scripts Ccache is now disabled within firebuild by default. * Ignore communication with GNU Make's and Cargo's jobserver This lets shortcutting processes that read tokens from the jobserver. * interceptor: Ignore getcap syscall It is used by Python and Node * supervisor: Don't store hash of dependency directory listing for rustc This allows shortcutting Rust + Cargo builds * interceptor: Report getrandom syscalls * interceptor: utimensat syscall accepts NULL pathname * supervisor: Allow configuring maximal cache entry size with default of 250MB * report: Include why shortcutting attempts failed * debian: Update short and long package description 0.2.12 / 2023-02-23 ================== * Handle system(NULL) * Detect inherited file backed fds that are not not seeked to the end in the interceptor * report: Don't break line inside a single environment variable * interceptor: Wide scanf variants return EOF, too, not WEOF * interceptor: Fix syscall template to honor rettype and ifdef_guard * supervisor: Prevent adding an fd with epoll_ctl() twice to avoid errors and crashes * supervisor: Deduplicate dynamically allocated reasons of not shortcutting * report: Include more verbose reasons for not not shortcutting * report: Don't escape "/" in JSON strings * report: Escape used files to not break the JSON format * Dutch translation of debconf messages * Spanish translation of the debconf template * Brazilian Portuguese debconf templates translation 0.2.11 / 2023-01-21 ================== * debian: Update copyright * debian: Initial German debconf translation * supervisor: Pass through HOME environment variable * tests: Skip testing orphan process handling on WSL * tests: Don't test with fakeroot if fakeroot does not work * debian: Build-depend on fakeroot (with ) * tests: Ignore ENOSYS error from memfd_create, timerfd_create and eventfd * tests: open(..., O_TMPFILE) returns EISDIR on WSL1, ignore that * supervisor: Fall back to emulating copy_file_range on ENOSYS error * supervisor: Fall back to renameat() if renameat2() return ENOSYS or EINVAL * interceptor: __gettimeofday64 is added only in Glibc 2.34 * debian: Don't require node-d3 for building firebuild * interceptor: Add #ifdef guards for a few functions not existing in glibc 2.27 * supervisor: Add minimal renameat2() wrapper when it is missing * interceptor: Define gettimeofday() for Glibc < 2.31, too * test: Ignore harmless extra symbols appearing with older toolchains * interceptor: Ignore new public symbols appearing on Bionic * common: Define CLONE_PIDFD and statx() structs and defines when they are missing * debian: Armel autopkgtests are slower * interceptor: Check and intercept missing __* variants * interceptor: Check if *64 syscall variants are intercepted * interceptor: Check the *_time64 bit variants of intercepted functions are also intercepted * interceptor: Check if *64 bit variants of intercepted functions are also intercepted * debian: Bump standards version No changes were needed * supervisor: Handle faccessat() with unknown dirfd * interceptor: MIPS has > 64 signals, don't assert() on that * common: Don't decode SIGSTKFLT in debugging when it is not defined * supervisor: Add --option=key=[] option to clear a configuration array * readme: bc is a required test dependency * interceptor: Intercept clone() with flags == CLONE_VFORK | SIGCHLD as a fork() * interceptor: Intercept _Fork calling pthread_atfork handlers directly * interceptor: Enable/disable interception of functions with #ifdef guards * supervisor: Add LAUNCH_TYPE_POSIX_SPAWN * interceptor: Don't check global intercepting_enabled in env_needs_fixup() * interceptor: Factor out fixing up and restoring environment to a macro * interceptor: Treat fallocate() and posix_fallocate() as pwrites * report: Fix typo in the title * debian: Show firebuild stats in autopkgtest firebuild-0.8.2/Doxyfile.in000066400000000000000000002316041447164520700156210ustar00rootroot00000000000000# Doxyfile 1.8.1.2 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project # # All text after a hash (#) is considered a comment and will be ignored # The format is: # TAG = value [value, ...] # For lists items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (" ") #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file # that follow. The default is UTF-8 which is also the encoding used for all # text before the first occurrence of this tag. Doxygen uses libiconv (or the # iconv built into libc) for the transcoding. See # http://www.gnu.org/software/libiconv for the list of possible encodings. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or sequence of words) that should # identify the project. Note that if you do not use Doxywizard you need # to put quotes around the project name if it contains spaces. PROJECT_NAME = Firebuild # The PROJECT_NUMBER tag can be used to enter a project or revision number. # This could be handy for archiving the generated documentation or # if some version control system is used. PROJECT_NUMBER = # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer # a quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = # With the PROJECT_LOGO tag one can specify an logo or icon that is # included in the documentation. The maximum height of the logo should not # exceed 55 pixels and the maximum width should not exceed 200 pixels. # Doxygen will copy the logo to the output directory. PROJECT_LOGO = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. # If a relative path is entered, it will be relative to the location # where doxygen was started. If left blank the current directory will be used. OUTPUT_DIRECTORY = @CMAKE_BINARY_DIR@/doc # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create # 4096 sub-directories (in 2 levels) under the output directory of each output # format and will distribute the generated files over these directories. # Enabling this option can be useful when feeding doxygen a huge amount of # source files, where putting all generated files in the same directory would # otherwise cause performance problems for the file system. CREATE_SUBDIRS = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # The default language is English, other supported languages are: # Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, # Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, # Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English # messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, # Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, # Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will # include brief member descriptions after the members that are listed in # the file and class documentation (similar to JavaDoc). # Set to NO to disable this. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend # the brief description of a member or function before the detailed description. # Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator # that is used to form the text in various listings. Each string # in this list, if found as the leading text of the brief description, will be # stripped from the text and the result after processing the whole list, is # used as the annotated text. Otherwise, the brief description is used as-is. # If left blank, the following values are used ("$name" is automatically # replaced with the name of the entity): "The $name class" "The $name widget" # "The $name file" "is" "provides" "specifies" "contains" # "represents" "a" "an" "the" ABBREVIATE_BRIEF = "The $name class" \ "The $name widget" \ "The $name file" \ is \ provides \ specifies \ contains \ represents \ a \ an \ the # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # Doxygen will generate a detailed section even if there is only a brief # description. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full # path before files name in the file list and in the header files. If set # to NO the shortest path that makes the file name unique will be used. FULL_PATH_NAMES = YES # If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag # can be used to strip a user-defined part of the path. Stripping is # only done if one of the specified strings matches the left-hand part of # the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the # path to strip. STRIP_FROM_PATH = # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of # the path mentioned in the documentation of a class, which tells # the reader which header file to include in order to use a class. # If left blank only the name of the header file containing the class # definition is used. Otherwise one should specify the include paths that # are normally passed to the compiler using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter # (but less readable) file names. This can be useful if your file system # doesn't support long names like on DOS, Mac, or CD-ROM. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen # will interpret the first line (until the first dot) of a JavaDoc-style # comment as the brief description. If set to NO, the JavaDoc # comments will behave just like regular Qt-style comments # (thus requiring an explicit @brief command for a brief description.) JAVADOC_AUTOBRIEF = NO # If the QT_AUTOBRIEF tag is set to YES then Doxygen will # interpret the first line (until the first dot) of a Qt-style # comment as the brief description. If set to NO, the comments # will behave just like regular Qt-style comments (thus requiring # an explicit \brief command for a brief description.) QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen # treat a multi-line C++ special comment block (i.e. a block of //! or /// # comments) as a brief description. This used to be the default behaviour. # The new default is to treat a multi-line C++ comment block as a detailed # description. Set this tag to YES if you prefer the old behaviour instead. MULTILINE_CPP_IS_BRIEF = NO # If the INHERIT_DOCS tag is set to YES (the default) then an undocumented # member inherits the documentation from any documented member that it # re-implements. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce # a new page for each member. If set to NO, the documentation of a member will # be part of the file/class/namespace that contains it. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. # Doxygen uses this value to replace tabs by spaces in code fragments. TAB_SIZE = 8 # This tag can be used to specify a number of aliases that acts # as commands in the documentation. An alias has the form "name=value". # For example adding "sideeffect=\par Side Effects:\n" will allow you to # put the command \sideeffect (or @sideeffect) in the documentation, which # will result in a user-defined paragraph with heading "Side Effects:". # You can put \n's in the value part of an alias to insert newlines. ALIASES = # This tag can be used to specify a number of word-keyword mappings (TCL only). # A mapping has the form "name=value". For example adding # "class=itcl::class" will allow you to use the command class in the # itcl::class meaning. TCL_SUBST = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C # sources only. Doxygen will then generate output that is more tailored for C. # For instance, some of the names that are used will be different. The list # of all members will be omitted, etc. OPTIMIZE_OUTPUT_FOR_C = NO # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java # sources only. Doxygen will then generate output that is more tailored for # Java. For instance, namespaces will be presented as packages, qualified # scopes will look different, etc. OPTIMIZE_OUTPUT_JAVA = NO # Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran # sources only. Doxygen will then generate output that is more tailored for # Fortran. OPTIMIZE_FOR_FORTRAN = NO # Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL # sources. Doxygen will then generate output that is tailored for # VHDL. OPTIMIZE_OUTPUT_VHDL = NO # Doxygen selects the parser to use depending on the extension of the files it # parses. With this tag you can assign which parser to use for a given extension. # Doxygen has a built-in mapping, but you can override or extend it using this # tag. The format is ext=language, where ext is a file extension, and language # is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, # C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make # doxygen treat .inc files as Fortran files (default is PHP), and .f files as C # (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions # you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. EXTENSION_MAPPING = # If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all # comments according to the Markdown format, which allows for more readable # documentation. See http://daringfireball.net/projects/markdown/ for details. # The output of markdown processing is further processed by doxygen, so you # can mix doxygen, HTML, and XML commands with Markdown formatting. # Disable only in case of backward compatibilities issues. MARKDOWN_SUPPORT = YES # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should # set this tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); v.s. # func(std::string) {}). This also makes the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. # Doxygen will parse them like normal C++ but will assume all classes use public # instead of private inheritance when no explicit protection keyword is present. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate getter # and setter methods for a property. Setting this option to YES (the default) # will make doxygen replace the get and set methods by a property in the # documentation. This will only work if the methods are indeed getting or # setting a simple type. If this is not the case, or you want to show the # methods anyway, you should set this option to NO. IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a # subgroup of that type (e.g. under the Public Functions section). Set it to # NO to prevent subgrouping. Alternatively, this can be done per class using # the \nosubgrouping command. SUBGROUPING = YES # When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and # unions are shown inside the group in which they are included (e.g. using # @ingroup) instead of on a separate page (for HTML and Man pages) or # section (for LaTeX and RTF). INLINE_GROUPED_CLASSES = NO # When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and # unions with only public data fields will be shown inline in the documentation # of the scope in which they are defined (i.e. file, namespace, or group # documentation), provided this scope is documented. If set to NO (the default), # structs, classes, and unions are shown on a separate page (for HTML and Man # pages) or section (for LaTeX and RTF). INLINE_SIMPLE_STRUCTS = NO # When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum # is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically # be useful for C code in case the coding convention dictates that all compound # types are typedef'ed and only the typedef is referenced, never the tag name. TYPEDEF_HIDES_STRUCT = NO # The SYMBOL_CACHE_SIZE determines the size of the internal cache use to # determine which symbols to keep in memory and which to flush to disk. # When the cache is full, less often used symbols will be written to disk. # For small to medium size projects (<1000 input files) the default value is # probably good enough. For larger projects a too small cache size can cause # doxygen to be busy swapping symbols to and from disk most of the time # causing a significant performance penalty. # If the system has enough physical memory increasing the cache will improve the # performance by keeping more symbols in memory. Note that the value works on # a logarithmic scale so increasing the size by one will roughly double the # memory usage. The cache size is given by this formula: # 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, # corresponding to a cache size of 2^16 = 65536 symbols. SYMBOL_CACHE_SIZE = 0 # Similar to the SYMBOL_CACHE_SIZE the size of the symbol lookup cache can be # set using LOOKUP_CACHE_SIZE. This cache is used to resolve symbols given # their name and scope. Since this can be an expensive process and often the # same symbol appear multiple times in the code, doxygen keeps a cache of # pre-resolved symbols. If the cache is too small doxygen will become slower. # If the cache is too large, memory is wasted. The cache size is given by this # formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range is 0..9, the default is 0, # corresponding to a cache size of 2^16 = 65536 symbols. LOOKUP_CACHE_SIZE = 0 #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. # Private class members and static file members will be hidden unless # the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES EXTRACT_ALL = YES # If the EXTRACT_PRIVATE tag is set to YES all private members of a class # will be included in the documentation. EXTRACT_PRIVATE = NO # If the EXTRACT_PACKAGE tag is set to YES all members with package or internal # scope will be included in the documentation. EXTRACT_PACKAGE = NO # If the EXTRACT_STATIC tag is set to YES all static members of a file # will be included in the documentation. EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) # defined locally in source files will be included in the documentation. # If set to NO only classes defined in header files are included. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local # methods, which are defined in the implementation section but not in # the interface are included in the documentation. # If set to NO (the default) only methods in the interface are included. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be # extracted and appear in the documentation as a namespace called # 'anonymous_namespace{file}', where file will be replaced with the base # name of the file that contains the anonymous namespace. By default # anonymous namespaces are hidden. EXTRACT_ANON_NSPACES = NO # If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members of documented classes, files or namespaces. # If set to NO (the default) these members will be included in the # various overviews, but no documentation section is generated. # This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. # If set to NO (the default) these classes will be included in the various # overviews. This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all # friend (class|struct|union) declarations. # If set to NO (the default) these declarations will be included in the # documentation. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. # If set to NO (the default) these blocks will be appended to the # function's detailed documentation block. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation # that is typed after a \internal command is included. If the tag is set # to NO (the default) then the documentation will be excluded. # Set it to YES to include the internal documentation. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate # file names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. CASE_SENSE_NAMES = NO # If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen # will show members with their full class and namespace scopes in the # documentation. If set to YES the scope will be hidden. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen # will put a list of the files that are included by a file in the documentation # of that file. SHOW_INCLUDE_FILES = YES # If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen # will list include files with double quotes in the documentation # rather than with sharp brackets. FORCE_LOCAL_INCLUDES = NO # If the INLINE_INFO tag is set to YES (the default) then a tag [inline] # is inserted in the documentation for inline members. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen # will sort the (detailed) documentation of file and class members # alphabetically by member name. If set to NO the members will appear in # declaration order. SORT_MEMBER_DOCS = YES # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the # brief documentation of file, namespace and class members alphabetically # by member name. If set to NO (the default) the members will appear in # declaration order. SORT_BRIEF_DOCS = NO # If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen # will sort the (brief and detailed) documentation of class members so that # constructors and destructors are listed first. If set to NO (the default) # the constructors will appear in the respective orders defined by # SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. # This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO # and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. SORT_MEMBERS_CTORS_1ST = NO # If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the # hierarchy of group names into alphabetical order. If set to NO (the default) # the group names will appear in their defined order. SORT_GROUP_NAMES = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be # sorted by fully-qualified names, including namespaces. If set to # NO (the default), the class list will be sorted only by class name, # not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the # alphabetical list. SORT_BY_SCOPE_NAME = NO # If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to # do proper type resolution of all parameters of a function it will reject a # match between the prototype and the implementation of a member function even # if there is only one candidate or it is obvious which candidate to choose # by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen # will still accept a match between prototype and implementation in such cases. STRICT_PROTO_MATCHING = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or # disable (NO) the todo list. This list is created by putting \todo # commands in the documentation. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or # disable (NO) the test list. This list is created by putting \test # commands in the documentation. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or # disable (NO) the bug list. This list is created by putting \bug # commands in the documentation. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or # disable (NO) the deprecated list. This list is created by putting # \deprecated commands in the documentation. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional # documentation sections, marked by \if sectionname ... \endif. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines # the initial value of a variable or macro consists of for it to appear in # the documentation. If the initializer consists of more lines than specified # here it will be hidden. Use a value of 0 to hide initializers completely. # The appearance of the initializer of individual variables and macros in the # documentation can be controlled using \showinitializer or \hideinitializer # command in the documentation regardless of this setting. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated # at the bottom of the documentation of classes and structs. If set to YES the # list will mention the files that were used to generate the documentation. SHOW_USED_FILES = YES # Set the SHOW_FILES tag to NO to disable the generation of the Files page. # This will remove the Files entry from the Quick Index and from the # Folder Tree View (if specified). The default is YES. SHOW_FILES = YES # Set the SHOW_NAMESPACES tag to NO to disable the generation of the # Namespaces page. This will remove the Namespaces entry from the Quick Index # and from the Folder Tree View (if specified). The default is YES. SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command , where is the value of # the FILE_VERSION_FILTER tag, and is the name of an input file # provided by doxygen. Whatever the program writes to standard output # is used as the file version. See the manual for examples. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed # by doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. # You can optionally specify a file name after the option, if omitted # DoxygenLayout.xml will be used as the name of the layout file. LAYOUT_FILE = # The CITE_BIB_FILES tag can be used to specify one or more bib files # containing the references data. This must be a list of .bib files. The # .bib extension is automatically appended if omitted. Using this command # requires the bibtex tool to be installed. See also # http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style # of the bibliography can be controlled using LATEX_BIB_STYLE. To use this # feature you need bibtex and perl available in the search path. CITE_BIB_FILES = #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated # by doxygen. Possible values are YES and NO. If left blank NO is used. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated by doxygen. Possible values are YES and NO. If left blank # NO is used. WARNINGS = YES # If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings # for undocumented members. If EXTRACT_ALL is set to YES then this flag will # automatically be disabled. WARN_IF_UNDOCUMENTED = YES # If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some # parameters in a documented function, or documenting parameters that # don't exist or using markup commands wrongly. WARN_IF_DOC_ERROR = YES # The WARN_NO_PARAMDOC option can be enabled to get warnings for # functions that are documented, but have no documentation for their parameters # or return value. If set to NO (the default) doxygen will only warn about # wrong or incomplete parameter documentation, but not about the absence of # documentation. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that # doxygen can produce. The string should contain the $file, $line, and $text # tags, which will be replaced by the file and line number from which the # warning originated and the warning text. Optionally the format may contain # $version, which will be replaced by the version of the file (if it could # be obtained via FILE_VERSION_FILTER) WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning # and error messages should be written. If left blank the output is written # to stderr. WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag can be used to specify the files and/or directories that contain # documented source files. You may enter file names like "myfile.cpp" or # directories like "/usr/src/myproject". Separate the files or directories # with spaces. INPUT = @CMAKE_SOURCE_DIR@/src # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is # also the default input encoding. Doxygen uses libiconv (or the iconv built # into libc) for the transcoding. See http://www.gnu.org/software/libiconv for # the list of possible encodings. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank the following patterns are tested: # *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh # *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py # *.f90 *.f *.for *.vhd *.vhdl FILE_PATTERNS = *.c \ *.cc \ *.cxx \ *.cpp \ *.c++ \ *.d \ *.java \ *.ii \ *.ixx \ *.ipp \ *.i++ \ *.inl \ *.h \ *.hh \ *.hxx \ *.hpp \ *.h++ \ *.idl \ *.odl \ *.cs \ *.php \ *.php3 \ *.inc \ *.m \ *.markdown \ *.md \ *.mm \ *.dox \ *.py \ *.f90 \ *.f \ *.for \ *.vhd \ *.vhdl # The RECURSIVE tag can be used to turn specify whether or not subdirectories # should be searched for input files as well. Possible values are YES and NO. # If left blank NO is used. RECURSIVE = YES # The EXCLUDE tag can be used to specify files and/or directories that should be # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # Note that relative paths are relative to the directory from which doxygen is # run. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded # from the input. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. Note that the wildcards are matched # against the file with absolute path, so to exclude all test directories # for example use the pattern */test/* EXCLUDE_PATTERNS = tpl*.c tpl*.h *_generated.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, # AClass::ANamespace, ANamespace::*Test EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or # directories that contain example code fragments that are included (see # the \include command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank all files are included. EXAMPLE_PATTERNS = * # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude # commands irrespective of the value of the RECURSIVE tag. # Possible values are YES and NO. If left blank NO is used. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or # directories that contain image that are included in the documentation (see # the \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command , where # is the value of the INPUT_FILTER tag, and is the name of an # input file. Doxygen will then use the output that the filter program writes # to standard output. If FILTER_PATTERNS is specified, this tag will be # ignored. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: # pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further # info on how filters are used. If FILTER_PATTERNS is empty or if # non of the patterns match the file name, INPUT_FILTER is applied. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will be used to filter the input files when producing source # files to browse (i.e. when SOURCE_BROWSER is set to YES). FILTER_SOURCE_FILES = NO # The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file # pattern. A pattern will override the setting for FILTER_PATTERN (if any) # and it is also possible to disable source filtering for a specific pattern # using *.ext= (so without naming a filter). This option only has effect when # FILTER_SOURCE_FILES is enabled. FILTER_SOURCE_PATTERNS = #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will # be generated. Documented entities will be cross-referenced with these sources. # Note: To get rid of all source code in the generated output, make sure also # VERBATIM_HEADERS is set to NO. SOURCE_BROWSER = YES # Setting the INLINE_SOURCES tag to YES will include the body # of functions and classes directly in the documentation. INLINE_SOURCES = NO # Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct # doxygen to hide any special comment blocks from generated source code # fragments. Normal C, C++ and Fortran comments will always remain visible. STRIP_CODE_COMMENTS = YES # If the REFERENCED_BY_RELATION tag is set to YES # then for each documented function all documented # functions referencing it will be listed. REFERENCED_BY_RELATION = NO # If the REFERENCES_RELATION tag is set to YES # then for each documented function all documented entities # called/used by that function will be listed. REFERENCES_RELATION = NO # If the REFERENCES_LINK_SOURCE tag is set to YES (the default) # and SOURCE_BROWSER tag is set to YES, then the hyperlinks from # functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will # link to the source code. Otherwise they will link to the documentation. REFERENCES_LINK_SOURCE = YES # If the USE_HTAGS tag is set to YES then the references to source code # will point to the HTML generated by the htags(1) tool instead of doxygen # built-in source browser. The htags tool is part of GNU's global source # tagging system (see http://www.gnu.org/software/global/global.html). You # will need version 4.8.6 or higher. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen # will generate a verbatim copy of the header file for each class for # which an include is specified. Set to NO to disable this. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index # of all compounds will be generated. Enable this if the project # contains a lot of classes, structs, unions or interfaces. ALPHABETICAL_INDEX = YES # If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then # the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns # in which this list will be split (can be a number in the range [1..20]) COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all # classes will be put under the same header in the alphabetical index. # The IGNORE_PREFIX tag can be used to specify one or more prefixes that # should be ignored while generating the index headers. IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES (the default) Doxygen will # generate HTML output. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `html' will be used as the default path. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for # each generated HTML page (for example: .htm,.php,.asp). If it is left blank # doxygen will generate files with .html extension. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a personal HTML header for # each generated HTML page. If it is left blank doxygen will generate a # standard header. Note that when using a custom header you are responsible # for the proper inclusion of any scripts and style sheets that doxygen # needs, which is dependent on the configuration options used. # It is advised to generate a default header using "doxygen -w html # header.html footer.html stylesheet.css YourConfigFile" and then modify # that header. Note that the header is subject to change so you typically # have to redo this when upgrading to a newer version of doxygen or when # changing the value of configuration settings such as GENERATE_TREEVIEW! HTML_HEADER = # The HTML_FOOTER tag can be used to specify a personal HTML footer for # each generated HTML page. If it is left blank doxygen will generate a # standard footer. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading # style sheet that is used by each HTML page. It can be used to # fine-tune the look of the HTML output. If the tag is left blank doxygen # will generate a default style sheet. Note that doxygen will try to copy # the style sheet file to the HTML output directory, so don't put your own # style sheet in the HTML output directory as well, or it will be erased! HTML_STYLESHEET = # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or # other source files which should be copied to the HTML output directory. Note # that these files will be copied to the base HTML output directory. Use the # $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these # files. In the HTML_STYLESHEET file, use the file name only. Also note that # the files will be copied as-is; there are no commands or markers available. HTML_EXTRA_FILES = # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. # Doxygen will adjust the colors in the style sheet and background images # according to this color. Hue is specified as an angle on a colorwheel, # see http://en.wikipedia.org/wiki/Hue for more information. # For instance the value 0 represents red, 60 is yellow, 120 is green, # 180 is cyan, 240 is blue, 300 purple, and 360 is red again. # The allowed range is 0 to 359. HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of # the colors in the HTML output. For a value of 0 the output will use # grayscales only. A value of 255 will produce the most vivid colors. HTML_COLORSTYLE_SAT = 100 # The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to # the luminance component of the colors in the HTML output. Values below # 100 gradually make the output lighter, whereas values above 100 make # the output darker. The value divided by 100 is the actual gamma applied, # so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, # and 100 does not change the gamma. HTML_COLORSTYLE_GAMMA = 80 # If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML # page will contain the date and time when the page was generated. Setting # this to NO can help when comparing the output of multiple runs. HTML_TIMESTAMP = YES # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. HTML_DYNAMIC_SECTIONS = NO # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of # entries shown in the various tree structured indices initially; the user # can expand and collapse entries dynamically later on. Doxygen will expand # the tree to such a level that at most the specified number of entries are # visible (unless a fully collapsed tree already exceeds this amount). # So setting the number of entries 1 will produce a full collapsed tree by # default. 0 is a special value representing an infinite number of entries # and will result in a full expanded tree by default. HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files # will be generated that can be used as input for Apple's Xcode 3 # integrated development environment, introduced with OSX 10.5 (Leopard). # To create a documentation set, doxygen will generate a Makefile in the # HTML output directory. Running make will produce the docset in that # directory and running "make install" will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find # it at startup. # See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html # for more information. GENERATE_DOCSET = NO # When GENERATE_DOCSET tag is set to YES, this tag determines the name of the # feed. A documentation feed provides an umbrella under which multiple # documentation sets from a single provider (such as a company or product suite) # can be grouped. DOCSET_FEEDNAME = "Doxygen generated docs" # When GENERATE_DOCSET tag is set to YES, this tag specifies a string that # should uniquely identify the documentation set bundle. This should be a # reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen # will append .docset to the name. DOCSET_BUNDLE_ID = org.doxygen.Project # When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify # the documentation publisher. This should be a reverse domain-name style # string, e.g. com.mycompany.MyDocSet.documentation. DOCSET_PUBLISHER_ID = org.doxygen.Publisher # The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES, additional index files # will be generated that can be used as input for tools like the # Microsoft HTML help workshop to generate a compiled HTML help file (.chm) # of the generated HTML documentation. GENERATE_HTMLHELP = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can # be used to specify the file name of the resulting .chm file. You # can add a path in front of the file if the result should not be # written to the html output directory. CHM_FILE = # If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can # be used to specify the location (absolute path including file name) of # the HTML help compiler (hhc.exe). If non-empty doxygen will try to run # the HTML help compiler on the generated index.hhp. HHC_LOCATION = # If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag # controls if a separate .chi index file is generated (YES) or that # it should be included in the master .chm file (NO). GENERATE_CHI = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING # is used to encode HtmlHelp index (hhk), content (hhc) and project file # content. CHM_INDEX_ENCODING = # If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag # controls whether a binary table of contents is generated (YES) or a # normal table of contents (NO) in the .chm file. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members # to the contents of the HTML help documentation and to the tree view. TOC_EXPAND = NO # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated # that can be used as input for Qt's qhelpgenerator to generate a # Qt Compressed Help (.qch) of the generated HTML documentation. GENERATE_QHP = NO # If the QHG_LOCATION tag is specified, the QCH_FILE tag can # be used to specify the file name of the resulting .qch file. # The path specified is relative to the HTML output folder. QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating # Qt Help Project output. For more information please see # http://doc.trolltech.com/qthelpproject.html#namespace QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating # Qt Help Project output. For more information please see # http://doc.trolltech.com/qthelpproject.html#virtual-folders QHP_VIRTUAL_FOLDER = doc # If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to # add. For more information please see # http://doc.trolltech.com/qthelpproject.html#custom-filters QHP_CUST_FILTER_NAME = # The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see # # Qt Help Project / Custom Filters. QHP_CUST_FILTER_ATTRS = # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this # project's # filter section matches. # # Qt Help Project / Filter Attributes. QHP_SECT_FILTER_ATTRS = # If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can # be used to specify the location of Qt's qhelpgenerator. # If non-empty doxygen will try to run qhelpgenerator on the generated # .qhp file. QHG_LOCATION = # If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files # will be generated, which together with the HTML files, form an Eclipse help # plugin. To install this plugin and make it available under the help contents # menu in Eclipse, the contents of the directory containing the HTML and XML # files needs to be copied into the plugins directory of eclipse. The name of # the directory within the plugins directory should be the same as # the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before # the help appears. GENERATE_ECLIPSEHELP = NO # A unique identifier for the eclipse help plugin. When installing the plugin # the directory name containing the HTML and XML files should also have # this name. ECLIPSE_DOC_ID = org.doxygen.Project # The DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) # at top of each HTML page. The value NO (the default) enables the index and # the value YES disables it. Since the tabs have the same information as the # navigation tree you can set this option to NO if you already set # GENERATE_TREEVIEW to YES. DISABLE_INDEX = NO # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. # If the tag value is set to YES, a side panel will be generated # containing a tree-like index structure (just like the one that # is generated for HTML Help). For this to work a browser that supports # JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). # Windows users are probably better off using the HTML help feature. # Since the tree basically has the same information as the tab index you # could consider to set DISABLE_INDEX to NO when enabling this option. GENERATE_TREEVIEW = YES # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values # (range [0,1..20]) that doxygen will group on one line in the generated HTML # documentation. Note that a value of 0 will completely suppress the enum # values from appearing in the overview section. ENUM_VALUES_PER_LINE = 4 # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be # used to set the initial width (in pixels) of the frame in which the tree # is shown. TREEVIEW_WIDTH = 250 # When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open # links to external symbols imported via tag files in a separate window. EXT_LINKS_IN_WINDOW = NO # Use this tag to change the font size of Latex formulas included # as images in the HTML documentation. The default is 10. Note that # when you change the font size after a successful doxygen run you need # to manually remove any form_*.png images from the HTML output directory # to force them to be regenerated. FORMULA_FONTSIZE = 10 # Use the FORMULA_TRANPARENT tag to determine whether or not the images # generated for formulas are transparent PNGs. Transparent PNGs are # not supported properly for IE 6.0, but are supported on all modern browsers. # Note that when changing this option you need to delete any form_*.png files # in the HTML output before the changes have effect. FORMULA_TRANSPARENT = YES # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax # (see http://www.mathjax.org) which uses client side Javascript for the # rendering instead of using prerendered bitmaps. Use this if you do not # have LaTeX installed or if you want to formulas look prettier in the HTML # output. When enabled you may also need to install MathJax separately and # configure the path to it using the MATHJAX_RELPATH option. USE_MATHJAX = NO # When MathJax is enabled you need to specify the location relative to the # HTML output directory using the MATHJAX_RELPATH option. The destination # directory should contain the MathJax.js script. For instance, if the mathjax # directory is located at the same level as the HTML output directory, then # MATHJAX_RELPATH should be ../mathjax. The default value points to # the MathJax Content Delivery Network so you can quickly see the result without # installing MathJax. However, it is strongly recommended to install a local # copy of MathJax from http://www.mathjax.org before deployment. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension # names that should be enabled during MathJax rendering. MATHJAX_EXTENSIONS = # When the SEARCHENGINE tag is enabled doxygen will generate a search box # for the HTML output. The underlying search engine uses javascript # and DHTML and should work on any modern browser. Note that when using # HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets # (GENERATE_DOCSET) there is already a search function so this one should # typically be disabled. For large projects the javascript based search engine # can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. SEARCHENGINE = YES # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a PHP enabled web server instead of at the web client # using Javascript. Doxygen will generate the search PHP script and index # file to put on the web server. The advantage of the server # based approach is that it scales better to large projects and allows # full text search. The disadvantages are that it is more difficult to setup # and does not have live searching capabilities. SERVER_BASED_SEARCH = NO #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES (the default) Doxygen will # generate Latex output. GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `latex' will be used as the default path. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. If left blank `latex' will be used as the default command name. # Note that when enabling USE_PDFLATEX this option is only used for # generating bitmaps for formulas in the HTML output, but not in the # Makefile that is written to the output directory. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to # generate index for LaTeX. If left blank `makeindex' will be used as the # default command name. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES Doxygen generates more compact # LaTeX documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used # by the printer. Possible values are: a4, letter, legal and # executive. If left blank a4wide will be used. PAPER_TYPE = a4 # The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX # packages that should be included in the LaTeX output. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for # the generated latex document. The header should contain everything until # the first chapter. If it is left blank doxygen will generate a # standard header. Notice: only use this tag if you know what you are doing! LATEX_HEADER = # The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for # the generated latex document. The footer should contain everything after # the last chapter. If it is left blank doxygen will generate a # standard footer. Notice: only use this tag if you know what you are doing! LATEX_FOOTER = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated # is prepared for conversion to pdf (using ps2pdf). The pdf file will # contain links (just like the HTML output) instead of page references # This makes the output suitable for online browsing using a pdf viewer. PDF_HYPERLINKS = YES # If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of # plain latex in the generated Makefile. Set this option to YES to get a # higher quality PDF documentation. USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. # command to the generated LaTeX files. This will instruct LaTeX to keep # running if errors occur, instead of asking the user for help. # This option is also used when generating formulas in HTML. LATEX_BATCHMODE = NO # If LATEX_HIDE_INDICES is set to YES then doxygen will not # include the index chapters (such as File Index, Compound Index, etc.) # in the output. LATEX_HIDE_INDICES = NO # If LATEX_SOURCE_CODE is set to YES then doxygen will include # source code with syntax highlighting in the LaTeX output. # Note that which sources are shown also depends on other settings # such as SOURCE_BROWSER. LATEX_SOURCE_CODE = NO # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See # http://en.wikipedia.org/wiki/BibTeX for more info. LATEX_BIB_STYLE = plain #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- # If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output # The RTF output is optimized for Word 97 and may not look very pretty with # other RTF readers or editors. GENERATE_RTF = NO # The RTF_OUTPUT tag is used to specify where the RTF docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `rtf' will be used as the default path. RTF_OUTPUT = rtf # If the COMPACT_RTF tag is set to YES Doxygen generates more compact # RTF documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_RTF = NO # If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated # will contain hyperlink fields. The RTF file will # contain links (just like the HTML output) instead of page references. # This makes the output suitable for online browsing using WORD or other # programs which support those fields. # Note: wordpad (write) and others do not support links. RTF_HYPERLINKS = NO # Load style sheet definitions from file. Syntax is similar to doxygen's # config file, i.e. a series of assignments. You only have to provide # replacements, missing definitions are set to their default value. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an rtf document. # Syntax is similar to doxygen's config file. RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES (the default) Doxygen will # generate man pages GENERATE_MAN = NO # The MAN_OUTPUT tag is used to specify where the man pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `man' will be used as the default path. MAN_OUTPUT = man # The MAN_EXTENSION tag determines the extension that is added to # the generated man pages (default is the subroutine's section .3) MAN_EXTENSION = .3 # If the MAN_LINKS tag is set to YES and Doxygen generates man output, # then it will generate one additional man file for each entity # documented in the real man page(s). These additional files # only source the real man page, but without them the man command # would be unable to find the correct page. The default is NO. MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- # If the GENERATE_XML tag is set to YES Doxygen will # generate an XML file that captures the structure of # the code including all documentation. GENERATE_XML = NO # The XML_OUTPUT tag is used to specify where the XML pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `xml' will be used as the default path. XML_OUTPUT = xml # The XML_SCHEMA tag can be used to specify an XML schema, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_SCHEMA = # The XML_DTD tag can be used to specify an XML DTD, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_DTD = # If the XML_PROGRAMLISTING tag is set to YES Doxygen will # dump the program listings (including syntax highlighting # and cross-referencing information) to the XML output. Note that # enabling this will significantly increase the size of the XML output. XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will # generate an AutoGen Definitions (see autogen.sf.net) file # that captures the structure of the code including all # documentation. Note that this feature is still experimental # and incomplete at the moment. GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- # If the GENERATE_PERLMOD tag is set to YES Doxygen will # generate a Perl module file that captures the structure of # the code including all documentation. Note that this # feature is still experimental and incomplete at the # moment. GENERATE_PERLMOD = NO # If the PERLMOD_LATEX tag is set to YES Doxygen will generate # the necessary Makefile rules, Perl scripts and LaTeX code to be able # to generate PDF and DVI output from the Perl module output. PERLMOD_LATEX = NO # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be # nicely formatted so it can be parsed by a human reader. This is useful # if you want to understand what is going on. On the other hand, if this # tag is set to NO the size of the Perl module output will be much smaller # and Perl will parse it just the same. PERLMOD_PRETTY = YES # The names of the make variables in the generated doxyrules.make file # are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. # This is useful so different doxyrules.make files included by the same # Makefile don't overwrite each other's variables. PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will # evaluate all C-preprocessor directives found in the sources and include # files. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro # names in the source code. If set to NO (the default) only conditional # compilation will be performed. Macro expansion can be done in a controlled # way by setting EXPAND_ONLY_PREDEF to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES # then the macro expansion is limited to the macros specified with the # PREDEFINED and EXPAND_AS_DEFINED tags. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES (the default) the includes files # pointed to by INCLUDE_PATH will be searched when a #include is found. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by # the preprocessor. INCLUDE_PATH = @CMAKE_SOURCE_DIR@/src @CMAKE_BINARY_DIR@/src # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will # be used. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that # are defined before the preprocessor is started (similar to the -D option of # gcc). The argument of the tag is a list of macros of the form: name # or name=definition (no spaces). If the definition and the = are # omitted =1 is assumed. To prevent a macro definition from being # undefined via #undef or recursively expanded use the := operator # instead of the = operator. PREDEFINED = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. # The macro definition that is found in the sources will be used. # Use the PREDEFINED tag if you want to use a different macro definition that # overrules the definition found in the source code. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all references to function-like macros # that are alone on a line, have an all uppercase name, and do not end with a # semicolon, because these will confuse the parser if not removed. SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- # The TAGFILES option can be used to specify one or more tagfiles. For each # tag file the location of the external documentation should be added. The # format of a tag file without this location is as follows: # TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: # TAGFILES = file1=loc1 "file2 = loc2" ... # where "loc1" and "loc2" can be relative or absolute paths # or URLs. Note that each tag file must have a unique name (where the name does # NOT include the path). If a tag file is not located in the directory in which # doxygen is run, you must also specify the path to the tagfile here. TAGFILES = # When a file name is specified after GENERATE_TAGFILE, doxygen will create # a tag file that is based on the input files it reads. GENERATE_TAGFILE = # If the ALLEXTERNALS tag is set to YES all external classes will be listed # in the class index. If set to NO only the inherited external classes # will be listed. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed # in the modules index. If set to NO, only the current project's groups will # be listed. EXTERNAL_GROUPS = YES # The PERL_PATH should be the absolute path and name of the perl script # interpreter (i.e. the result of `which perl'). PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- # If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will # generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base # or super classes. Setting the tag to NO turns the diagrams off. Note that # this option also works with HAVE_DOT disabled, but it is recommended to # install and use dot, since it yields more powerful graphs. CLASS_DIAGRAMS = NO # You can define message sequence charts within doxygen comments using the \msc # command. Doxygen will then run the mscgen tool (see # http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the # documentation. The MSCGEN_PATH tag allows you to specify the directory where # the mscgen tool resides. If left empty the tool is assumed to be found in the # default search path. MSCGEN_PATH = # If set to YES, the inheritance and collaboration graphs will hide # inheritance and usage relations if the target is undocumented # or is not a class. HIDE_UNDOC_RELATIONS = YES # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz, a graph visualization # toolkit from AT&T and Lucent Bell Labs. The other options in this section # have no effect if this option is set to NO (the default) HAVE_DOT = YES # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is # allowed to run in parallel. When set to 0 (the default) doxygen will # base this on the number of processors available in the system. You can set it # explicitly to a value larger than 0 to get control over the balance # between CPU load and processing speed. DOT_NUM_THREADS = 0 # By default doxygen will use the Helvetica font for all dot files that # doxygen generates. When you want a differently looking font you can specify # the font name using DOT_FONTNAME. You need to make sure dot is able to find # the font, which can be done by putting it in a standard location or by setting # the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the # directory containing the font. DOT_FONTNAME = Helvetica # The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. # The default size is 10pt. DOT_FONTSIZE = 10 # By default doxygen will tell dot to use the Helvetica font. # If you specify a different font using DOT_FONTNAME you can use DOT_FONTPATH to # set the path where dot can find it. DOT_FONTPATH = # If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect inheritance relations. Setting this tag to YES will force the # CLASS_DIAGRAMS tag to NO. CLASS_GRAPH = YES # If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect implementation dependencies (inheritance, containment, and # class references variables) of the class with other documented classes. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen # will generate a graph for groups, showing the direct groups dependencies GROUP_GRAPHS = YES # If the UML_LOOK tag is set to YES doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. UML_LOOK = NO # If the UML_LOOK tag is enabled, the fields and methods are shown inside # the class node. If there are many fields or methods and many nodes the # graph may become too big to be useful. The UML_LIMIT_NUM_FIELDS # threshold limits the number of items for each type to make the size more # managable. Set this to 0 for no limit. Note that the threshold may be # exceeded by 50% before the limit is enforced. UML_LIMIT_NUM_FIELDS = 10 # If set to YES, the inheritance and collaboration graphs will show the # relations between templates and their instances. TEMPLATE_RELATIONS = NO # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT # tags are set to YES then doxygen will generate a graph for each documented # file showing the direct and indirect include dependencies of the file with # other documented files. INCLUDE_GRAPH = YES # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and # HAVE_DOT tags are set to YES then doxygen will generate a graph for each # documented header file showing the documented files that directly or # indirectly include this file. INCLUDED_BY_GRAPH = YES # If the CALL_GRAPH and HAVE_DOT options are set to YES then # doxygen will generate a call dependency graph for every global function # or class method. Note that enabling this option will significantly increase # the time of a run. So in most cases it will be better to enable call graphs # for selected functions only using the \callgraph command. CALL_GRAPH = NO # If the CALLER_GRAPH and HAVE_DOT tags are set to YES then # doxygen will generate a caller dependency graph for every global function # or class method. Note that enabling this option will significantly increase # the time of a run. So in most cases it will be better to enable caller # graphs for selected functions only using the \callergraph command. CALLER_GRAPH = NO # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen # will generate a graphical hierarchy of all classes instead of a textual one. GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH and HAVE_DOT tags are set to YES # then doxygen will show the dependencies a directory has on other directories # in a graphical way. The dependency relations are determined by the #include # relations between the files in the directories. DIRECTORY_GRAPH = YES # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. Possible values are svg, png, jpg, or gif. # If left blank png will be used. If you choose svg you need to set # HTML_FILE_EXTENSION to xhtml in order to make the SVG files # visible in IE 9+ (other browsers do not have this requirement). DOT_IMAGE_FORMAT = png # If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to # enable generation of interactive SVG images that allow zooming and panning. # Note that this requires a modern browser other than Internet Explorer. # Tested and working are Firefox, Chrome, Safari, and Opera. For IE 9+ you # need to set HTML_FILE_EXTENSION to xhtml in order to make the SVG files # visible. Older versions of IE do not have SVG support. INTERACTIVE_SVG = NO # The tag DOT_PATH can be used to specify the path where the dot tool can be # found. If left blank, it is assumed the dot tool can be found in the path. DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the # \dotfile command). DOTFILE_DIRS = # The MSCFILE_DIRS tag can be used to specify one or more directories that # contain msc files that are included in the documentation (see the # \mscfile command). MSCFILE_DIRS = # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of # nodes that will be shown in the graph. If the number of nodes in a graph # becomes larger than this value, doxygen will truncate the graph, which is # visualized by representing a node as a red box. Note that doxygen if the # number of direct children of the root node in a graph is already larger than # DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note # that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. DOT_GRAPH_MAX_NODES = 50 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the # graphs generated by dot. A depth value of 3 means that only nodes reachable # from the root by following a path via at most 3 edges will be shown. Nodes # that lay further from the root node will be omitted. Note that setting this # option to 1 or 2 may greatly reduce the computation time needed for large # code bases. Also note that the size of a graph can be further restricted by # DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. MAX_DOT_GRAPH_DEPTH = 0 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent # background. This is disabled by default, because dot on Windows does not # seem to support this out of the box. Warning: Depending on the platform used, # enabling this option may lead to badly anti-aliased labels on the edges of # a graph (i.e. they become hard to read). DOT_TRANSPARENT = NO # Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) # support this, this feature is disabled by default. DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will # generate a legend page explaining the meaning of the various boxes and # arrows in the dot generated graphs. GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES (the default) Doxygen will # remove the intermediate dot files that are used to generate # the various graphs. DOT_CLEANUP = YES WARN_AS_ERROR = YES firebuild-0.8.2/LICENSE.md000066400000000000000000000014401447164520700151030ustar00rootroot00000000000000Copyright © 2022 Firebuild Inc. All rights reserved. * Free for personal use or commercial trial * Non-trial commercial use requires licenses available from [firebuild.com](https://firebuild.com) * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. firebuild-0.8.2/README.md000066400000000000000000000113111447164520700147540ustar00rootroot00000000000000# Firebuild Firebuild logo ## Introduction Firebuild is an automatic build accelerator. It works by caching the outputs of executed commands and replaying the results when the same commands are executed with the same parameters within the same environment. The commands can be compilation or other build artifact generation steps, tests, or any command that produces predictable output. The commands to cache and replay from the cache are determined automatically based on `firebuild`'s [configuration](etc/firebuild.conf) and each command's and its children's observed behavior. Firebuild accelerates cc and ld with LTO ## Usage Prefix your build command with `firebuild`: firebuild The first build is typically a 5-10% slower due to the overhead of analyzing the build and populating the cache. Successive builds can be 5-20 times or even faster depending on the project and the changes between the builds. ### Improving hit rate If the hit rate (check with `firebuild -s`) is not at the expected levels run firebuild with `-d proc` to see why particular commands can't be stored to the cache or with `-d shortcut` to see what prevents shortcutting of particular commands. Firebuild can also generate a report about the whole build process with `firebuild -r ` that helps finding the slowest parts of the build and also helps finding what could not be shortcut. #### Clang PCH's embedded timestamps prevents shortcutting Clang embeds timestamps in precompiled headers (PCHs) by default on Linux in the PCH generation steps (`-emit-pch`). To let Firebuild cache PCHs use `-Xclang -fno-pch-timestamp` with `clang`. ## How it compares to other build accelerators? ### Ccache, sccache and other compiler wrappers Ccache works by having an understanding of how the C/C++ compiler works and it supports only the C/C++/Obj-C family of languages. Sccache adds Rust on top of those, and the concept is the same. Firebuild supports accelerating any command that behaves reasonably well, like not downloading files from the network to produce output. As a result you can accelerate the linker (even with LTO) , or the configure and code generation steps of C/C++ project builds, which together made it consistently beat ccache in [our testing](https://github.com/firebuild/firebuild-infra/pull/59). Firebuild also supports many other compilers, such as Fortran, Java (including Javadoc generation) and Scala compilers which are not accelerated by ccache. ### Bazel and similar build systems Bazel requires maintaining the Bazel build system, while Firebuilds works with any build system that does not implement its own caching. You just need to prefix your build command with firebuild, like it is done for accelerating bash's build in the autopkgtest: [debian/tests/recompile-bash](debian/tests/recompile-bash) ``` ... Build times: real1=88.28 user1=113.08 sys1=26.20 real2=12.38 user2=8.31 sys2=5.67 CPU time of the second build was 10% of the first build autopkgtest [14:24:12]: test recompile-bash: -----------------------] recompile-bash PASS ``` You can even accelerate a single command without any build system: `firebuild `. ### Firebuild shortcomings Firebuild does not support [compressing cache entries](https://github.com/firebuild/firebuild/issues/1087), nor [remote caches](https://github.com/firebuild/firebuild/issues/19) yet. Firebuild's interception works by preloading libfirebuild.so to the intercepted processes and interposing libc and system calls. As a result it can't intercept nor shortcut statically linked binaries. ## Installation Firebuild is available in [Arch Linux (AUR)](https://aur.archlinux.org/packages/firebuild), [Debian](https://tracker.debian.org/pkg/firebuild), [Ubuntu](https://launchpad.net/ubuntu/+source/firebuild), and other Debian derivatives. Back-ported packages for supported Ubuntu releases can be downloaded from the [official PPA](https://launchpad.net/~firebuild/+archive/ubuntu/stable): sudo add-apt-repository ppa:firebuild/stable sudo apt install firebuild If you would like to use `firebuild` in your GitHub pipeline there is a [GitHub Action](https://github.com/marketplace/actions/firebuild-for-github-actions) to do just that. ## Building from source For Ubuntu earlier than 21.04 (xxhash earlier than 0.8.0 or Valgrind earlier than 3.17.0): sudo apt-add-repository ppa:firebuild/build-deps Install the build dependencies: sudo apt update sudo apt install clang cmake bats bc graphviz libconfig++-dev node-d3 libxxhash-dev libjemalloc-dev libtsl-hopscotch-map-dev moreutils python3-jinja2 fakeroot Build: cmake . && make firebuild-0.8.2/cmake/000077500000000000000000000000001447164520700145605ustar00rootroot00000000000000firebuild-0.8.2/cmake/modules/000077500000000000000000000000001447164520700162305ustar00rootroot00000000000000firebuild-0.8.2/cmake/modules/FindLibConfig.cmake000066400000000000000000000041421447164520700216700ustar00rootroot00000000000000# GPLv3 from https://github.com/schnorr/pajeng/blob/master/cmake/FindLibConfig.cmake # TODO clarify copyright # Find the CUnit includes and library # # This module defines # LIBCONFIG_INCLUDE_DIR, where to find cppunit include files, etc. # LIBCONFIG_LIBRARIES, the libraries to link against to use CppUnit. # LIBCONFIG_STATIC_LIBRARIY_PATH # LIBCONFIG_FOUND, If false, do not try to use CppUnit. # also defined, but not for general use are # LIBCONFIG_LIBRARY, where to find the CUnit library. #MESSAGE("Searching for libconfig library") FIND_PATH(LIBCONFIG_INCLUDE_DIR libconfig.h /usr/local/include /usr/include ) FIND_PATH(LIBCONFIGPP_INCLUDE_DIR libconfig.h++ /usr/local/include /usr/include ) FIND_LIBRARY(LIBCONFIG_LIBRARY config /usr/local/lib /usr/lib ) FIND_LIBRARY(LIBCONFIGPP_LIBRARY config++ /usr/local/lib /usr/lib ) FIND_LIBRARY(LIBCONFIG_STATIC_LIBRARY "libconfig${CMAKE_STATIC_LIBRARY_SUFFIX}" /usr/local/lib /usr/lib ) FIND_LIBRARY(LIBCONFIGPP_STATIC_LIBRARY "libconfig++${CMAKE_STATIC_LIBRARY_SUFFIX}" /usr/local/lib /usr/lib ) IF(LIBCONFIG_INCLUDE_DIR) IF(LIBCONFIG_LIBRARY) SET(LIBCONFIG_FOUND TRUE) SET(LIBCONFIG_LIBRARIES ${LIBCONFIG_LIBRARY}) SET(LIBCONFIG_STATIC_LIBRARY_PATH ${LIBCONFIG_STATIC_LIBRARY}) ENDIF(LIBCONFIG_LIBRARY) ENDIF(LIBCONFIG_INCLUDE_DIR) IF(LIBCONFIGPP_INCLUDE_DIR) IF(LIBCONFIGPP_LIBRARY) SET(LIBCONFIGPP_FOUND TRUE) SET(LIBCONFIGPP_LIBRARIES ${LIBCONFIGPP_LIBRARY}) SET(LIBCONFIGPP_STATIC_LIBRARY_PATH ${LIBCONFIGPP_STATIC_LIBRARY}) ENDIF(LIBCONFIGPP_LIBRARY) ENDIF(LIBCONFIGPP_INCLUDE_DIR) IF (LIBCONFIG_FOUND) IF (NOT LibConfig_FIND_QUIETLY) MESSAGE(STATUS "Found LibConfig++: ${LIBCONFIGPP_LIBRARIES}" ) MESSAGE(STATUS "Found LibConfig: ${LIBCONFIG_LIBRARIES}") MESSAGE(STATUS "static LibConfig path: ${LIBCONFIG_STATIC_LIBRARY_PATH}") ENDIF (NOT LibConfig_FIND_QUIETLY) ELSE (LIBCONFIG_FOUND) IF (LibConfig_FIND_REQUIRED) MESSAGE(SEND_ERROR "Could NOT find LibConfig") ENDIF (LibConfig_FIND_REQUIRED) ENDIF (LIBCONFIG_FOUND) MARK_AS_ADVANCED(LIBCONFIG_INCLUDE_DIR LIBCONFIG_LIBRARIES)firebuild-0.8.2/data/000077500000000000000000000000001447164520700144115ustar00rootroot00000000000000firebuild-0.8.2/data/build-report.html000066400000000000000000000512621447164520700177150ustar00rootroot00000000000000 Firebuild build report
firebuild-0.8.2/data/firebuild-logo.svg000066400000000000000000000355501447164520700200450ustar00rootroot00000000000000 Firebuild Logo Firebuild Logo 2022 Firebuild Inc. Released under Firebuild's license, see accompanied LICENSE.md firebuild-0.8.2/debian/000077500000000000000000000000001447164520700147225ustar00rootroot00000000000000firebuild-0.8.2/debian/changelog000066400000000000000000000002161447164520700165730ustar00rootroot00000000000000firebuild (0.1) UNRELEASED; urgency=medium * Initial Release. -- Balint Reczey Sun, 21 Feb 2021 22:47:40 +0100 firebuild-0.8.2/debian/config000066400000000000000000000003761447164520700161200ustar00rootroot00000000000000#! /bin/sh set -e . /usr/share/debconf/confmodule db_get firebuild/license-accepted if [ "$RET" != "true" ]; then # show license again db_fset firebuild/license-accepted seen false fi db_input critical firebuild/license-accepted || true db_go firebuild-0.8.2/debian/control000066400000000000000000000052441447164520700163320ustar00rootroot00000000000000Source: firebuild Section: non-free/devel Priority: optional Maintainer: Balint Reczey Build-Depends: bats, bc, clang, cmake (>= 3.13), debhelper-compat (= 12), docbook-xsl, fakeroot , gcc (>= 4:10) | gcc-10, g++ (>= 4:10) | g++-10, graphviz, libconfig++-dev, libjemalloc-dev, libtsl-hopscotch-map-dev, libxxhash-dev (>= 0.8), moreutils, po-debconf, python3-jinja2, xsltproc Standards-Version: 4.6.2 Homepage: https://firebuild.com Rules-Requires-Root: no XS-Autobuild: yes Package: firebuild Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, ${glibc:Depends}, libfirebuild0 (= ${binary:Version}) Pre-Depends: debconf | debconf-2.0 Suggests: graphviz, node-d3 Description: Automatic build accelerator cache It works by caching the outputs of executed commands and replaying the results when the same commands are executed with the same parameters within the same environment. . The commands can be compilation or other build artifact generation steps, tests, or any command that produces predictable output. The commands to cache and replay from the cache are determined automatically based on firebuild's configuration and each command's and its children's observed behavior. . Firebuild supports caching compilation results of C, C++, Fortran, Java, Rust, Scala and other compilers and outputs of scripts written in Bash, Perl, Python and other interpreted languages. Package: libfirebuild0 Architecture: any Multi-Arch: same Depends: ${shlibs:Depends}, ${misc:Depends}, ${glibc:Depends} Pre-Depends: debconf | debconf-2.0 Description: Automatic build accelerator cache - shared library It works by caching the outputs of executed commands and replaying the results when the same commands are executed with the same parameters within the same environment. . The commands can be compilation or other build artifact generation steps, tests, or any command that produces predictable output. The commands to cache and replay from the cache are determined automatically based on firebuild's configuration and each command's and its children's observed behavior. . Firebuild supports caching compilation results of C, C++, Fortran, Java, Rust, Scala and other compilers and outputs of scripts written in Bash, Perl, Python and other interpreted languages. . This package provides the shared library preloaded by the intercepted processes. firebuild-0.8.2/debian/copyright000066400000000000000000000052561447164520700166650ustar00rootroot00000000000000Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: firebuild Upstream-Contact: info@firebuild.com Source: https://firebuild.com Files: * Copyright: 2022-2023 Firebuild Inc. License: Firebuild-license Files: debian/* Copyright: 2022-2023 Firebuild Inc. License: Expat Comment: The packaging explicitly uses a more liberal license to clearly allow and encourage modifications for integrating the software into software distributions. Files: debian/po/de.po Copyright: 2023 Helge Kreutzmann License: Expat Files: debian/po/es.po Copyright: 2023 Camaleón License: Expat Files: debian/po/nl.po Copyright: 2023 Frans Spiesschaert License: Expat Files: debian/po/pt_BR.po Copyright: 2023 Paulo Henrique de Lima Santana (phls) License: Expat License: Firebuild-license All rights reserved. . Free for personal use or commercial trial . Non-trial commercial use requires licenses available from https://firebuild.com. . Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license . THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. License: Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: . The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. . THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. firebuild-0.8.2/debian/firebuild.install000066400000000000000000000000471447164520700202600ustar00rootroot00000000000000debian/tmp/etc/* etc usr/bin usr/share firebuild-0.8.2/debian/libfirebuild0.install000066400000000000000000000000101447164520700210150ustar00rootroot00000000000000usr/lib firebuild-0.8.2/debian/libfirebuild0.lintian-overrides000066400000000000000000000002261447164520700230160ustar00rootroot00000000000000# The shared library is preloaded and should not be linked with where the dev package would be useful libfirebuild0: non-dev-pkg-with-shlib-symlink * firebuild-0.8.2/debian/po/000077500000000000000000000000001447164520700153405ustar00rootroot00000000000000firebuild-0.8.2/debian/po/POTFILES.in000066400000000000000000000000441447164520700171130ustar00rootroot00000000000000[type: gettext/rfc822deb] templates firebuild-0.8.2/debian/po/de.po000066400000000000000000000077611447164520700163030ustar00rootroot00000000000000# SOME DESCRIPTIVE TITLE. # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the firebuild package. # Helge Kreutzmann , 2023. # msgid "" msgstr "" "Project-Id-Version: firebuild 0.2.10-1\n" "Report-Msgid-Bugs-To: firebuild@packages.debian.org\n" "POT-Creation-Date: 2022-11-14 17:54+0100\n" "PO-Revision-Date: 2023-01-20 16:44+0100\n" "Last-Translator: Helge Kreutzmann \n" "Language-Team: German \n" "Language: de\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" #. Type: boolean #. Description #: ../templates:1001 msgid "Do you accept the terms of the Firebuild license?" msgstr "Akzeptieren Sie die Bedingungen der Firebuild-Lizenz?" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Firebuild users are required to acknowledge and accept the license. Please " "find it below. If you accept this license, the package installation will " "continue. If you refuse it, it will be interrupted." msgstr "" "Benutzer von Firebuild müssen die Lizenz bestätigen und akzeptieren. Sie " "finden diese nachfolgend. Falls Sie der Lizenz zustimmen, wird die " "Paketinstallation fortfahren. Falls Sie diese ablehnen, wird die " "Installation unterbrochen." #. Type: boolean #. Description #: ../templates:1001 msgid "Firebuild is free for personal use or commercial trial." msgstr "" "Firebuild ist für die persönliche Verwendung und kommerzielle Erprobung " "kostenlos." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Non-trial commercial use requires licenses available from https://firebuild." "com." msgstr "" "Kommerzielle Verwendung jenseits der Erprobung benötigt eine von " "https://firebuild.com erhältliche Lizenz." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Modification and redistribution are permitted, but commercial use of " "derivative works is subject to the same requirements of this license." msgstr "" "Veränderung und Weiterverteilung sind erlaubt, aber kommerzielle Verwendung " "von abgeleiteten Werken unterliegt den gleichen Bedingungen dieser Lizenz." #. Type: boolean #. Description #: ../templates:1001 msgid "" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR " "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE " "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER " "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING " "FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS " "IN THE SOFTWARE." msgstr "" "DIE SOFTWARE WIRD \"WIE BESEHEN\" ZUR VERFÜGUNG GESTELLT, OHNE GARANTIE " "JEGLICHER ART, AUSDRÜCKLICH ODER STILLSCHWEIGEND, EINSCHLIESSLICH, ABER " "NICHT BESCHRÄNKT AUF DIE GARANTIE DER MARKTGÄNGIGKEIT, EIGNUNG FÜR EINEN " "BESTIMMTEN ZWECK UND NICHTVERLETZUNG VON RECHTEN. IN KEINEM FALL SIND DIE " "AUTOREN ODER URHEBERRECHTSINHABER FÜR IRGENDWELCHE ANSPRÜCHE, SCHÄDEN ODER " "ANDERE HAFTUNGSGRÜNDE HAFTBAR, EGAL OB AUS VERTRAGSAUSÜBUNG, UNERLAUBTER " "HANDLUNG ODER ANDERWEITIG, DIE AUS ODER IN VERBINDUNG MIT DER SOFTWARE ODER " "DER NUTZUNG ODER DEM SONSTIGEN UMGANG MIT DER SOFTWARE ENTSTEHEN." #. Type: error #. Description #: ../templates:2001 msgid "The license has been refused" msgstr "Die Lizenz wurde abgelehnt" #. Type: error #. Description #: ../templates:2001 msgid "" "Please remove the firebuild packages or reinstall the firebuild package (e." "g. via apt-get install --reinstall firebuild) to get prompted to accept the " "license again." msgstr "" "Bitte entfernen Sie die Firebuild-Pakete oder installieren Sie die Firebuild-" "Pakete neu (z.B. mittels apt-get install --reinstall firebuild), um erneut " "bezüglich der Zustimmung zur Lizenz gefragt zu werden." firebuild-0.8.2/debian/po/es.po000066400000000000000000000076571447164520700163260ustar00rootroot00000000000000# firebuild po-debconf translation to Spanish # Copyright (C) 2023 firebuild # This file is distributed under the same license as the firebuild package. # Camaleón , 2023. # msgid "" msgstr "" "Project-Id-Version: firebuild 0.2.11-1\n" "Report-Msgid-Bugs-To: firebuild@packages.debian.org\n" "POT-Creation-Date: 2022-11-14 17:54+0100\n" "PO-Revision-Date: 2023-02-04 10:38+0100\n" "Language-Team: Spanish \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "X-Generator: Poedit 2.4.2\n" "Last-Translator: Camaleón \n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" "Language: es\n" #. Type: boolean #. Description #: ../templates:1001 msgid "Do you accept the terms of the Firebuild license?" msgstr "¿Acepta los términos de la licencia de Firebuild?" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Firebuild users are required to acknowledge and accept the license. Please " "find it below. If you accept this license, the package installation will " "continue. If you refuse it, it will be interrupted." msgstr "" "Los usuarios de Firebuild deben conocer y aceptar los términos de la " "licencia. Puede consultarlos más abajo. Si acepta la licencia, continuará la " "instalación del paquete. Si la rechaza, se cancelará la instalación del " "paquete." #. Type: boolean #. Description #: ../templates:1001 msgid "Firebuild is free for personal use or commercial trial." msgstr "Firebuild es gratuito para uso personal o como versión de prueba." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Non-trial commercial use requires licenses available from https://firebuild." "com." msgstr "" "El uso comercial requiere una licencia disponible en «https://firebuild.com»." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Modification and redistribution are permitted, but commercial use of " "derivative works is subject to the same requirements of this license." msgstr "" "Se permite la modificación y redistribución de este programa, pero el uso " "comercial de trabajos derivados está sujeto a los mismos requerimientos de " "la licencia." #. Type: boolean #. Description #: ../templates:1001 msgid "" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR " "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE " "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER " "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING " "FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS " "IN THE SOFTWARE." msgstr "" "EL SOFTWARE SE PROPORCIONA \"COMO ESTÁ\", SIN GARANTÍA DE NINGÚN TIPO, " "EXPRESA O IMPLÍCITA, INCLUYENDO PERO NO LIMITADO A GARANTÍAS DE " "COMERCIALIZACIÓN, IDONEIDAD PARA UN PROPÓSITO PARTICULAR E INCUMPLIMIENTO. " "EN NINGÚN CASO LOS AUTORES O PROPIETARIOS DE LOS DERECHOS DE AUTOR SERÁN " "RESPONSABLES DE NINGUNA RECLAMACIÓN, DAÑOS U OTRAS RESPONSABILIDADES, YA SEA " "EN UNA ACCIÓN DE CONTRATO, AGRAVIO O CUALQUIER OTRO MOTIVO, DERIVADAS DE, " "FUERA DE O EN CONEXIÓN CON EL SOFTWARE O SU USO U OTRO TIPO DE ACCIONES EN " "EL SOFTWARE." #. Type: error #. Description #: ../templates:2001 msgid "The license has been refused" msgstr "Ha rechazado la licencia" #. Type: error #. Description #: ../templates:2001 msgid "" "Please remove the firebuild packages or reinstall the firebuild package (e." "g. via apt-get install --reinstall firebuild) to get prompted to accept the " "license again." msgstr "" "Elimine o reinstale el paquete de firebuild (p. ej., con la orden «apt-get " "install --reinstall firebuild») para que se le pregunte nuevamente si acepta " "los términos de la licencia." firebuild-0.8.2/debian/po/nl.po000066400000000000000000000101131447164520700163050ustar00rootroot00000000000000# Dutch translation of firebuild debconf templates. # SOME DESCRIPTIVE TITLE. # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the firebuild package. # FIRST AUTHOR , YEAR. # Frans Spiesschaert , 2023. # msgid "" msgstr "" "Project-Id-Version: firebuild_0.2.10-1\n" "Report-Msgid-Bugs-To: firebuild@packages.debian.org\n" "POT-Creation-Date: 2022-11-14 17:54+0100\n" "PO-Revision-Date: 2023-01-20 16:48+0100\n" "Language: nl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Last-Translator: Frans Spiesschaert \n" "Language-Team: Debian Dutch l10n Team \n" "X-Generator: Poedit 2.2.1\n" #. Type: boolean #. Description #: ../templates:1001 msgid "Do you accept the terms of the Firebuild license?" msgstr "Aanvaardt u de voorwaarden van de Firebuild-licentie?" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Firebuild users are required to acknowledge and accept the license. Please " "find it below. If you accept this license, the package installation will " "continue. If you refuse it, it will be interrupted." msgstr "" "Gebruikers van Firebuild moeten ontvangst van de licentie bevestigen en deze " "accepteren. U vindt ze hieronder. Als u deze licentie accepteert, zal de " "installatie van het pakket doorgaan. Als u ze weigert, zal de installatie " "worden afgebroken." #. Type: boolean #. Description #: ../templates:1001 msgid "Firebuild is free for personal use or commercial trial." msgstr "" "Firebuild is vrij te gebruiken voor persoonlijk gebruik of voor een " "commerciële proefperiode." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Non-trial commercial use requires licenses available from https://firebuild." "com." msgstr "" "Voor commercieel gebruik buiten de proefperiode is een licentie vereist, " "welke beschikbaar zijn op https://firebuild.com." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Modification and redistribution are permitted, but commercial use of " "derivative works is subject to the same requirements of this license." msgstr "" "Wijziging en herdistributie zijn toegestaan, maar commercieel gebruik van " "afgeleide werken is onderworpen aan dezelfde vereisten uit deze licentie." #. Type: boolean #. Description #: ../templates:1001 msgid "" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR " "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE " "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER " "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING " "FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS " "IN THE SOFTWARE." msgstr "" "DE SOFTWARE WORDT GELEVERD \"ZOALS DEZE IS\", ZONDER EENDER WELKE GARANTIE, " "UITDRUKKELIJK OF IMPLICIET, MET INBEGRIP VAN, MAAR NIET BEPERKT TOT DE " "GARANTIE VAN VERKOOPBAARHEID, GESCHIKTHEID VOOR EEN BEPAALD DOEL OF VRIJ TE " "ZIJN VAN INBREUKEN. IN GEEN GEVAL KUNNEN DE AUTEURS OF DE " "AUTEURSRECHTENHOUDERS AANSPRAKELIJK ZIJN VOOR ENIGE SCHULDVORDERING, SCHADE-" "EIS OF ANDERE AANSPRAKELIJKHEID, ONGEACHT OF DEZE KADERT IN EEN CONTRACTUELE " "HANDELING, EEN ONRECHTMATIGE DAAD OF ANDERSZINS, HET GEVOLG IS VAN, " "VOORTVLOEIT UIT OF IN VERBAND STAAT MET DE SOFTWARE OF HET GEBRUIK ERVAN OF " "VAN ANDERE HANDELINGEN IN DE SOFTWARE." #. Type: error #. Description #: ../templates:2001 msgid "The license has been refused" msgstr "De licentie werd geweigerd" #. Type: error #. Description #: ../templates:2001 msgid "" "Please remove the firebuild packages or reinstall the firebuild package (e." "g. via apt-get install --reinstall firebuild) to get prompted to accept the " "license again." msgstr "" "Verwijder de firebuild-pakketten of installeer het firebuild-pakket opnieuw " "(bijvoorbeeld via apt-get install --reinstall firebuild) om opnieuw de vraag " "te krijgen de licentie te accepteren." firebuild-0.8.2/debian/po/pt_BR.po000066400000000000000000000074631447164520700167200ustar00rootroot00000000000000# Debconf translations for firebuild. # Copyright (C) 2022 THE firebuild'S COPYRIGHT HOLDER # This file is distributed under the same license as the firebuild package. # Paulo Henrique de Lima Santana (phls) , 2023. # msgid "" msgstr "" "Project-Id-Version: firebuild_0.2.10-1\n" "Report-Msgid-Bugs-To: firebuild@packages.debian.org\n" "POT-Creation-Date: 2022-11-14 17:54+0100\n" "PO-Revision-Date: 2023-02-14 19:20-0300\n" "Last-Translator: Paulo Henrique de Lima Santana (phls) \n" "Language-Team: Brazilian Portuguese \n" "Language: pt_BR\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n > 1)\n" "X-Generator: Gtranslator 42.0\n" #. Type: boolean #. Description #: ../templates:1001 msgid "Do you accept the terms of the Firebuild license?" msgstr "Você aceita os termos da licença do Firebuild?" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Firebuild users are required to acknowledge and accept the license. Please " "find it below. If you accept this license, the package installation will " "continue. If you refuse it, it will be interrupted." msgstr "" "Os usuários do Firebuild são obrigados a reconhecer e aceitar a licença. Por " "favor, leia-a abaixo. Se você aceitar esta licença, a instalação do pacote " "continuará. Se recusar, ela será interrompida." #. Type: boolean #. Description #: ../templates:1001 msgid "Firebuild is free for personal use or commercial trial." msgstr "O Firebuild é livre para uso pessoal ou para teste comercial." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Non-trial commercial use requires licenses available from https://firebuild." "com." msgstr "" "O uso comercial não experimental requer licenças disponíveis em https://" "firebuild.com." #. Type: boolean #. Description #: ../templates:1001 msgid "" "Modification and redistribution are permitted, but commercial use of " "derivative works is subject to the same requirements of this license." msgstr "" "A modificação e a redistribuição são permitidas, mas o uso comercial de " "trabalhos derivados está sujeito aos mesmos requisitos desta licença." #. Type: boolean #. Description #: ../templates:1001 msgid "" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR " "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE " "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER " "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING " "FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS " "IN THE SOFTWARE." msgstr "" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR " "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE " "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER " "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING " "FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS " "IN THE SOFTWARE." #. Type: error #. Description #: ../templates:2001 msgid "The license has been refused" msgstr "A licença foi recusada" #. Type: error #. Description #: ../templates:2001 msgid "" "Please remove the firebuild packages or reinstall the firebuild package (e." "g. via apt-get install --reinstall firebuild) to get prompted to accept the " "license again." msgstr "" "Por favor remova os pacotes firebuild ou reinstale o pacote firebuild (por " "exemplo, via apt-get install --reinstall firebuild) para que você seja " "solicitado a aceitar a licença novamente." firebuild-0.8.2/debian/po/templates.pot000066400000000000000000000045561447164520700200740ustar00rootroot00000000000000# SOME DESCRIPTIVE TITLE. # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the firebuild package. # FIRST AUTHOR , YEAR. # #, fuzzy msgid "" msgstr "" "Project-Id-Version: firebuild\n" "Report-Msgid-Bugs-To: firebuild@packages.debian.org\n" "POT-Creation-Date: 2022-11-14 17:54+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=CHARSET\n" "Content-Transfer-Encoding: 8bit\n" #. Type: boolean #. Description #: ../templates:1001 msgid "Do you accept the terms of the Firebuild license?" msgstr "" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Firebuild users are required to acknowledge and accept the license. Please " "find it below. If you accept this license, the package installation will " "continue. If you refuse it, it will be interrupted." msgstr "" #. Type: boolean #. Description #: ../templates:1001 msgid "Firebuild is free for personal use or commercial trial." msgstr "" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Non-trial commercial use requires licenses available from https://firebuild." "com." msgstr "" #. Type: boolean #. Description #: ../templates:1001 msgid "" "Modification and redistribution are permitted, but commercial use of " "derivative works is subject to the same requirements of this license." msgstr "" #. Type: boolean #. Description #: ../templates:1001 msgid "" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR " "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE " "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER " "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING " "FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS " "IN THE SOFTWARE." msgstr "" #. Type: error #. Description #: ../templates:2001 msgid "The license has been refused" msgstr "" #. Type: error #. Description #: ../templates:2001 msgid "" "Please remove the firebuild packages or reinstall the firebuild package (e." "g. via apt-get install --reinstall firebuild) to get prompted to accept the " "license again." msgstr "" firebuild-0.8.2/debian/postinst000066400000000000000000000006771447164520700165420ustar00rootroot00000000000000#!/bin/sh set -e . /usr/share/debconf/confmodule case "$1" in configure) db_get firebuild/license-accepted if [ "$RET" != "true" ]; then db_input critical firebuild/license-refused || true db_go exit 1 fi ;; abort-upgrade|abort-remove|abort-deconfigure) ;; *) echo "postinst called with unknown argument \`$1'" >&2 exit 1 ;; esac #DEBHELPER# firebuild-0.8.2/debian/rules000077500000000000000000000026171447164520700160100ustar00rootroot00000000000000#!/usr/bin/make -f # Use GCC 10 instead of earlier versions for better c++20 support ifeq ($(shell $(CXX) --version | head -n1 | cut -d\ -f4 | xargs dpkg --compare-versions 10 lt || echo buggy-cxx-20),buggy-cxx-20) export CC = gcc-10 export CXX = g++-10 endif # Clang does not support -ffat-lto-objects and this breaks check_c_compiler_flag() tests in cmake ifeq ($(CC),clang) export DEB_CFLAGS_MAINT_STRIP = -ffat-lto-objects export DEB_CPPFLAGS_MAINT_STRIP = -ffat-lto-objects export DEB_CXXFLAGS_MAINT_STRIP = -ffat-lto-objects export DEB_LDLAGS_MAINT_STRIP = -ffat-lto-objects endif VER_GLIBC := $(shell dpkg -s libc6 | grep ^Version: | cut -f2 -d' ' | cut -f1 -d '-') %: dh $@ ifneq (,$(filter $(distrelease),bionic)) # Bionic's jemalloc does not ship pkgconfig files needed to detect it. # TODO check performance with and without jemalloc on Bionic and maybe change # detection to work without pkgconfig override_dh_auto_configure: dh_auto_configure -- -DJEMALLOC_LIBRARIES=-ljemalloc endif # dwz does not support dwarf-5 binaries emitted by clang # https://sourceware.org/bugzilla/show_bug.cgi?id=28985 override_dh_dwz: ifneq ($(CC),clang) dh_dwz endif override_dh_auto_test: ifeq ($(filter nocheck,$(DEB_BUILD_OPTIONS)),) make -C obj-* check endif override_dh_gencontrol: dh_gencontrol -- '-Vglibc:Depends=libc6 (>= '$(VER_GLIBC)')' override_dh_installchangelogs: dh_installchangelogs -k ChangeLog firebuild-0.8.2/debian/source/000077500000000000000000000000001447164520700162225ustar00rootroot00000000000000firebuild-0.8.2/debian/source/format000066400000000000000000000000151447164520700174310ustar00rootroot000000000000003.0 (native) firebuild-0.8.2/debian/templates000066400000000000000000000025111447164520700166420ustar00rootroot00000000000000Template: firebuild/license-accepted Type: boolean Default: false _Description: Do you accept the terms of the Firebuild license? Firebuild users are required to acknowledge and accept the license. Please find it below. If you accept this license, the package installation will continue. If you refuse it, it will be interrupted. . Firebuild is free for personal use or commercial trial. . Non-trial commercial use requires licenses available from https://firebuild.com. . Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license. . THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Template: firebuild/license-refused Type: error _Description: The license has been refused Please remove the firebuild packages or reinstall the firebuild package (e.g. via apt-get install --reinstall firebuild) to get prompted to accept the license again. firebuild-0.8.2/debian/tests/000077500000000000000000000000001447164520700160645ustar00rootroot00000000000000firebuild-0.8.2/debian/tests/build-time-tests-with-installed-firebuild000066400000000000000000000003521447164520700260730ustar00rootroot00000000000000#!/bin/sh set -e echo debconf firebuild/license-accepted select true | debconf-set-selections apt-get -yqq install firebuild firebuild --version cmake . make check-bins env -C test/ TEST_INSTALLED_FIREBUILD=1 bats integration.bats firebuild-0.8.2/debian/tests/control000066400000000000000000000003141447164520700174650ustar00rootroot00000000000000Tests: recompile-bash Depends: bash, bc, debconf, time Restrictions: needs-root, allow-stderr Tests: build-time-tests-with-installed-firebuild Depends: @builddeps@ Restrictions: needs-root, allow-stderr firebuild-0.8.2/debian/tests/recompile-bash000066400000000000000000000020021447164520700206730ustar00rootroot00000000000000#!/bin/sh set -e echo debconf firebuild/license-accepted select true | debconf-set-selections apt-get -yqq install firebuild firebuild --version for i in 1 2; do rm -rf bash-* apt-get source bash 2>&1 apt-get -yqq build-dep bash (cd bash-* && /usr/bin/time -f "real${i}=%e\nuser${i}=%U\nsys${i}=%S" -a --output=../time.log firebuild -s -- sh -c "./configure && make -j4") done echo "Build times:" cat time.log # ./configure is harder to accelerate, but the second run should still be at least 80% faster CPU_TIME_PERCENT2=$( (cat time.log ; echo "(user2+sys2)*100/(user1+sys1)") | bc) echo "CPU time of the second build was ${CPU_TIME_PERCENT2}% of the first build" case $(dpkg-architecture -q DEB_HOST_ARCH) in armel) # armel results typically just below 20%, while the hit rate is similar to other architectures' [ $CPU_TIME_PERCENT2 -lt 30 ] ;; *) # 2nd builds are typically just below 10% of the first builds [ $CPU_TIME_PERCENT2 -lt 20 ] ;; esac firebuild-0.8.2/debian/watch000066400000000000000000000002601447164520700157510ustar00rootroot00000000000000version=4 opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*@ARCHIVE_EXT@)%@PACKAGE@-$1%" \ https://github.com/firebuild/firebuild/tags \ (?:.*?/)?v?@ANY_VERSION@@ARCHIVE_EXT@ firebuild-0.8.2/doc/000077500000000000000000000000001447164520700142455ustar00rootroot00000000000000firebuild-0.8.2/doc/README.developer000066400000000000000000000011431447164520700171100ustar00rootroot00000000000000Test wether Firebuild intercepts all syscalls: firebuild -i strace -o/tmp/ls.trace ls awk 'BEGINFILE {SKIP=0} /FIREBUILD.*intercept-begin/ {SKIP=1; next} /FIREBUILD.*intercept-end/ {SKIP=0; next} /execve\(/ {SKIP=0; print FILENAME ": " $0; next} {if (FNR == 1) {SKIP=1; next;} if (SKIP == 0 ) { print FILENAME ": " $0;} else {next;}}' /tmp/ls.trace The source code follows Google C++ Style Guide (more or less :-)) Emacs code indentation style: (require 'google-c-style) (add-hook 'c-mode-common-hook 'google-set-c-style) (add-hook 'c-mode-common-hook 'google-make-newline-indent) firebuild-0.8.2/doc/parallel-make-acceleration.svg000066400000000000000000010106321447164520700221270ustar00rootroot00000000000000 Collapse Expand Show details Collapse Collapse Collapse Expand Collapse Expand Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Show details Expand Show details Show details Show details Collapse Show details Show details Show details Collapse Show details Show details Show details Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Collapse Show details Show details Show details Show details Collapse Show details Show details Show details Expand Show details Show details Show details Collapse Show details Show details Show details Expand Show details Show details Show details Collapse Show details Show details Collapse Show details Show details Show details firebuild-0.8.2/doc/static-structure-diagram.dia000066400000000000000000001615451447164520700216670ustar00rootroot00000000000000 #A4# #Process# ## #test# #const type# #process_type# ## ## #process_state# #int# #FB_PROC_STARTED# ## #fb_pid# #int # ## ## #pid# #int # ## ## #ppid# #int # ## ## #libs# #set<string> # ## ## #exit_status# #int # ## ## #utime_u# #int# ## ## #stime_u# #int# ## ## #children# #vector<Process*># ## ## #exec_child# #Process*# ## ## #resource_usage# ## #void # ## #utime_u# #int# ## ## #stime_u# #int# ## ## #Process# ## ## ## #pid# #int# ## ## #ppid# #int# ## ## ## ## #ExecedProcess# ## ## #const type# #process_type# #FB_PROC_EXEC_STARTED# ## #exec_parent# #Process*# ## ## #cwd# #string # ## ## #args# #vector<string> # ## ## #env_vars# #set<string> # ## ## #executable# #string # ## ## #ExecedProcess# ## ## ## #scpq# #firebuild::msg::ShortCutProcessQuery&# ## ## #ForkedProcess# ## ## #const type# #process_type# #FB_PROC_FORK_STARTED# ## #fork_parent# #Process*# ## ## #ForkedProcess# ## ## ## #scpq# #firebuild::msg::ForkChild&# ## ## ## ## #ProcessTree# ## ## #root# #Process*# ## ## #sock2proc# #unordered_map<int, Process*># ## ## #pid2proc# #unordered_map<int, Process*># ## ## #insert# ## #void# ## #p# #Process&# ## ## #socket# #int# ## ## #exit# ## #void# # # #p# #Process&# ## ## #socket# #int# ## ## ## ## ## ## ## ## ## ## ## ## firebuild-0.8.2/etc/000077500000000000000000000000001447164520700142535ustar00rootroot00000000000000firebuild-0.8.2/etc/firebuild.conf000066400000000000000000000176701447164520700171020ustar00rootroot00000000000000// Default configuration file for Firebuild version = 1.0; // enviromnent variables passed to the build command env_vars = { // the following environment variables are passed to the build command unchanged pass_through = [ "HOME", "PATH", "SHELL", "PWD", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"]; // These env vars are skipped when computing an intercepted command's fingerprint. fingerprint_skip = [ "MAKE_TERMOUT", "MAKE_TERMERR" ]; // The folloving environment variables are pre-set to the values configured below. // Note that FB_SOCKET, FB_READ_ONLY_LOCATIONS, FB_IGNORE_LOCATIONS, FB_JOBSERVER_USERS // and LD_PRELOAD (DYLD_INSERT_LIBRARIES and DYLD_FORCE_FLAT_NAMESPACE on OS X) // are also set by firebuild internally. // The variables should be in "NAME=value" format. preset = [ // Make ccache call the original compiler. This allows firebuild rather than ccache // cache the build results. To enable ccache's caching in firebuild-intercepted builds // don't set CCACHE_DISABLE=1 and add "ccache" to the processes.dont_shortcut list. "CCACHE_DISABLE=1" ]; }; processes = { // Shortcut only the listed processes (together with their descendants). // This can keep the cache size really small and allows limiting shortcutting to // the known to be well-behaving processes. // Note that the descendants are not shortcut on their own, only if they are listed here, too. shortcut_allow_list = []; // Processes that should always be executed. This means that their ancestors // can't be cached and shortcut either. dont_shortcut = [ // orchestrating tools that compare timestamps "make", "ninja", // Sleep is used sometimes to provide periodic status of the build, which breaks if sleep is shortcut // and finish very fast. By default the sleep command is shortcut. Disable shortcutting if your build // system relies on sleep's original behavior. // "sleep", // Part of the file information returned by stat() e.g. inode number is ignored // for shortcutting purposes and is not stored in the cache. As a result the stat command // would very often provide invalid output when shortcut thus it is safer to just not shortcut it. "stat", // The ls command can print inodes and other ignored file information, too. "ls" ]; // Processes that should not be intercepted. This means that they and their ancestors // and children can't be cached and shortcut either. dont_intercept = [ // starts a daemon and causes deadlock with interception enabled "fakeroot", // Similar build accelerator which is unlikely to be shortcut efficiently. // Interception is enabled because firebuild also sets CCACHE_DISABLE=1 by default thus // ccache will just call the original compiler. // "ccache", // hangs, also has its own caching framework "gradle", // Go has its own caching framework "go" ]; // Processes that we could cache and shortcut, but prefer not to (for example // because they are fast enough). // This has no effect on potentially caching and shortcutting their ancestors. skip_cache = [ // The default shell. Not caching it may improve performance of builds running a lot of // very quick shell scripts. // "sh", // coreutils "arch", "basename", "cat", "chgrp", "chmod", "chown", "cp", "cut", "dd", "dir", "dirname", "expr", "head", "install", "link", "ln", "mkdir", "mv", "readlink", "realpath", "rm", "rmdir", "seq", "tail", "touch", "tr", "unlink", // usually shell builtins, but not always "[", "echo", "false", "printf", "pwd", "test", "true", // other standard utils "egrep", "fgrep", "grep", "rgrep", "sed" ]; // Shells to cache instead of child when the shell just executes the child. // For example "foo ..." is not cached, when its parent /bin/sh -c "foo ..." can be cached instead. shells = [ "/bin/bash", "/bin/dash", "/bin/sh", "bash", "dash", "sh" ]; // Programs that use Make's or a similar jobserver. // Firebuild ignores the below programs' communication over jobserver fds to allow shortcutting // them. // Only the file name without the directory name can be listed here, // for example list "rustc" instead of "/usr/bin/rustc". jobserver_users = [ "cargo", // Make is not shortcut by default, thus its communication over the jobserver fds does not make // a difference. // "make", "rustc" ]; }; // Ignore operations affecting these files, directories, or any // location under these directories. ignore_locations = [ "/dev/full", "/dev/null", "/dev/random", "/dev/tty", "/dev/urandom", "/dev/zero", "/proc/cpuinfo", "/proc/filesystems", "/proc/net", "/proc/meminfo", "/proc/self", "/proc/stat", "/sys/devices", "/sys/fs/cgroup" ]; // These files, directores, or any location under these directories assumed to be // read-only files not changing while firebuild is running. Opening a file with // those prefixes does not delay program execution while the file hash is saved. read_only_locations = [ "/bin", "/etc", "/lib", "/lib32", "/libx32", "/lib64", "/opt", "/proc/sys", "/sbin", "/snap", "/usr" ]; // Only cache results of processes consuming more CPU time (system + user) in seconds than this value. // The CPU time includes all children's CPU time. If a process cached its ancestors can still be cached // if their cumulative CPU time exceeds this limit. min_cpu_time = 0.000; // Quirks are adjustments of firebuild's behavior typically to allow shortcutting processes // in cases which are considered to be safe most of the time. quirks = [ // Many processes (such as gcc) query the time with clock_gettime() and with similar calls but // don't use it in the output (unless the __TIME__ or __DATE__ preprocessor macros are used). // This quirk ignores all time queries for all processes. "ignore-time-queries", // When processes open a directory with opendir() or with a similar call the contents are hashed and // the processes are shorcut if the directory contents match the contents stored in the cache entry. // /tmp or TMPDIR OTOH is used by many processes outside of the intercepted builds, thus the // contents will most likely be different with each run making any process opening the temporary // directory not shortcutable almost every time they run. // This quirk ignores /tmp's directory listing for shortcutting purposes. // (Files used in /tmp are still tracked.) "ignore-tmp-listing", // lto-wrapper starts make internally which is not shortcutable in general. // This quirk allows shortcutting make and touch when it is run by lto-wrapper. // Since the internal make is started in /tmp the ignore-tmp-listing quirk needs to // be enabled as well to shortcut lto-wrapper. "lto-wrapper", // dot (via libfontconfig) and other commands call (f)stafs() which don't impact the output. // Ignore those calls to allow shortcutting such processes. "ignore-statfs", // Guess if a command parameter is a file and include the hash of the file in the process fingerprint. // This typically speeds up shortcutting, but may prevent shortcutting when the files passed to a command // as parameters already exist and their content would not be read by the command. "guess-file-params" ]; // The cache can contain multiple candidate entres for shortcutting a command. // Run the intercepted command after failing to shortcut it using a preset number of candidates. shortcut_tries = 20; // Maximum size of the files stored in the cache, in GB. // This is not a hard limit. When the cache size at the end of a build exceeds this limit // garbage collection is started to remove the unusable cache entries first, then the last // recently used ones until the cache size is decreased to at least 20% under the limit. max_cache_size = 5.0 // Maximum size of one cache entry in MB. // This includes the size of all outputs to be replayed when using the cache entry for shortcutting // and the entry's size. max_entry_size = 250.0 firebuild-0.8.2/man/000077500000000000000000000000001447164520700142535ustar00rootroot00000000000000firebuild-0.8.2/man/.gitignore000066400000000000000000000000251447164520700162400ustar00rootroot00000000000000firebuild.1.xml man1 firebuild-0.8.2/man/CMakeLists.txt000066400000000000000000000013331447164520700170130ustar00rootroot00000000000000find_program(XSLTPROC xsltproc) if(XSLTPROC) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/firebuild.1.xml.in ${CMAKE_CURRENT_BINARY_DIR}/firebuild.1.xml @ONLY) add_custom_target(man ${XSLTPROC} --nonet --param "man.authors.section.enabled" "0" --param "man.output.in.separate.dir" "1" http://docbook.sourceforge.net/release/xsl/current/manpages/profile-docbook.xsl ${CMAKE_CURRENT_BINARY_DIR}/firebuild.1.xml DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/firebuild.1.xml WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMENT "Generating man pages with xsltproc" VERBATIM) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/man1/firebuild.1" DESTINATION "${CMAKE_INSTALL_MANDIR}/man1") add_dependencies(firebuild man) endif(XSLTPROC) firebuild-0.8.2/man/firebuild.1.xml.in000066400000000000000000000150211447164520700175050ustar00rootroot00000000000000 Balint Reczey Creation, 2022 firebuild 1 User Commands firebuild @FIREBUILD_VERSION@ firebuild accelerate build firebuild OPTIONS BUILD COMMAND DESCRIPTION Execute BUILD COMMAND with Firebuild instrumentation OPTIONS Mandatory arguments to long options are mandatory for short options too. , use FILE as configuration file , change directory before running the command , comma separated list of debug flags, to get a list. , Comma separated list of commands to debug. Debug messages related to processes which are not listed are suppressed. , Garbage collect the cache. Keeps debugging entries related to kept files when used together with . , Generate a report on the build command execution. The report's filename can be specified (firebuild-build-report.html by default). , show this help , Add or replace a scalar in the config , Clear an array in the config , Append to an array of scalars in the config , Remove from an array of scalars in the config , Show cache hit statistics. When used together with BUILD COMMAND, the statistics of the current run are shown. Without a BUILD COMMAND it shows the cumulative statistics of all prior runs (since the creation of the used cache directory). , Zero cache hit statistics before performing any other action. , Perform open("/FIREBUILD debug message", 0) calls to let users find unintercepted calls using strace or ltrace. This works in debug builds only. output version information and exit EXIT STATUS Exit status of the BUILD COMMAND or 1 in case of failure in firebuild. firebuild-0.8.2/src/000077500000000000000000000000001447164520700142675ustar00rootroot00000000000000firebuild-0.8.2/src/.gitignore000066400000000000000000000000331447164520700162530ustar00rootroot00000000000000fbbcomm.? fbbcomm_decode.c firebuild-0.8.2/src/CMakeLists.txt000066400000000000000000000020441447164520700170270ustar00rootroot00000000000000include_directories("${CMAKE_CURRENT_SOURCE_DIR}") include_directories("${CMAKE_CURRENT_BINARY_DIR}") set_source_files_properties(common/debug_sysflags.c PROPERTIES COMPILE_FLAGS "-fPIC ${FAT_ARCH_C_FLAGS}") set_source_files_properties(common/firebuild_common.c PROPERTIES COMPILE_FLAGS "-fPIC ${FAT_ARCH_C_FLAGS}") set_source_files_properties(fbbcomm.c PROPERTIES COMPILE_FLAGS "-fPIC ${FAT_ARCH_C_FLAGS}") add_custom_command( OUTPUT fbbcomm.c fbbcomm.cc fbbcomm.h fbbcomm_decode.c DEPENDS common/fbbcomm.def common/fbb/generate_fbb common/fbb/tpl.c common/fbb/tpl.h WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/common COMMAND ./fbb/generate_fbb fbbcomm ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating fbbcomm files") add_custom_target(fbbcomm_gen_files ALL DEPENDS fbbcomm.c fbbcomm.cc fbbcomm.h fbbcomm_decode.c) add_library(common_objs OBJECT common/firebuild_common.c common/debug_sysflags.c) add_library(fbbcomm_c OBJECT fbbcomm.c) add_library(fbbcomm_cc OBJECT fbbcomm.cc) add_subdirectory(interceptor) add_subdirectory(firebuild) firebuild-0.8.2/src/CPPLINT.cfg000066400000000000000000000000731447164520700160610ustar00rootroot00000000000000set noparent root=. linelength=100 exclude_files=fbbcomm..*firebuild-0.8.2/src/common/000077500000000000000000000000001447164520700155575ustar00rootroot00000000000000firebuild-0.8.2/src/common/CPPLINT.cfg000066400000000000000000000000341447164520700173460ustar00rootroot00000000000000filter=-readability/casting firebuild-0.8.2/src/common/README_MSG_FRAME.txt000066400000000000000000000032641447164520700207020ustar00rootroot00000000000000Firebuild Message Frame Format ============================== Protocol -------- The interceptor<->supervisor communication protocol is a mixture of 2 different types of messages: - Empty - FBBCOMM (FirebuildBuffers) Empty messages aren't literally empty, their header contains an ack id. They're only used in the supervisor->interceptor direction, for acking. The transfer protocol is: ┌─────────────────────┐ ┐ │ msg_size (uint32_t) │ │ │ ack_id (uint16_t) │ ├ msg_header │ fd_count (uint16_t) │ │ ├─────────────────────┼╌╌╌╌╌┐ ┐ ┘ ┆ FBBCOMM ┆ anc ┆ ├ payload ("msg_size" bytes) └╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┴╌╌╌╌╌┘ ┘ A nonzero "ack_id" indicates that the interceptor wishes to receive a response from the supervisor with the same ack_id. A value of 0 means that no response is expected, or ack_id is otherwise irrelevant. "msg_size" is the FBBCOMM payload's length, i.e. excluding the header and the ancillary data. An empty message ends here ("msg_size" is 0). A FirebuildBuffers message continues with the serialized FBBCOMM message. Refer to fbb/README_FBB.txt for further details. A message might also have file descriptors attached as ancillary data; see SCM_RIGHTS in cmsg(3) and unix(7). In that case the header and the payload are sent as separate steps and the ancillary data is attached to the payload (that is, to its first byte). The number of such file descriptors is placed in the header as "fd_count". firebuild-0.8.2/src/common/cstring_view.h000066400000000000000000000026521447164520700204400ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef COMMON_CSTRING_VIEW_H_ #define COMMON_CSTRING_VIEW_H_ /** * A lightweight structure containing a pointer to a string and the string's length. * * Intended to be used in a way that the string is a plain C '\0'-terminated string, * the length value doesn't include the trailing zero. * * Could be replaced by std::cstring_view, had this proposal not been rejected: * - http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1402r0.pdf * - https://github.com/cplusplus/papers/issues/189 */ typedef struct { const char *c_str; uint32_t length; } cstring_view; #endif // COMMON_CSTRING_VIEW_H_ firebuild-0.8.2/src/common/debug_sysflags.c000066400000000000000000000445341447164520700207360ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "common/debug_sysflags.h" #include #include #include #include #include #include #include "common/platform.h" #ifdef __cplusplus extern "C" { #endif /* Convenience macros to debug-print a bitfield. */ #define DEBUG_BITMAP_START(f, var) \ { \ const char *sep = ""; #define DEBUG_BITMAP_FLAG(f, var, flag) \ if (var & flag) { \ fprintf(f, "%s%s", sep, #flag); \ var &= ~flag; \ sep = "|"; \ } #define DEBUG_BITMAP_END_OCT(f, flags) \ if (flags) { \ /* Remaining unrecognized flags */ \ fprintf(f, "%s0%o", sep, flags); \ } \ } #define DEBUG_BITMAP_END_HEX(f, flags) \ if (flags) { \ /* Remaining unrecognized flags */ \ fprintf(f, "%s0x%X", sep, flags); \ } \ } /* Convenience macros to debug-print a variable that is supposed to have one of several values. */ #define DEBUG_VALUE_START(f, var) \ switch (var) { \ /* NOLINT(whitespace/blank_line) */ #define DEBUG_VALUE_VALUE(f, var, value) \ case value: \ fprintf(f, #value); \ break; #define DEBUG_VALUE_END_OCT(f, var) \ default: \ fprintf(f, "0%o", var); \ } #define DEBUG_VALUE_END_DEC(f, var) \ default: \ fprintf(f, "%d", var); \ } #define DEBUG_VALUE_END_HEX(f, var) \ default: \ fprintf(f, "0x%X", var); \ } /** * Debug-print O_* flags, as usually seen in the 'flags' parameter of dup3(), open(), pipe2(), * posix_spawn_file_actions_addopen() etc. calls. */ void debug_open_flags(FILE *f, int flags) { DEBUG_BITMAP_START(f, flags) int accmode = flags & O_ACCMODE; DEBUG_VALUE_START(f, accmode) DEBUG_VALUE_VALUE(f, accmode, O_RDONLY); DEBUG_VALUE_VALUE(f, accmode, O_WRONLY); DEBUG_VALUE_VALUE(f, accmode, O_RDWR); DEBUG_VALUE_END_OCT(f, accmode) flags &= ~O_ACCMODE; sep = "|"; DEBUG_BITMAP_FLAG(f, flags, O_APPEND) DEBUG_BITMAP_FLAG(f, flags, O_ASYNC) DEBUG_BITMAP_FLAG(f, flags, O_CLOEXEC) DEBUG_BITMAP_FLAG(f, flags, O_CREAT) #ifdef O_DIRECT DEBUG_BITMAP_FLAG(f, flags, O_DIRECT) #endif DEBUG_BITMAP_FLAG(f, flags, O_DIRECTORY) DEBUG_BITMAP_FLAG(f, flags, O_DSYNC) DEBUG_BITMAP_FLAG(f, flags, O_EXCL) #ifdef O_LARGEFILE DEBUG_BITMAP_FLAG(f, flags, O_LARGEFILE) #endif #ifdef O_NOATIME DEBUG_BITMAP_FLAG(f, flags, O_NOATIME) #endif DEBUG_BITMAP_FLAG(f, flags, O_NOCTTY) DEBUG_BITMAP_FLAG(f, flags, O_NOFOLLOW) DEBUG_BITMAP_FLAG(f, flags, O_NONBLOCK) #ifdef O_PATH DEBUG_BITMAP_FLAG(f, flags, O_PATH) #endif DEBUG_BITMAP_FLAG(f, flags, O_SYNC) #ifdef O_TMPFILE DEBUG_BITMAP_FLAG(f, flags, O_TMPFILE) #endif DEBUG_BITMAP_FLAG(f, flags, O_TRUNC) DEBUG_BITMAP_END_HEX(f, flags) } /** * Debug-print AT_* flags, as usually seen in the 'flags' parameter of execveat(), faccessat(), * fchmodat(), fchownat(), fstatat(), linkat(), statx(), unlinkat(), utimensat() etc. calls. */ void debug_at_flags(FILE *f, int flags) { DEBUG_BITMAP_START(f, flags) /* AT_EACCESS has different semantics but the same value as AT_REMOVEDIR. * FIXME Print whichever semantically matches the current context. */ /* DEBUG_BITMAP_FLAG(f, flags, AT_EACCESS) */ #ifdef AT_EMPTY_PATH DEBUG_BITMAP_FLAG(f, flags, AT_EMPTY_PATH) #endif #ifdef AT_NO_AUTOMOUNT DEBUG_BITMAP_FLAG(f, flags, AT_NO_AUTOMOUNT) #endif #ifdef AT_RECURSIVE DEBUG_BITMAP_FLAG(f, flags, AT_RECURSIVE) #endif DEBUG_BITMAP_FLAG(f, flags, AT_REMOVEDIR) #ifdef AT_STATX_DONT_SYNC DEBUG_BITMAP_FLAG(f, flags, AT_STATX_DONT_SYNC) #endif #ifdef AT_STATX_FORCE_SYNC DEBUG_BITMAP_FLAG(f, flags, AT_STATX_FORCE_SYNC) #endif #ifdef AT_STATX_SYNC_AS_STAT DEBUG_BITMAP_FLAG(f, flags, AT_STATX_SYNC_AS_STAT) #endif #ifdef AT_STATX_SYNC_TYPE DEBUG_BITMAP_FLAG(f, flags, AT_STATX_SYNC_TYPE) #endif DEBUG_BITMAP_FLAG(f, flags, AT_SYMLINK_FOLLOW) DEBUG_BITMAP_FLAG(f, flags, AT_SYMLINK_NOFOLLOW) DEBUG_BITMAP_END_HEX(f, flags) } /** * Debug-print the 'cmd' parameter of an fcntl() call. */ void debug_fcntl_cmd(FILE *f, int cmd) { DEBUG_VALUE_START(f, cmd) DEBUG_VALUE_VALUE(f, cmd, F_DUPFD) DEBUG_VALUE_VALUE(f, cmd, F_DUPFD_CLOEXEC) DEBUG_VALUE_VALUE(f, cmd, F_GETFD) DEBUG_VALUE_VALUE(f, cmd, F_SETFD) DEBUG_VALUE_VALUE(f, cmd, F_GETFL) DEBUG_VALUE_VALUE(f, cmd, F_SETFL) DEBUG_VALUE_VALUE(f, cmd, F_GETLK) DEBUG_VALUE_VALUE(f, cmd, F_SETLK) DEBUG_VALUE_VALUE(f, cmd, F_SETLKW) DEBUG_VALUE_VALUE(f, cmd, F_GETOWN) DEBUG_VALUE_VALUE(f, cmd, F_SETOWN) #ifdef F_GETOWN_EX DEBUG_VALUE_VALUE(f, cmd, F_GETOWN_EX) #endif #ifdef F_SETOWN_EX DEBUG_VALUE_VALUE(f, cmd, F_SETOWN_EX) #endif #ifdef F_GETSIG DEBUG_VALUE_VALUE(f, cmd, F_GETSIG) #endif #ifdef F_SETSIG DEBUG_VALUE_VALUE(f, cmd, F_SETSIG) #endif #ifdef F_GETLEASE DEBUG_VALUE_VALUE(f, cmd, F_GETLEASE) #endif #ifdef F_SETLEASE DEBUG_VALUE_VALUE(f, cmd, F_SETLEASE) #endif #ifdef F_NOTIFY DEBUG_VALUE_VALUE(f, cmd, F_NOTIFY) #endif #ifdef F_GETPIPE_SZ DEBUG_VALUE_VALUE(f, cmd, F_GETPIPE_SZ) #endif #ifdef F_SETPIPE_SZ DEBUG_VALUE_VALUE(f, cmd, F_SETPIPE_SZ) #endif #ifdef F_ADD_SEALS DEBUG_VALUE_VALUE(f, cmd, F_ADD_SEALS) #endif #ifdef F_GET_SEALS DEBUG_VALUE_VALUE(f, cmd, F_GET_SEALS) #endif #ifdef F_GET_RW_HINT DEBUG_VALUE_VALUE(f, cmd, F_GET_RW_HINT) #endif #ifdef F_SET_RW_HINT DEBUG_VALUE_VALUE(f, cmd, F_SET_RW_HINT) #endif #ifdef F_GET_FILE_RW_HINT DEBUG_VALUE_VALUE(f, cmd, F_GET_FILE_RW_HINT) #endif #ifdef F_SET_FILE_RW_HINT DEBUG_VALUE_VALUE(f, cmd, F_SET_FILE_RW_HINT) #endif DEBUG_VALUE_END_DEC(f, cmd) } /** * Debug-print fcntl()'s 'arg'parameter or return value. The debugging format depends on 'cmd'. */ void debug_fcntl_arg_or_ret(FILE *f, int cmd, int arg_or_ret) { switch (cmd) { case F_GETFD: case F_SETFD: if (arg_or_ret) { DEBUG_BITMAP_START(f, arg_or_ret) DEBUG_BITMAP_FLAG(f, arg_or_ret, FD_CLOEXEC) DEBUG_BITMAP_END_HEX(f, arg_or_ret) } else { fprintf(f, "0"); } break; case F_GETFL: case F_SETFL: debug_open_flags(f, arg_or_ret); break; default: fprintf(f, "%d", arg_or_ret); } } /** * Debug-print an error number. */ void debug_error_no(FILE *f, int error_no) { // FIXME: glibc 2.32 adds strerrorname_np(), switch to that one day. DEBUG_VALUE_START(f, error_no) DEBUG_VALUE_VALUE(f, error_no, E2BIG) DEBUG_VALUE_VALUE(f, error_no, EACCES) DEBUG_VALUE_VALUE(f, error_no, EADDRINUSE) DEBUG_VALUE_VALUE(f, error_no, EADDRNOTAVAIL) #ifdef EADV DEBUG_VALUE_VALUE(f, error_no, EADV) #endif DEBUG_VALUE_VALUE(f, error_no, EAFNOSUPPORT) DEBUG_VALUE_VALUE(f, error_no, EAGAIN) DEBUG_VALUE_VALUE(f, error_no, EALREADY) #ifdef EBADE DEBUG_VALUE_VALUE(f, error_no, EBADE) #endif DEBUG_VALUE_VALUE(f, error_no, EBADF) #ifdef EBADFD DEBUG_VALUE_VALUE(f, error_no, EBADFD) #endif DEBUG_VALUE_VALUE(f, error_no, EBADMSG) #ifdef EBADR DEBUG_VALUE_VALUE(f, error_no, EBADR) #endif #ifdef EBADRQC DEBUG_VALUE_VALUE(f, error_no, EBADRQC) #endif #ifdef EBADSLT DEBUG_VALUE_VALUE(f, error_no, EBADSLT) #endif #ifdef EBFONT DEBUG_VALUE_VALUE(f, error_no, EBFONT) #endif DEBUG_VALUE_VALUE(f, error_no, EBUSY) DEBUG_VALUE_VALUE(f, error_no, ECANCELED) DEBUG_VALUE_VALUE(f, error_no, ECHILD) #ifdef ECHRNG DEBUG_VALUE_VALUE(f, error_no, ECHRNG) #endif #ifdef ECOMM DEBUG_VALUE_VALUE(f, error_no, ECOMM) #endif DEBUG_VALUE_VALUE(f, error_no, ECONNABORTED) DEBUG_VALUE_VALUE(f, error_no, ECONNREFUSED) DEBUG_VALUE_VALUE(f, error_no, ECONNRESET) DEBUG_VALUE_VALUE(f, error_no, EDEADLK) /* DEBUG_VALUE_VALUE(f, error_no, EDEADLOCK) - same as EDEADLK on Linux */ DEBUG_VALUE_VALUE(f, error_no, EDESTADDRREQ) DEBUG_VALUE_VALUE(f, error_no, EDOM) #ifdef EDOTDOT DEBUG_VALUE_VALUE(f, error_no, EDOTDOT) #endif DEBUG_VALUE_VALUE(f, error_no, EDQUOT) DEBUG_VALUE_VALUE(f, error_no, EEXIST) DEBUG_VALUE_VALUE(f, error_no, EFAULT) DEBUG_VALUE_VALUE(f, error_no, EFBIG) DEBUG_VALUE_VALUE(f, error_no, EHOSTDOWN) DEBUG_VALUE_VALUE(f, error_no, EHOSTUNREACH) #ifdef EHWPOISON DEBUG_VALUE_VALUE(f, error_no, EHWPOISON) #endif DEBUG_VALUE_VALUE(f, error_no, EIDRM) DEBUG_VALUE_VALUE(f, error_no, EILSEQ) DEBUG_VALUE_VALUE(f, error_no, EINPROGRESS) DEBUG_VALUE_VALUE(f, error_no, EINTR) DEBUG_VALUE_VALUE(f, error_no, EINVAL) DEBUG_VALUE_VALUE(f, error_no, EIO) DEBUG_VALUE_VALUE(f, error_no, EISCONN) DEBUG_VALUE_VALUE(f, error_no, EISDIR) #ifdef EISNAM DEBUG_VALUE_VALUE(f, error_no, EISNAM) #endif #ifdef EKEYEXPIRED DEBUG_VALUE_VALUE(f, error_no, EKEYEXPIRED) #endif #ifdef EKEYREJECTED DEBUG_VALUE_VALUE(f, error_no, EKEYREJECTED) #endif #ifdef EKEYREVOKED DEBUG_VALUE_VALUE(f, error_no, EKEYREVOKED) #endif #ifdef EL2HLT DEBUG_VALUE_VALUE(f, error_no, EL2HLT) #endif #ifdef EL2NSYNC DEBUG_VALUE_VALUE(f, error_no, EL2NSYNC) #endif #ifdef EL3HLT DEBUG_VALUE_VALUE(f, error_no, EL3HLT) #endif #ifdef EL3RST DEBUG_VALUE_VALUE(f, error_no, EL3RST) #endif #ifdef ELIBACC DEBUG_VALUE_VALUE(f, error_no, ELIBACC) #endif #ifdef ELIBBAD DEBUG_VALUE_VALUE(f, error_no, ELIBBAD) #endif #ifdef ELIBEXEC DEBUG_VALUE_VALUE(f, error_no, ELIBEXEC) #endif #ifdef ELIBMAX DEBUG_VALUE_VALUE(f, error_no, ELIBMAX) #endif #ifdef ELIBSCN DEBUG_VALUE_VALUE(f, error_no, ELIBSCN) #endif #ifdef ELNRNG DEBUG_VALUE_VALUE(f, error_no, ELNRNG) #endif DEBUG_VALUE_VALUE(f, error_no, ELOOP) #ifdef EMEDIUMTYPE DEBUG_VALUE_VALUE(f, error_no, EMEDIUMTYPE) #endif DEBUG_VALUE_VALUE(f, error_no, EMFILE) DEBUG_VALUE_VALUE(f, error_no, EMLINK) DEBUG_VALUE_VALUE(f, error_no, EMSGSIZE) DEBUG_VALUE_VALUE(f, error_no, EMULTIHOP) DEBUG_VALUE_VALUE(f, error_no, ENAMETOOLONG) #ifdef ENAVAIL DEBUG_VALUE_VALUE(f, error_no, ENAVAIL) #endif DEBUG_VALUE_VALUE(f, error_no, ENETDOWN) DEBUG_VALUE_VALUE(f, error_no, ENETRESET) DEBUG_VALUE_VALUE(f, error_no, ENETUNREACH) DEBUG_VALUE_VALUE(f, error_no, ENFILE) #ifdef ENOANO DEBUG_VALUE_VALUE(f, error_no, ENOANO) #endif DEBUG_VALUE_VALUE(f, error_no, ENOBUFS) #ifdef ENOCSI DEBUG_VALUE_VALUE(f, error_no, ENOCSI) #endif DEBUG_VALUE_VALUE(f, error_no, ENODATA) DEBUG_VALUE_VALUE(f, error_no, ENODEV) DEBUG_VALUE_VALUE(f, error_no, ENOENT) DEBUG_VALUE_VALUE(f, error_no, ENOEXEC) #ifdef ENOKEY DEBUG_VALUE_VALUE(f, error_no, ENOKEY) #endif DEBUG_VALUE_VALUE(f, error_no, ENOLCK) DEBUG_VALUE_VALUE(f, error_no, ENOLINK) #ifdef ENOMEDIUM DEBUG_VALUE_VALUE(f, error_no, ENOMEDIUM) #endif DEBUG_VALUE_VALUE(f, error_no, ENOMEM) DEBUG_VALUE_VALUE(f, error_no, ENOMSG) #ifdef ENONET DEBUG_VALUE_VALUE(f, error_no, ENONET) #endif #ifdef ENOPKG DEBUG_VALUE_VALUE(f, error_no, ENOPKG) #endif DEBUG_VALUE_VALUE(f, error_no, ENOPROTOOPT) DEBUG_VALUE_VALUE(f, error_no, ENOSPC) DEBUG_VALUE_VALUE(f, error_no, ENOSR) DEBUG_VALUE_VALUE(f, error_no, ENOSTR) DEBUG_VALUE_VALUE(f, error_no, ENOSYS) DEBUG_VALUE_VALUE(f, error_no, ENOTBLK) DEBUG_VALUE_VALUE(f, error_no, ENOTCONN) DEBUG_VALUE_VALUE(f, error_no, ENOTDIR) DEBUG_VALUE_VALUE(f, error_no, ENOTEMPTY) #ifdef ENOTNAM DEBUG_VALUE_VALUE(f, error_no, ENOTNAM) #endif DEBUG_VALUE_VALUE(f, error_no, ENOTRECOVERABLE) DEBUG_VALUE_VALUE(f, error_no, ENOTSOCK) DEBUG_VALUE_VALUE(f, error_no, ENOTSUP) DEBUG_VALUE_VALUE(f, error_no, ENOTTY) #ifdef ENOTUNIQ DEBUG_VALUE_VALUE(f, error_no, ENOTUNIQ) #endif DEBUG_VALUE_VALUE(f, error_no, ENXIO) /* DEBUG_VALUE_VALUE(f, error_no, EOPNOTSUPP) - same as ENOTSUPP on Linux */ DEBUG_VALUE_VALUE(f, error_no, EOVERFLOW) DEBUG_VALUE_VALUE(f, error_no, EOWNERDEAD) DEBUG_VALUE_VALUE(f, error_no, EPERM) DEBUG_VALUE_VALUE(f, error_no, EPFNOSUPPORT) DEBUG_VALUE_VALUE(f, error_no, EPIPE) DEBUG_VALUE_VALUE(f, error_no, EPROTO) DEBUG_VALUE_VALUE(f, error_no, EPROTONOSUPPORT) DEBUG_VALUE_VALUE(f, error_no, EPROTOTYPE) DEBUG_VALUE_VALUE(f, error_no, ERANGE) #ifdef EREMCHG DEBUG_VALUE_VALUE(f, error_no, EREMCHG) #endif DEBUG_VALUE_VALUE(f, error_no, EREMOTE) #ifdef EREMOTEIO DEBUG_VALUE_VALUE(f, error_no, EREMOTEIO) #endif #ifdef ERESTART DEBUG_VALUE_VALUE(f, error_no, ERESTART) #endif #ifdef ERFKILL DEBUG_VALUE_VALUE(f, error_no, ERFKILL) #endif DEBUG_VALUE_VALUE(f, error_no, EROFS) DEBUG_VALUE_VALUE(f, error_no, ESHUTDOWN) DEBUG_VALUE_VALUE(f, error_no, ESOCKTNOSUPPORT) DEBUG_VALUE_VALUE(f, error_no, ESPIPE) DEBUG_VALUE_VALUE(f, error_no, ESRCH) #ifdef ESRMNT DEBUG_VALUE_VALUE(f, error_no, ESRMNT) #endif DEBUG_VALUE_VALUE(f, error_no, ESTALE) #ifdef ESTRPIPE DEBUG_VALUE_VALUE(f, error_no, ESTRPIPE) #endif DEBUG_VALUE_VALUE(f, error_no, ETIME) DEBUG_VALUE_VALUE(f, error_no, ETIMEDOUT) DEBUG_VALUE_VALUE(f, error_no, ETOOMANYREFS) DEBUG_VALUE_VALUE(f, error_no, ETXTBSY) #ifdef EUCLEAN DEBUG_VALUE_VALUE(f, error_no, EUCLEAN) #endif #ifdef EUNATCH DEBUG_VALUE_VALUE(f, error_no, EUNATCH) #endif DEBUG_VALUE_VALUE(f, error_no, EUSERS) /* DEBUG_VALUE_VALUE(f, error_no, EWOULDBLOCK) - same as EAGAIN on Linux */ DEBUG_VALUE_VALUE(f, error_no, EXDEV) #ifdef EXFULL DEBUG_VALUE_VALUE(f, error_no, EXFULL) #endif DEBUG_VALUE_END_DEC(f, error_no) } /** * Debug-print a signal number. */ void debug_signum(FILE *f, int signum) { DEBUG_VALUE_START(f, signum); DEBUG_VALUE_VALUE(f, signum, SIGHUP); DEBUG_VALUE_VALUE(f, signum, SIGINT); DEBUG_VALUE_VALUE(f, signum, SIGQUIT); DEBUG_VALUE_VALUE(f, signum, SIGILL); DEBUG_VALUE_VALUE(f, signum, SIGTRAP); DEBUG_VALUE_VALUE(f, signum, SIGABRT); DEBUG_VALUE_VALUE(f, signum, SIGBUS); DEBUG_VALUE_VALUE(f, signum, SIGFPE); DEBUG_VALUE_VALUE(f, signum, SIGKILL); DEBUG_VALUE_VALUE(f, signum, SIGUSR1); DEBUG_VALUE_VALUE(f, signum, SIGSEGV); DEBUG_VALUE_VALUE(f, signum, SIGUSR2); DEBUG_VALUE_VALUE(f, signum, SIGPIPE); DEBUG_VALUE_VALUE(f, signum, SIGALRM); DEBUG_VALUE_VALUE(f, signum, SIGTERM); #ifdef SIGSTKFLT DEBUG_VALUE_VALUE(f, signum, SIGSTKFLT); #endif DEBUG_VALUE_VALUE(f, signum, SIGCHLD); DEBUG_VALUE_VALUE(f, signum, SIGCONT); DEBUG_VALUE_VALUE(f, signum, SIGSTOP); DEBUG_VALUE_VALUE(f, signum, SIGTSTP); DEBUG_VALUE_VALUE(f, signum, SIGTTIN); DEBUG_VALUE_VALUE(f, signum, SIGTTOU); DEBUG_VALUE_VALUE(f, signum, SIGURG); DEBUG_VALUE_VALUE(f, signum, SIGXCPU); DEBUG_VALUE_VALUE(f, signum, SIGXFSZ); DEBUG_VALUE_VALUE(f, signum, SIGVTALRM); DEBUG_VALUE_VALUE(f, signum, SIGPROF); DEBUG_VALUE_VALUE(f, signum, SIGWINCH); DEBUG_VALUE_VALUE(f, signum, SIGIO); #ifdef SIGPWR DEBUG_VALUE_VALUE(f, signum, SIGPWR); #endif DEBUG_VALUE_VALUE(f, signum, SIGSYS); DEBUG_VALUE_END_DEC(f, signum); } /** * Debug-print a mode_t variable. * * mode_t sometimes contains the file type (e.g. when returned by a stat() call) and sometimes * doesn't (e.g. when it's a parameter to an open(), chmod(), umask() call). * * Luckily, at least on Linux, none of the S_IF* constants are defined as 0. This means that we can * determine which category we fall into and we can produce nice debug output in both cases, without * having to maintain two separate functions. */ void debug_mode_t(FILE *f, mode_t mode) { const char *sep = "|"; mode_t type = mode & S_IFMT; DEBUG_VALUE_START(f, type) DEBUG_VALUE_VALUE(f, type, S_IFREG) DEBUG_VALUE_VALUE(f, type, S_IFDIR) DEBUG_VALUE_VALUE(f, type, S_IFLNK) DEBUG_VALUE_VALUE(f, type, S_IFBLK) DEBUG_VALUE_VALUE(f, type, S_IFCHR) DEBUG_VALUE_VALUE(f, type, S_IFIFO) DEBUG_VALUE_VALUE(f, type, S_IFSOCK) case 0: /* File type info is not available. Don't print anything here. */ sep = ""; break; DEBUG_VALUE_END_OCT(f, type) mode &= ~S_IFMT; fprintf(f, "%s0%03o", sep, mode); } /* * Debug-print a "wait status", as usually seen in the non-error return value of system() and * pclose(), and in the "wstatus" out parameter of the wait*() family. */ void debug_wstatus(FILE *f, int wstatus) { const char *sep = ""; fprintf(f, "%d (", wstatus); if (WIFEXITED(wstatus)) { fprintf(f, "%sexitstatus=%d", sep, WEXITSTATUS(wstatus)); sep = ", "; } if (WIFSIGNALED(wstatus)) { fprintf(f, "%stermsig=", sep); debug_signum(f, WTERMSIG(wstatus)); if (WCOREDUMP(wstatus)) { fprintf(f, ", coredump"); } sep = ", "; } if (WIFSTOPPED(wstatus)) { fprintf(f, "%sstopsig=", sep); debug_signum(f, WTERMSIG(wstatus)); sep = ", "; } if (WIFCONTINUED(wstatus)) { fprintf(f, "%scontinued", sep); sep = ", "; } fprintf(f, ")"); } /** * Debug-print CLONE_* flags, as usually seen in the 'flags' parameter of clone(). */ void debug_clone_flags(FILE *f, int flags) { DEBUG_BITMAP_START(f, flags); /* If CLONE_VM is not defined most likely other flags are not defined either. */ #ifdef CLONE_VM DEBUG_BITMAP_FLAG(f, flags, CLONE_VM); DEBUG_BITMAP_FLAG(f, flags, CLONE_FS); DEBUG_BITMAP_FLAG(f, flags, CLONE_FILES); DEBUG_BITMAP_FLAG(f, flags, CLONE_SIGHAND); DEBUG_BITMAP_FLAG(f, flags, CLONE_PIDFD); DEBUG_BITMAP_FLAG(f, flags, CLONE_PTRACE); DEBUG_BITMAP_FLAG(f, flags, CLONE_VFORK); DEBUG_BITMAP_FLAG(f, flags, CLONE_PARENT); DEBUG_BITMAP_FLAG(f, flags, CLONE_THREAD); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWNS); DEBUG_BITMAP_FLAG(f, flags, CLONE_SYSVSEM); DEBUG_BITMAP_FLAG(f, flags, CLONE_SETTLS); DEBUG_BITMAP_FLAG(f, flags, CLONE_PARENT_SETTID); DEBUG_BITMAP_FLAG(f, flags, CLONE_CHILD_CLEARTID); DEBUG_BITMAP_FLAG(f, flags, CLONE_DETACHED); DEBUG_BITMAP_FLAG(f, flags, CLONE_UNTRACED); DEBUG_BITMAP_FLAG(f, flags, CLONE_CHILD_SETTID); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWCGROUP); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWUTS); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWIPC); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWUSER); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWPID); DEBUG_BITMAP_FLAG(f, flags, CLONE_NEWNET); DEBUG_BITMAP_FLAG(f, flags, CLONE_IO); #endif DEBUG_BITMAP_END_HEX(f, flags & ~0xff); fprintf(f, "|"); debug_signum(f, flags & 0xff); } #ifdef __cplusplus } /* extern "C" */ #endif firebuild-0.8.2/src/common/debug_sysflags.h000066400000000000000000000027711447164520700207400ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef COMMON_DEBUG_SYSFLAGS_H_ #define COMMON_DEBUG_SYSFLAGS_H_ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #ifdef __cplusplus extern "C" { #endif void debug_open_flags(FILE *f, int flags); void debug_at_flags(FILE *f, int flags); void debug_fcntl_cmd(FILE *f, int cmd); void debug_fcntl_arg_or_ret(FILE *f, int cmd, int arg); void debug_error_no(FILE *f, int error_no); void debug_signum(FILE *f, int signum); void debug_mode_t(FILE *f, mode_t mode); void debug_wstatus(FILE *f, int wstatus); void debug_clone_flags(FILE *f, int flags); #ifdef __cplusplus } /* extern "C" */ #endif #endif /* COMMON_DEBUG_SYSFLAGS_H_ */ firebuild-0.8.2/src/common/fbb/000077500000000000000000000000001447164520700163105ustar00rootroot00000000000000firebuild-0.8.2/src/common/fbb/CPPLINT.cfg000066400000000000000000000000241447164520700200760ustar00rootroot00000000000000exclude_files=tpl.* firebuild-0.8.2/src/common/fbb/README_FBB.txt000066400000000000000000000575551447164520700205000ustar00rootroot00000000000000FirebuildBuffers (FBB) – User documentation =========================================== FBB is a similar concept to many popular data serialization solutions, such as Protocol Buffers, FlatBuffers etc. It is tailored to Firebuild's needs, including these features: - high performance, - async-signal-safety (e.g. doesn't malloc) (*). On the other hand, it does *NOT* support the following typical features: - architecture independence, - backwards compatibility with earlier FBB defitions. (*) The basic plain C API, including setters, getters, serializing etc. does not perform malloc. However, the debugging methods, and the convenience C++ API might malloc. Namespaces ---------- Namespaces, or prefixes correspond to different worlds, e.g. we have an "fbbcomm" namespace for messages used for interceptor-supervisor communication, "fbbfp" for fingerprinting a process, "fbbstore" for data stored in the cache, and "fbbtest" for unittesting. There's no link between separate namespaces, no explicit support is provided to encapsulate a message of one namespace inside a message of another (although you can transfer it in a char array as a blob). They are meant to be distinct worlds, in a way that an application might use multiple of them at the same time without having to worry about name conflicts. In the rest of this document we look at a single example namespace called "fbbns", often automatically capitalized to "FBBNS" by the code generator software. Basics ------ There are multiple object types, distinguished from each other using the "tag" field. For example, an FBBNS message with the tag "open" might represent the opening of a file, consisting of a string (filename), an int32 (flags) and another int32 (the returned fd). The "rename" tag might represent the renaming of a file, consisting of two strings (two filenames). And so on. Every FBB object, no matter if it's a builder or in serialized format (see later) knows its own tag. A generic FBB message means one that is of any of these tags, i.e. the tag is only known at runtime. You can consider this as the union of all the particular FBBNS_foo types. However, unlike with C unions, the object knows its own tag at runtime, and the object is just as small as required for this particular tag. The top level object is of this generic (a.k.a. union) type, and so are the embedded (nested) FBBNS members. Members of a particular FBBNS object can be of these types: 1. Scalar (fixed size): - bool - (unsigned) char, short, int, long, long long - (u)int{8,16,32,64}_t - uid_t, gid_t, mode_t, pid_t etc. - float, double - small custom structures, copied by value (you need to write a debugger method, see below) - etc. 2. More complex object (dynamic size): - string (C-style '\0'-terminated) - FBBNS (embedded message of the same namespace, with no pre-determined tag) Each of these fields can be: A. required (exactly one) Must be set before sending the message, otherwise it's an assertion failure in debug build, undefined behavior (might even crash) in prod build. B. optional (at most one) May or may not be set. C. array (zero, one or more items, decided at runtime) Note that there's no distinction between an unset and a zero-sized array. Defaults to zero-sized, no need to set if that's the desired value. 1A (required scalar): This is simply the scalar itself. 1B (optional scalar): This is the scalar, plus an additional accompanying boolean telling whether the value has been set. 1C (array of scalars): This is a pointer to such scalars, plus the length stored separately. 2A (required complex): This is equivalent to a non-NULL C pointer to the object. 2B (optional complex): This is equivalent to a C pointer, with the value of NULL denoting if the field is missing. 2C (array of complexes): This is equivalent to a pointer to a NULL-terminated array of pointers in C, e.g. "char **" or "void **". In order to send a blob whose size isn't known at compile time, use a char array. [Rationale: I wanted to maintain consistent design in that sense that for every data type we support, we support an array of them too. Unlike with array of strings ("char**") or array of FBBs ("void**"), there's no standard C practice to specify an array of blobs (pointers and lengths). The currently available recursive FBB method is not any more complicated in order to construct an array of char arrays (array of blobs) than any custom API for this one-off case would have been. And for just a single blob, a char array is already available. This is why there's no "native" blob support.] In order to send a fixed (and reasonably small) sized blob, or some compound data of fixed size, a more convenient and faster way is to declare (and typedef) a C struct that contains the necessary items, and refer to this struct as a scalar. This way setting the blob copies the value, rather than remembering the pointer only. Memory management becomes simpler when you construct the message. Message definition ------------------ Choose a namespace, in the examples we'll go with "fbbns". Create a .def file, i.e. "fbbns.def" with contents like this (it's a Python dictionary): { "tags": [ ("foo", [ (REQUIRED, "uint16_t", "myuint"), (OPTIONAL, STRING, "mystring"), (OPTIONAL, "int", "myint", "myint_debugger"), (ARRAY, "uint16_t", "myuintarray"), (ARRAY, STRING, "mystringarray"), (ARRAY, FBB, "myfbbarray"), ]), ("bar", [ (REQUIRED, "int", "something"), ]), ] } This defines message tags "foo" and "bar", and a couple of fields for them. For scalar data types the exact C data type should be spelled out (inside quotes). It might consist of multiple words, e.g. "unsigned long long". For strings and FBBs, use the uppercase STRING and FBB constants (without quotes). From the directory where fbbns.def resides, use the command path/to/generate_fbb fbbns outputdir to generate the C (and bit of C++) source code into the given output directory. Two formats: builder vs. serialized ----------------------------------- There are two ways an FBB can be represented in memory. One is the read-write "builder". This is used while constructing a message. It might contain raw C pointers (to strings, FBBs, arrays), and in case of FBB members the structure is recursive, i.e. contains pointers to other FBB structures within the same namespace. The other is the read-only "serialized" format. It's a blob (a sequence of bytes) that does not contain raw C pointers, can be passed from one process to another, or stored in a file and later read back. The builder can be serialized to the serialized format. There's no deserialization step, getter methods work directly on the serialized data. Building a message (C-style API) -------------------------------- The specific type "FBBNS_Builder_foo" refers to an entire builder object of tag "foo". Methods that work on a pre-determined tag take a pointer to such objects. The generic type "FBBNS_Builder" is more of a symbolic thing for cleaner code (to write the readable "FBBNS_Builder*" instead of the meaningless "void*"). Methods that work on any tag take such pointers. If needed, you need to cast the pointers manually. Let's suppose you want to construct a message of tag "foo". Get a corresponding builder structure somewhere (practically on the stack if you need async-signal-safety). You must initialize it too, this sets the "tag" field to the desired value and zeroes out the rest. FBBNS_Builder_foo mybuilder; fbbns_builder_foo_init(&mybuilder); Then you can set some values: fbbns_builder_foo_set_myuint(&mybuilder, 42); Refer to the section "Setters and getters" for details on these. IMPORTANT: The strings, FBBs, and arrays are not copied; however, certain operations (e.g. length computation) might happen when they are set. The caller owns these values, and is responsible for keeping them unaltered in the memory until the builder object is no longer used. Serializing a message (C-style API) ----------------------------------- First you need to measure how big the serialized format will be. size_t len = fbbns_builder_measure((FBBNS_Builder *) &mybuilder); Then you need to allocate a sufficiently large buffer to serialize into. Go with the stack or shared memory if you need async-signal-safety, otherwise you might allocate on the heap too. Example: char *buf = alloca(len); Now serialize the message into this area: fbbns_builder_serialize((FBBNS_Builder *) &mybuilder, buf); Note that this method also performs integrity checks on the builder, e.g. whether all required fields have indeed been set. Not setting a required field is considered programming error, thus results in an assertion failure in debug builds and undefined behavior in production builds, rather than some soft error to handle. Once serialized, you can throw away the builder object or any data referenced by that, the serialized version will remain intact. The serialized format does not know its own length, and running the getters on a truncated (or otherwise corrupted) serialized data might result in crash or incorrect behavior. Therefore the serialized format on its own is not suitable to be transferred over a continous stream; the data has to be prefixed with its own length in order to reconstruct the message boundaries. You can allocate an accordingly larger buffer, serialize to the proper offset, and then fill out the header. This is outside of the scope of FBB. Receiving and decoding a message (C-style API) ---------------------------------------------- You should use the symbolic type FBBNS_Serialized when pointing to a serialized FBBNS message of a generic tag, and FBBNS_Serialized_foo when pointing to a serialized FBBNS message of the particular tag "foo". Cast between the pointer types if necessary. Note, however, that the serialized message is most likely longer than the area covered by the FBBNS_Serialized or FBBNS_Serialized_foo structure. Don't copy these structures because getters will not work on the copies. Read the message into a contiguous area in the memory. Cast the pointer to the generic type (FBBNS_Serialized) in order to check the tag, verify that it's the value you expect, or branch on it if you allow multiple values in the given context. Then cast to the specific type (e.g. FBBNS_Serialized_foo) for the getters. FBBNS_Serialized *msg_generic = (FBBNS_Serialized *) buf; int tag = FBBNS_Serialized_get_tag(msg_generic); assert(tag == FBBNS_TAG_foo); FBBNS_Serialized_foo *msg = (FBBNS_Serialized_foo *) msg_generic; Now you can use the getters to read the message. uint16_t myuint = fbbns_serialized_foo_get_myuint(msg); Refer to the section "Setters and getters" for details on these. IMPORTANT: The getter, if used on a string, an FBB, or an array of anything, will return a pointer that points into the message data, but beyond the FBBNS_Serialized_foo structure. Do not copy the FBBNS_Serialized_foo structure and perform operations on the copy, because in that case the following storage area isn't copied and the pointer computations go wrong. Also, make sure the received message resides in memory unaltered as long as any of these pointers (returned by the getters) are in use. Setters and getters (C-style API) --------------------------------- Setters work on the builder only (obviously). Getters are available both for the builder and the serialized format. The ones for the serialized version are shown below, they always have an identical counterpart for the builder version, with one exception as noted below. (Getters of the builder are useful e.g. when sorting an array of FBBs, or for debugging.) The examples use these variables: FBBNS_Builder_foo bldr; fbbns_builder_foo_init(&bldr); const FBBNS_Serialized_foo *msg; ### Required scalar Nothing special. Setter: fbbns_builder_foo_set_myuint(&bldr, 42); Getter for the value or its address: uint16_t val = fbbns_serialized_foo_get_myuint(msg); const uint16_t *ptr = fbbns_serialized_foo_get_myuint_ptr(msg); /* ptr is non-NULL */ ### Optional scalar Setter is the same as for required scalars. Note that once you set a value, there's no way to unset: fbbns_builder_foo_set_myuint(&bldr, 42); Check if a value has been set: bool myuint_was_set = fbbns_serialized_foo_has_myuint(msg); Read the value. Must only be called after checking that the value has been set. Running the getter on an unset field is considered programming error and results in assertion failure. if (myuint_was_set) { uint16_t val = fbbns_serialized_foo_get_myuint(msg); } Or get a pointer, which returns NULL if the value wasn't set: const uint16_t *ptr = fbbns_serialized_foo_get_myuint_ptr(msg); if (ptr == NULL) { ... } Or you can use this convenience wrapper which returns the given fallback value if the field was unset. Needless to say, it cannot tell if the field was actually set to that fallback value. uint16_t val = fbbns_serialized_foo_get_myuint_with_fallback(msg, 100); ### Array of scalars Setter: pass the array with the item count. Note that you cannot append elements to an array one by one, you need to set the entire array in a single step. const uint16_t myuintarray[] = { 25, 20, 23 }; fbbns_builder_foo_set_myuintarray(&bldr, &myuintarray, 3); Get the item count: size_t count = fbbns_serialized_foo_get_myuintarray_count(msg); Get a particular item by value (the index must be a valid one): uint16_t val = fbbns_serialized_foo_get_myuintarray_at(msg, index); Get a pointer to the entire array: uint16_t *arr = fbbns_serialized_foo_get_myuintarray(msg); Get the entire array, C++ convenience API. Note that this allocates memory (hence not async-signal-safe) and copies the data: std::vector values = fbbns_serialized_foo_get_myuintarray_as_vector(msg); ### Required or optional string Set (you can unset by setting to NULL, but why would you need it): fbbns_builder_foo_set_mystring(&bldr, "Hello world!"); Set with length. Note that the string must still be '\0'-terminated, the passed value has to be its strlen(). This method is for speeding up things if you already know the length, not for introducing blob support. fbbns_builder_foo_set_mystring_with_length(&bldr, "Hello world!", 12); Set, convenience C++ wrapper (use this only if the string has no embedded '\0'): std::string str("Hello world!"); fbbns_builder_foo_set_mystring(str); Getter. It's okay to call this on unset optional strings, it'll return NULL: const char *str = fbbns_serialized_foo_get_mystring(msg); Get the length: fbb_size_t len = fbbns_serialized_foo_get_mystring_len(msg); Get both the string and the length: fbb_size_t len; const char *str = fbbns_serialized_foo_get_mystring_with_len(msg, &len); Check if set (only for optional strings). Same as checking if the getters returns NULL: if (fbbns_serialized_foo_has_mystring(msg)) { ... } Get the string, C++ convenience API. Note that this allocates memory (hence not async-signal-safe) and copies the data. Also, it's an assertion failure to call it on an unset optional string. std::string str = fbbns_serialized_foo_get_mystring_as_string(msg); ### Required or optional FBBs Essentially the same as required and optional strings. No setter variant that would take the length, and no setter/getter C++ variants working with std::string. FBBNS_Builder_bar inner_bldr; /* initialize and fill in inner_bldr's fields here */ fbbns_builder_foo_set_myfbb(&bldr, (FBBNS_Builder *) &inner_bldr); const FBBNS_Serialized *inner_msg = fbbns_serialized_foo_get_myfbb(msg); ### Array of strings Note that - as with arrays of scalars - you cannot append elements to an array one by one, you need to set the entire array in a single step. Setter - NULL-terminated array of pointers, in the usual C "char**" way: const char *loremipsum[] = { "lorem", "ipsum", NULL }; fbbns_builder_foo_set_mystringarray(&bldr, loremipsum); Setter - from an array of pointers, and their length. No need for a trailing NULL, but all intermediate items must be non-NULL: const char *dolorsitamet[] = { "dolor", "sit", "amet" }; fbbns_builder_foo_set_mystringarray_with_count(&bldr, dolorsitamet, 3); Setter - convenience C++ wrapper. Note that it's just a thin wrapper around the C setters. Note that there's no version taking the more typical std::vector because then the memory layout of things is not what FBB expects; use the generic setter with the "item_fn" callback function to hook up a vector of C++ strings. std::vector c_string_array = ...; fbbns_builder_foo_set_mystringarray(&bldr, c_string_array); Setter - generic version. Takes the length, and a getter "item_fn" callback function that has to return the value for any valid index and also the length if the given pointer is non-NULL. The callback won't be called with out-of-bounds index. This method can work as an adaptor between FBB and any data structure the caller might have: const char *mystemfn(fbb_size_t index, void *user_data, fbb_size_t *len_out) { if (len_out != NULL) { *len_out = 4; } if (index == 0) return "this" else return "that"; } fbbns_builder_foo_set_mystringarray_item_fn(&bldr, 2 /* item count */, myitemfn, myuserdata); Getter - get the count: size_t count = fbbns_serialized_foo_get_mystringarray_count(msg); Getter - get a particular item: const char *str = fbbns_serialized_foo_get_mystringarray_at(msg, index); Getter - get a particular item's length: fbb_size_t len = fbbns_serialized_foo_get_mystringarray_len_at(msg, index); Getter - get a particular item along with its length: fbb_size_t len; const char *str = fbbns_serialized_foo_get_mystringarray_with_len_at(msg, index, &len); Get the entire array, C++ convenience API. Note that this allocates memory (hence not async-signal-safe) and copies the data: std::vector values = fbbns_serialized_foo_get_mystringarray_as_vector(msg); ### Array of FBBs The same as the array of strings, except that the type is "const FBBNS_Builder *" on the builder and "const FBBNS_Serialized *" on the serialized format, instead of "const char *" or "std::string". Debugging --------- Debug a message from source code: FILE *f = ...; fbbns_builder_debug(f, (FBBNS_Builder *) &bldr); fbbns_serialized_debug(f, (FBBNS_Serialized *) msg); Note that the debugger uses stdio, and hence is not async-signal-safe. (Directly writing to the fd without buffering would be async-signal-safe, but significantly slower.) Debug from command line: The fbbns_decode application pretty-prints the serialized FBB stored in the given file, for easy debugging. The debug format is valid JSON, and almost valid Python (you need to set null=None, true=True and false=False before eval'ing it). Another method that might come handy gets a text representation of the tag: const char *str = FBBNS_tag_to_string(tag); C++-style API ------------- If you use FBB from C++, you are free to use the C-style API shown above, but you can also use the more concise C++-style API. Each method, except FBBNS_tag_to_string(), has a C++ counterpart which you call on the Builder or Serialized object. The method name is much shorter: it omits the namespace, the "builder" or "serialized" word, and the tag name. The Builder's constructor automatically runs init() as well, you don't need to call that method. You need to call init() if the memory is allocated independently and the area is cast to a Builder. Examples: FBBNS_Builder_foo mybuilder; mybuilder.set_myuint(42); size_t len = ((FBBNS_Builder *) &mybuilder)->measure(); ((FBBNS_Builder *) &mybuilder)->serialize(buf); FBBNS_Serialized *msg_generic = (FBBNS_Serialized *) buf; int tag = msg_generic->get_tag(); FBBNS_Serialized_foo *msg = (FBBNS_Serialized_foo *) msg_generic; uint16_t myuint = msg->get_myuint(); Custom scalars -------------- Scalars can have types that are declared in some specific system-wide or 3rd-party header files (such as mode_t, pid_t, XXH128hash_t etc.). You might need to include some header files to declare them. Also you must write a debugger method for them, unless the type can be automatically converted to a "long long int" and you're okay with printing the value as such, or if you specify a custom debugger method for every such field. Custom debugging of certain types --------------------------------- It is possible to define a custom debugger for fields of a certain type, overriding the default debugger, or implementing one if it has no default. For example, you can have a custom debugger for all mode_t fields to print them differently than just simply a decimal integer, or all XXH128hash_t fields that don't have a default debugger. Next to the outmost "tags" key in the Python definition file, add the keys "extra_h" for source code to place in the .h file and "extra_c" for a snippet placed in the .c file, and "types_with_custom_debugger" being a list of all the scalar types that need a custom debugger. These debuggers should be implemented in "extra_c" or elsewhere. The debugger method has to have the name "fbbns_debug_type", whereas "fbbns" is the actual namespace, and "type" is the type we're talking about, with ' ' and ':' characters replaced by '_'. For example, in fbbcomm.def we overwrite the debugger of type "mode_t" by adding "mode_t" to the "types_with_custom_debugger" array and defining the "fbbcomm_debug_mode_t" function. The debugger method takes two parameters: the stdio stream to print to, and the value to print. Custom debugging of certain variable names ------------------------------------------ It is possible to define a custom debugger for fields of a certain name, across all the tags in the FBB. For example, you can have a custom debugger for all the error_no fields. This overrides the debugger determined by the variable's type. Next to the outmost "tags" key in the Python definition file, add the keys "extra_h" for source code to place in the .h file and "extra_c" for a snippet placed in the .c file, and "varnames_with_custom_debugger" being a list of all the variable names that need a custom debugger. These debuggers should be implemented in "extra_c" or elsewhere. The debugger method has to have the name "fbbns_debug_varname", whereas "fbbns" is the actual namespace, and "varname" is the name of the fields we're talking about. For example, in fbbcomm.def we overwrite the debugger of variables named "error_no" by adding "error_no" to the "varnames_with_custom_debugger" array and defining the "fbbcomm_debug_error_no" function. The debugger method takes two parameters: the stdio stream to print to, and the value to print. Custom debugging of certain fields ---------------------------------- Individual fields can have a custom debugger method. This overrides the debugger determined by the variable's type or name. Just specify the method name as the 4th field in the field's tuple, e.g. as for "myint_debugger" above. The debugger method takes four parameters: the stdio stream to print to, the value to print, whether we're debugging a builder (false) or serialized (true) FBB object, and a pointer to that builder or serialized FBB. The 3rd and 4th parameters are useful if the way to debug a field depends on other FBB fields, for example fcntl's argument or return value should be debug-printed differently based on fcntl's command. Serialization format -------------------- See README_FBB_INTERNAL.txt. Compatibility ------------- FBB is guaranteed to be consistent within one particular architecture and one particular FBB version only. If required, the caller needs to ensure that serialized formats created by one build are not used from an incompatible one. FBB should add some support to make this easier. See #544. firebuild-0.8.2/src/common/fbb/README_FBB_INTERNAL.txt000066400000000000000000000143361447164520700217620ustar00rootroot00000000000000FirebuildBuffers (FBB) – Internals ================================== This document assumes that you're already familiar with README_FBB.txt. Builder structures ------------------ The builder structure of any particular tag consists of two major parts: - The "wire" field contains bits that will be part of the serialized format as-is. This includes the values of scalars, the lengths of required or optional strings, and the item count of arrays. - Additional fields that won't be part of the serialized message. This includes the raw pointers (for strings, FBBs, and arrays of anything), or the item getter callback function (for arrays of strings or FBBs). This also includes the debug boolean to check that a required scalar has indeed been set. Serialized structure -------------------- The serialized structure of a particular tag is the same as the "wire" field of the builder, providing direct access to certain values. This is followed by other data in the memory, as discussed below. Relptr ------ The concept of "relptr" (relative pointer) is similar to a C pointer, but in a way that doesn't care about the actual memory location: - a positive integer denotes the byte offset where the said data begins in memory, relative to the beginning of the FBB object, - or 0 represents the NULL pointer. In case of nested FBBs, relptr is relative to the address of the innermost (directly encapsulating) FBB object. This way neither the serialization methods nor the getters have to care whether an FBB is the toplevel or a nested one. The following table shows the number of pointer indirections necessary for each of the available types. | required / optional │ array │ ─────────────────────────────┼─────────────────────┼───────────┤ scalars (bool, int etc.) │ 0 │ 1 │ ─────────────────────────────┼─────────────────────┼───────────┤ complex types (string, FBB) │ 1 │ 2 │ ─────────────────────────────┴─────────────────────┴───────────┘ In case of the Builder it's the number of times you have to follow a raw C pointer, in case of the Serialized format it's the number of times you have to follow a relptr to get to the actual data. Serialized format ----------------- The serialized format goes as follows: First there is the structure representing the concrete message tag. In all cases this begins with the tag itself, and then contains the scalars, the string lengths, and the array item counts - these are the common parts between the Builder and the Serialized structure, namely the "wire" field of the Builder. ┌───────────────────────────────┐ ┐ │ tag (int) │ │ │ scalar │ │ │ string length │ ├ FBBNS_Serialized_foo │ array item count │ │ │ has_* for optional scalar │ │ └───────────────────────────────┘ ┘ Then it is followed by another structure that contains the first-hop relptrs. That is, relptrs that point directly to the data (where the table above has the number 1), and the first hop of indirect pointers (where the table has the number 2). ┌───────────────────────────────┐ ┐ │ scalar array relptr │ ──┐ │ │ string relptr │ ──│──┐ │ │ FBBNS relptr │ ──│──│──┐ ├ FBBNS_Relptrs_foo │ string array first relptr │ ──│──│──│──┐ │ │ FBBNS array first relptr │ ──│──│──│──│──┐ │ └───────────────────────────────┘ ┆ ┆ ┆ ┆ ┆ ┘ This structure is omitted if it would be empty, because C and C++ disagree on the size of empty structs. Then it is followed by the variable-length data. This includes the scalar arrays, the '\0'-terminated strings, the serialized nested FBBs, and arrays of the latter two. For arrays of complex types there's an additional hop needed. For arrays of strings the first relptr points to an alternating array of second relptrs and string lengths. For arrays of FBBs the first relptr points to the array of the second relptrs. The example shows one of each kind, with the arrays containing 2 items each (arrows continued from the previous figure): ┌───────────────────────────────┐ ┆ ┆ ┆ ┆ ┆ │ scalar array │ <─┘ │ │ │ │ │ string '\0' │ <────┘ │ │ │ │ FBBNS serialized │ <───────┘ │ │ │ string_array[0] second relptr │ <─┬────────┘ │ │ string_array[0] length │ │ │ │ string_array[1] second relptr │ ──│──┐ │ │ string_array[1] length │ │ │ │ │ string_array[0] '\0' │ <─┘ │ │ │ string_array[1] '\0' │ <────┘ │ │ FBBNS_array[0] second relptr │ <─┬───────────┘ │ FBBNS_array[1] second relptr │ ──│──┐ │ FBBNS_array[0] serialized │ <─┘ │ │ FBBNS_array[1] serialized │ <────┘ └───────────────────────────────┘ Padding might be added at some places, they are not shown in the pictures. firebuild-0.8.2/src/common/fbb/generate_fbb000077500000000000000000000075341447164520700206520ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import sys from jinja2 import Environment, FileSystemLoader if len(sys.argv) != 3: print("Usage: generate_fbb namespace outputdir", file=sys.stderr) print(" Processes the input file \"\".def from the current directory", file=sys.stderr) exit(1) fbbdir = os.path.dirname(sys.argv[0]) # wherever generate_fbb and tpl.[ch] reside namespace = sys.argv[1] outdir = sys.argv[2] env = Environment(loader=FileSystemLoader(fbbdir), line_statement_prefix='###', trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, extensions=['jinja2.ext.do']) # Use a practically unique temporary filename, see #314 tmpsuffix = ".tmp." + str(os.getpid()) # Symbolic constants to reduce the chance of typos REQUIRED = "required" OPTIONAL = "optional" ARRAY = "array" STRING = "string" FBB = "fbb" def gen_fbb(params): msgs = params.get("tags") # Python can easily unpack tuples whose size isn't known in advance. Jinja cannot. I don't want to # bloat the FBB definitions by requiring a None for all the rarely used debugging-related 4th # fields. Alter the big msgs object here so that all its tuples contain all 4 fields. for (msg, fields) in msgs: for (index, (req, type, var, *args)) in enumerate(fields): debugger_method = args[0] if len(args) > 0 else None fields[index] = (req, type, var, debugger_method) for (msg, fields) in msgs: for (req, type, var, dbgfn) in fields: if req not in [REQUIRED, OPTIONAL, ARRAY]: print("Unknown value instead of REQUIRED, OPTIONAL or ARRAY", file=sys.stderr) exit(1) for extension in [".c", ".h", "_decode.c"]: template = env.get_template("tpl" + extension) rendered = template.render(ns=namespace, msgs=msgs, types_with_custom_debugger=params.get("types_with_custom_debugger", []), varnames_with_custom_debugger=params.get("varnames_with_custom_debugger", []), extra_c=params.get("extra_c", ""), extra_h=params.get("extra_h", ""), REQUIRED=REQUIRED, OPTIONAL=OPTIONAL, ARRAY=ARRAY, STRING=STRING, FBB=FBB) if rendered: filename = outdir + "/" + namespace + extension with open(filename + tmpsuffix, "w") as f: f.write(rendered) os.rename(filename + tmpsuffix, filename) # Generate the same file with both .c and .cc extension, so that cmake can # easily compile the C and C++ variants separately. if extension == ".c": extension = ".cc" filename = outdir + "/" + namespace + extension with open(filename + tmpsuffix, "w") as f: f.write(rendered) os.rename(filename + tmpsuffix, filename) with open("" + namespace + ".def", "r") as f: params = eval(f.read()) gen_fbb(params) firebuild-0.8.2/src/common/fbb/tpl.c000066400000000000000000000442501447164520700172600ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template to generate {{ ns }}.c. #} {# ------------------------------------------------------------------ #} /* Auto-generated by generate_fbb, do not edit */ {# Well, not here, #} {# this is the manually edited template file, #} {# placing this message in the output. #} {% set NS = ns|upper %} #include "{{ ns }}.h" #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-variable" /* Round up the offset number to the nearest multiple of 8. * Updates the given offset variable to the new value. * The old offset value must be nonnegative. */ #define ADD_PADDING_LEN(offset) do { \ offset = (offset + 7) & ~0x07; \ } while (0); /* Pad with zeros to the next offset that's a multiple of 8. * Zeroes out 0 to 7 memory cells beginning at p + offset. * Updates the given offset variable accordingly to the new value. * The old offset value must be nonnegative. */ #define PAD(p, offset) do { \ int len = 7 - ((offset + 7) & 0x07); \ memset(p + offset, 0, len); \ offset += len; \ } while (0); /* Beginning of extra_c */ {{ extra_c }} /* End of extra_c */ static void {{ ns }}_debug_string(FILE *f, const char *str) { fputc('"', f); while (*str) { size_t quick_run = strcspn(str, "\\\"\b\f\n\r\t"); fwrite(str, 1, quick_run, f); str += quick_run; if (!*str) break; switch (*str) { case '\\': fputs("\\\\", f); break; case '"': fputs("\\\"", f); break; case '\b': fputs("\\b", f); break; case '\f': fputs("\\f", f); break; case '\n': fputs("\\n", f); break; case '\r': fputs("\\r", f); break; case '\t': fputs("\\t", f); break; default: assert(0); } str++; } fputc('"', f); } static void {{ ns }}_builder_debug_indent(FILE *f, const {{ NS }}_Builder *msg, int indent); static void {{ ns }}_serialized_debug_indent(FILE *f, const {{ NS }}_Serialized *msg, int indent); ### for (msg, fields) in msgs /****************************************************************************** * {{ msg }} ******************************************************************************/ ### set jinjans = namespace(has_relptr=False) ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY or type in [STRING, FBB] ### set jinjans.has_relptr = True ### endif ### endfor ### for variant in ['builder', 'serialized'] /* * {{ variant|capitalize }} - Debug a '{{ msg }}' message */ static void {{ ns }}_{{ variant }}_{{ msg }}_debug_indent(FILE *f, const {{ NS }}_{{ variant|capitalize }}_{{ msg }} *msg, int indent) { const char *sep; const int indent_step = 4; indent += indent_step; fprintf(f, "{\n%*s\"[{{ NS }}_TAG]\": \"%s\"", indent, "", "{{ msg }}"); ### for (quant, type, var, dbgfn) in fields ### if quant in [REQUIRED, OPTIONAL] ### if quant == OPTIONAL /* Optional {{ type }} '{{ var }}' */ if ({{ ns }}_{{ variant }}_{{ msg }}_has_{{ var }}(msg)) { ### else /* Required {{ type }} '{{ var }}' */ if (1) { ### endif fprintf(f, ",\n%*s\"{{ var }}\": ", indent, ""); ### if dbgfn {{ dbgfn }}(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg), {% if variant == 'builder' %}false{% else %}true {% endif %}, msg); ### elif var in varnames_with_custom_debugger {{ ns }}_debug_{{ var }}(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg)); ### elif type == STRING {{ ns }}_debug_string(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg)); ### elif type == FBB {{ ns }}_{{ variant }}_debug_indent(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg), indent); ### elif type in types_with_custom_debugger {{ ns }}_debug_{{ type|replace(" ","_")|replace(":","_") }}(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg)); ### elif type == "bool" fprintf(f, "%s", {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg) ? "true" : "false"); ### else fprintf(f, "%lld", (long long) {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}(msg)); ### endif ### if quant == OPTIONAL } else { fprintf(f, ",\n%*s\"// {{ var }}\": null", indent, ""); ### endif } ### else /* {{ type }} array '{{ var }}' */ fprintf(f, ",\n%*s\"{{ var }}\": [", indent, ""); indent += indent_step; sep = ""; for (fbb_size_t idx = 0; idx < {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_count(msg); idx++) { fprintf(f, "%s\n%*s", sep, indent, ""); ### if dbgfn {{ dbgfn }}(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx) {% if variant == 'builder' %}false{% else %}true {% endif %}, msg); ### elif var in varnames_with_custom_debugger {{ ns }}_debug_{{ var }}(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx)); ### elif type == STRING {{ ns }}_debug_string(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx)); ### elif type == FBB {{ ns }}_{{ variant }}_debug_indent(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx), indent); ### elif type in types_with_custom_debugger {{ ns }}_debug_{{ type|replace(" ","_")|replace(":","_") }}(f, {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx)); ### elif type == "bool" fprintf(f, "%s", {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx) ? "true" : "false"); ### else fprintf(f, "%lld", (long long) {{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_at(msg, idx)); ### endif sep = ","; } indent -= indent_step; if ({{ ns }}_{{ variant }}_{{ msg }}_get_{{ var }}_count(msg) > 0) fprintf(f, "\n%*s", indent, ""); fprintf(f, "]"); ### endif ### endfor indent -= indent_step; fprintf(f, "\n%*s}", indent, ""); } ### endfor /* * Builder - Measure a '{{ msg }}' message * * Has to be kept in sync (wrt. paddings and such) with {{ ns }}_builder_{{ msg }}_serialize() * to make sure that they return the same length. */ static fbb_size_t {{ ns }}_builder_{{ msg }}_measure({{ NS }}_Builder_{{ msg }} *msgbldr) { /* The base structure */ fbb_size_t len = sizeof({{ NS }}_Serialized_{{ msg }}); ### if jinjans.has_relptr /* Relptrs */ len += sizeof({{ NS }}_Relptrs_{{ msg }}); ### endif /* Padding */ ADD_PADDING_LEN(len); /* Sizes of scalar arrays */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type not in [STRING, FBB] len += msgbldr->wire.{{ var }}_count_ * sizeof(*msgbldr->{{ var }}_); ADD_PADDING_LEN(len); ### endif ### endfor /* Sizes of required and optional strings */ ### for (quant, type, var, dbgfn) in fields ### if quant in [REQUIRED, OPTIONAL] and type == STRING if (msgbldr->{{ var }}_ != NULL) { len += msgbldr->wire.{{ var }}_len_ + 1; ADD_PADDING_LEN(len); } ### endif ### endfor /* Recurse into required and optional FBBs */ ### for (quant, type, var, dbgfn) in fields ### if quant in [REQUIRED, OPTIONAL] and type == FBB if (msgbldr->{{ var }}_ != NULL) { len += {{ ns }}_builder_measure(msgbldr->{{ var }}_); /* already includes padding */ } ### endif ### endfor /* The second hop for arrays of strings */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type == STRING len += 2 * msgbldr->wire.{{ var }}_count_ * sizeof(fbb_size_t); /* we'll build an alternating list of offsets and lengths */ ADD_PADDING_LEN(len); for (fbb_size_t idx = 0; idx < msgbldr->wire.{{ var }}_count_; idx++) { len += {{ ns }}_builder_{{ msg }}_get_{{ var }}_len_at(msgbldr, idx) + 1; ADD_PADDING_LEN(len); } ### endif ### endfor /* The second hop for arrays of FBBs, including recursion */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type == FBB len += msgbldr->wire.{{ var }}_count_ * sizeof(fbb_size_t); ADD_PADDING_LEN(len); for (fbb_size_t idx = 0; idx < msgbldr->wire.{{ var }}_count_; idx++) { len += {{ ns }}_builder_measure({{ ns }}_builder_{{ msg }}_get_{{ var }}_at(msgbldr, idx)); /* already includes padding */ } ### endif ### endfor ADD_PADDING_LEN(len); return len; } /* * Builder - Serialize a '{{ msg }}' message to memory * * Has to be kept in sync (wrt. paddings and such) with {{ ns }}_builder_{{ msg }}_measure() * to make sure that they return the same length. */ static fbb_size_t {{ ns }}_builder_{{ msg }}_serialize(const {{ NS }}_Builder_{{ msg }} *msgbldr, char *dst) { #ifdef FB_EXTRA_DEBUG /* Verify that the required fields were set */ ### for (quant, type, var, dbgfn) in fields ### if quant == REQUIRED ### if type in [STRING, FBB] assert(msgbldr->{{ var }}_ != NULL); ### else assert(msgbldr->has_{{ var }}_); ### endif ### endif ### endfor #endif fbb_size_t offset = 0; /* relative to the beginning of this (sub)message */ /* The wire structure */ memcpy(dst, &msgbldr->wire, sizeof(msgbldr->wire)); offset = sizeof(msgbldr->wire); /* No padding here (other than the one contained within msgbldr->wire). * If more padding is added here then the offset of the Relptr structure * will need to be adjusted in the serialized getters. */ ### if jinjans.has_relptr /* First hop relptrs. Zero out, just in case there's a padding at the end of this structure that we * won't initialize (or we could go with attribute packed, too). Will set the actual values soon. */ {{ NS }}_Relptrs_{{ msg }} *relptrs = ({{ NS }}_Relptrs_{{ msg }} *) (dst + offset); memset(dst + offset, 0, sizeof({{ NS }}_Relptrs_{{ msg }})); offset += sizeof({{ NS }}_Relptrs_{{ msg }}); ### endif /* Padding */ PAD(dst, offset); /* Arrays of scalars */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type not in [STRING, FBB] if (msgbldr->wire.{{ var }}_count_ > 0) { relptrs->{{ var }}_relptr_ = offset; fbb_size_t size = msgbldr->wire.{{ var }}_count_ * sizeof(msgbldr->{{ var }}_[0]); memcpy(dst + offset, msgbldr->{{ var }}_, size); offset += size; PAD(dst, offset); } else { relptrs->{{ var }}_relptr_ = 0; } ### endif ### endfor /* Required and optional strings */ ### for (quant, type, var, dbgfn) in fields ### if quant in [REQUIRED, OPTIONAL] and type == STRING if (msgbldr->{{ var }}_ != NULL) { relptrs->{{ var }}_relptr_ = offset; fbb_size_t size = msgbldr->wire.{{ var }}_len_ + 1; memcpy(dst + offset, msgbldr->{{ var }}_, size); offset += size; PAD(dst, offset); } else { relptrs->{{ var }}_relptr_ = 0; } ### endif ### endfor /* Required and optional FBBs */ ### for (quant, type, var, dbgfn) in fields ### if quant in [REQUIRED, OPTIONAL] and type == FBB if (msgbldr->{{ var }}_ != NULL) { relptrs->{{ var }}_relptr_ = offset; fbb_size_t size = {{ ns }}_builder_serialize(msgbldr->{{ var }}_, dst + offset); offset += size; /* FBB is serialized with a final padding, so no more padding is added here */ } else { relptrs->{{ var }}_relptr_ = 0; } ### endif ### endfor /* Arrays of strings */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type == STRING if (msgbldr->wire.{{ var }}_count_ > 0) { relptrs->{{ var }}_relptr_ = offset; fbb_size_t *hops = (fbb_size_t *) (dst + offset); fbb_size_t size = 2 * msgbldr->wire.{{ var }}_count_ * sizeof(fbb_size_t); /* room for alternating list of offsets and lengths */ offset += size; PAD(dst, offset); for (fbb_size_t idx = 0; idx < msgbldr->wire.{{ var }}_count_; idx++) { fbb_size_t len = 0; const char *str = {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_len_at(msgbldr, idx, &len); size = len + 1; *hops++ = offset; /* build up an alternating list of offsets and lengths */ *hops++ = len; memcpy(dst + offset, str, size); offset += size; PAD(dst, offset); } } else { relptrs->{{ var }}_relptr_ = 0; } ### endif ### endfor /* Arrays of FBBs */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type == FBB if (msgbldr->wire.{{ var }}_count_ > 0) { relptrs->{{ var }}_relptr_ = offset; fbb_size_t *hops = (fbb_size_t *) (dst + offset); fbb_size_t size = msgbldr->wire.{{ var }}_count_ * sizeof(fbb_size_t); offset += size; PAD(dst, offset); for (fbb_size_t idx = 0; idx < msgbldr->wire.{{ var }}_count_; idx++) { *hops++ = offset; const {{ NS }}_Builder *fbb = {{ ns }}_builder_{{ msg }}_get_{{ var }}_at(msgbldr, idx); offset += {{ ns }}_builder_serialize(fbb, dst + offset); /* recurse */ /* FBB is serialized with a final padding, so no more padding is added here */ } } else { relptrs->{{ var }}_relptr_ = 0; } ### endif ### endfor PAD(dst, offset); return offset; } ### endfor /****************************************************************************** * Global ******************************************************************************/ /* * Lookup array for the message name of a particular message tag */ static const char *{{ ns }}_tag_to_string_array[] = { NULL, ### for (msg, _) in msgs "{{ msg }}", ### endfor }; /* * Get the tag as string */ const char *{{ ns }}_tag_to_string(int tag) { assert(tag >= 1 && tag < {{ NS }}_TAG_NEXT); return {{ ns }}_tag_to_string_array[tag]; } /* * Builder - Lookup array for the debugger function of a particular message tag */ static void (*{{ ns }}_builder_debuggers_array[])(FILE *, const {{ NS }}_Builder *, int) = { NULL, ### for (msg, _) in msgs (void (*) (FILE *, const {{ NS }}_Builder *, int)) {{ ns }}_builder_{{ msg }}_debug_indent, ### endfor }; /* * Builder - Debug any message with custom indentation and no final newline and no flushing */ static void {{ ns }}_builder_debug_indent(FILE *f, const {{ NS }}_Builder *msg, int indent) { int tag = * ((int *) msg); assert(tag >= 1 && tag < {{ NS }}_TAG_NEXT); (*{{ ns }}_builder_debuggers_array[tag])(f, msg, indent); } /* * Builder - Debug any message */ void {{ ns }}_builder_debug(FILE *f, const {{ NS }}_Builder *msg) { {{ ns }}_builder_debug_indent(f, msg, 0); fprintf(f, "\n"); fflush(f); } /* * Serialized - Lookup array for the debugger function of a particular message tag */ static void (*{{ ns }}_serialized_debuggers_array[])(FILE *, const {{ NS }}_Serialized *, int) = { NULL, ### for (msg, _) in msgs (void (*) (FILE *, const {{ NS }}_Serialized *, int)) {{ ns }}_serialized_{{ msg }}_debug_indent, ### endfor }; /* * Serialized - Debug any message with custom indentation and no final newline and no flushing */ static void {{ ns }}_serialized_debug_indent(FILE *f, const {{ NS }}_Serialized *msg, int indent) { int tag = * ((int *) msg); assert(tag >= 1 && tag < {{ NS }}_TAG_NEXT); (*{{ ns }}_serialized_debuggers_array[tag])(f, msg, indent); } /* * Serialized - Debug any message */ void {{ ns }}_serialized_debug(FILE *f, const {{ NS }}_Serialized *msg) { {{ ns }}_serialized_debug_indent(f, msg, 0); fprintf(f, "\n"); fflush(f); } /* * Builder - Lookup array for the measurer function of a particular message tag */ static fbb_size_t (*{{ ns }}_builder_measures_array[])(const {{ NS }}_Builder *) = { NULL, ### for (msg, _) in msgs (fbb_size_t (*) (const {{ NS }}_Builder *)) {{ ns }}_builder_{{ msg }}_measure, ### endfor }; /* * Builder - Measure any message * * See the documentation in tpl.h. */ fbb_size_t {{ ns }}_builder_measure(const {{ NS }}_Builder *msgbldr) { /* Invoke the particular measurer for this message type */ int tag = * ((int *) msgbldr); assert(tag >= 1 && tag < {{ NS }}_TAG_NEXT); return (*{{ ns }}_builder_measures_array[tag])(msgbldr); } /* * Builder - Lookup array for the serializer function of a particular message tag */ static fbb_size_t (*{{ ns }}_builder_serializers_array[])(const {{ NS }}_Builder *, char *) = { NULL, ### for (msg, _) in msgs (fbb_size_t (*) (const {{ NS }}_Builder *, char *)) {{ ns }}_builder_{{ msg }}_serialize, ### endfor }; /* * Builder - Serialize any message to memory * * See the documentation in tpl.h. */ fbb_size_t {{ ns }}_builder_serialize(const {{ NS }}_Builder *msgbldr, char *dst) { #ifdef FB_EXTRA_DEBUG /* If the measured value is incorrect then a nasty buffer overrun can occur. * In order to guarantee FBB's correct behavior, let's do the debug assertion internally here in * FBB, rather than the caller having to do it. Which means we need to run measure() again (the * caller has already run it but we don't know that value). */ fbb_size_t len_measured = {{ ns }}_builder_measure(msgbldr); #endif /* Invoke the particular serializer for this message type */ int tag = * ((int *) msgbldr); assert(tag >= 1 && tag < {{ NS }}_TAG_NEXT); fbb_size_t len = (*{{ ns }}_builder_serializers_array[tag])(msgbldr, dst); #ifdef FB_EXTRA_DEBUG assert(len == len_measured); #endif return len; } #pragma GCC diagnostic pop firebuild-0.8.2/src/common/fbb/tpl.h000066400000000000000000001706421447164520700172720ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template to generate {{ ns }}.h. #} {# ------------------------------------------------------------------ #} /* Auto-generated by generate_fbb, do not edit */ {# Well, not here, #} {# this is the manually edited template file, #} {# placing this message in the output. #} {% set NS = ns|upper %} #ifndef {{ NS }}_H #define {{ NS }}_H 1 /* Beginning of extra_h */ {{ extra_h }} /* End of extra_h */ #ifdef __cplusplus #include #include #endif #include #include #include #include #include #include #include "common/cstring_view.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" {# # Looks like a clang bug: with -std=c11 the code compiles fine, but with -std=gnu11 # (the default, which is supposed to be c11 plus some extensions) it complains that: # "redefinition of typedef [...] is a C11 feature" #} #ifdef __clang__ #pragma GCC diagnostic ignored "-Wtypedef-redefinition" #endif #ifdef __cplusplus extern "C" { #endif typedef uint32_t fbb_size_t; enum { /* Values are spelled out for easier debugging. * Start at 1 so that it's easier to catch a forgotten initialization. */ {{ NS }}_TAG_UNUSED = 0, ### for (msg, _) in msgs {{ NS }}_TAG_{{ msg }} = {{ loop.index }}, ### endfor {{ NS }}_TAG_NEXT }; typedef enum { /* A standard plain C "char**" containing the strings pointers. */ {{ NS }}_STRING_INPUT_FORMAT_ARRAY, /* A "cstring_view*" containing the (pointer, length) pairs. */ {{ NS }}_STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY, #ifdef __cplusplus /* C++ only: A "std::string*" pointing to the string array. */ {{ NS }}_STRING_INPUT_FORMAT_CXX_STRING_ARRAY, #endif /* An item_fn callback that returns the string pointer and length for a given index. */ {{ NS }}_STRING_INPUT_FORMAT_CALLBACK, } {{ NS }}_String_Input_Format; typedef enum { /* A plain C-style array containing the FBB pointers as items. */ {{ NS }}_FBB_INPUT_FORMAT_ARRAY, /* An item_fn callback that returns the FBB pointer for a given index. */ {{ NS }}_FBB_INPUT_FORMAT_CALLBACK, } {{ NS }}_FBB_Input_Format; /* Forward declaration of the structs (classes). */ #ifdef __cplusplus struct {{ NS }}_Builder; struct {{ NS }}_Serialized; ### for (msg, _) in msgs struct {{ NS }}_Builder_{{ msg }}; struct {{ NS }}_Serialized_{{ msg }}; ### endfor #else typedef struct _{{ NS }}_Builder {{ NS }}_Builder; typedef struct _{{ NS }}_Serialized {{ NS }}_Serialized; ### for (msg, _) in msgs typedef struct _{{ NS }}_Builder_{{ msg }} {{ NS }}_Builder_{{ msg }}; typedef struct _{{ NS }}_Serialized_{{ msg }} {{ NS }}_Serialized_{{ msg }}; ### endfor #endif /* Forward declaration of the main methods. */ /* * Get the tag from the builder */ static inline int {{ ns }}_builder_get_tag(const {{ NS }}_Builder *msg); /* * Get the tag from the serialized version */ static inline int {{ ns }}_serialized_get_tag(const {{ NS }}_Serialized *msg); /* * Get the tag as string */ const char *{{ ns }}_tag_to_string(int tag); /* * Builder - Debug any message * * Generate valid JSON (and almost valid Python - just set null=None before parsing it) * so that it's easier to postprocess with random tools. */ void {{ ns }}_builder_debug(FILE *f, const {{ NS }}_Builder *msg); /* * Serialized - Debug any message * * Generate valid JSON (and almost valid Python - just set null=None before parsing it) * so that it's easier to postprocess with random tools. */ void {{ ns }}_serialized_debug(FILE *f, const {{ NS }}_Serialized *msg); /* * Builder - Measure any message * * Return the length of the serialized form. */ fbb_size_t {{ ns }}_builder_measure(const {{ NS }}_Builder *msg); /* * Builder - Serialize any message to memory * * Takes a buffer that is large enough to hold the serialized form, as guaranteed by a preceding {{ ns }}_builder_measure() call. * * Return the length of the serialized form. */ fbb_size_t {{ ns }}_builder_serialize(const {{ NS }}_Builder *msg, char *dst); /* These are just so that you can "FBB_Builder *" or "FBB_Serialized *" instead of the more generic "void *", * resulting in nicer code. */ #ifdef __cplusplus struct {{ NS }}_Builder { #else typedef struct _{{ NS }}_Builder { #endif int {{ ns }}_tag_; #ifdef __cplusplus inline int get_tag() const { return {{ ns }}_builder_get_tag(this); } inline fbb_size_t measure() const { return {{ ns }}_builder_measure(this); } inline fbb_size_t serialize(char *dst) const { return {{ ns }}_builder_serialize(this, dst); } inline void debug(FILE *f) const { {{ ns }}_builder_debug(f, this); } #endif #ifdef __cplusplus }; #else } {{ NS }}_Builder; #endif #ifdef __cplusplus struct {{ NS }}_Serialized { #else typedef struct _{{ NS }}_Serialized { #endif int {{ ns }}_tag_; #ifdef __cplusplus inline int get_tag() const { return {{ ns }}_serialized_get_tag(this); } inline void debug(FILE *f) const { {{ ns }}_serialized_debug(f, this); } #endif #ifdef __cplusplus }; #else } {{ NS }}_Serialized; #endif #ifdef __cplusplus /* Make sure the layout is the same in C and C++. */ static_assert(std::is_standard_layout_v<{{ NS }}_Serialized>); #endif /* Definition of some of the global functions - the rest are defined in tpl.c. */ /* * Get the tag from the builder */ static inline int {{ ns }}_builder_get_tag(const {{ NS }}_Builder *msg) { return msg->{{ ns }}_tag_; } /* * Get the tag from the serialized version */ static inline int {{ ns }}_serialized_get_tag(const {{ NS }}_Serialized *msg) { return msg->{{ ns }}_tag_; } #ifdef __cplusplus } /* close extern "C" for the inline methods so that we can use C++ function overloading */ #endif ### for (msg, fields) in msgs /****************************************************************************** * {{ msg }} ******************************************************************************/ {# # For each field in the message, we might need to generate 3-4-5-6 or so different methods # depending on its type (setter/getter on the builder, getter on the serialized format, array count # getter, and convenience methods with slightly different signatures). Each such method has a # C-style and a C++-style API, and we need to forward-declare the C-style method in order not to # break a dependency loop. That is, for each field, we need to emit code at 3 different places in # the output file. # # Jinja doesn't seem to have a nice solution for this, and I couldn't find a different template # engine either which would solve this nicely. # # So here we iterate through all the fields of a message tag and decide what methods with what # source code body we'll need, but we don't emit anything yet. We just collect these in the # "builder_funcs" and "serialized_funcs" arrays. Once we've collected everything then we'll emit # them in multiple rounds. # # Each block enclosed between "....." and "^^^^^" markers represents one logical getter or setter, # which will have a C-style and a C++-style API. The temporary multiline variables "comment", # "cfunc" and "cxxfunc" are defined to hold the comment, the C-style definition and the C++-style # definition, the latter one simply calling the C-style implementation. # # I use the wording "C-style" because the API looks like plain old C, but the method might be C or # C++. For consistency even C++ methods have a C-style interface, we might revise this decision at # one point. # # The C-style declaration is automatically created from the C-style definition (it's a trivial # string operation). Theoretically the C++-style definition could also be automatically created # from the C-style definition, but it's not easy, especially with jinja's limited toolset, it's # easier to write them manually for now. # # Finally, in each such block, we encapsulate these in a tuple and append that to builder_funcs or # serialized_funcs. The last member of the tuple must be 'c' or 'c++' specifying the language used # for the C-style API, i.e. if the code is actually C++ then it will be #ifdef'ed accordingly. #} ### set builder_funcs = [] ### set serialized_funcs = [] {# # Builder setters #} ### for (quant, type, var, dbgfn) in fields {% set ctype = "const char *" if type == STRING else "const " + NS + "_Builder *" if type == FBB else type %} ### if type not in [STRING, FBB] ### if quant == REQUIRED {# .......................................................................... #} ### set comment /* * Builder setter - required scalar * {{ type }} {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, {{ type }} value) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->wire.{{ var }}_ = value; #ifdef FB_EXTRA_DEBUG msg->has_{{ var }}_ = true; #endif } ### endset ### set cxxfunc inline void set_{{ var }}({{ type }} value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, value); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### elif quant == OPTIONAL {# .......................................................................... #} ### set comment /* * Builder setter - optional scalar * {{ type }} {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, {{ type }} value) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->wire.{{ var }}_ = value; msg->wire.has_{{ var }}_ = true; } ### endset ### set cxxfunc inline void set_{{ var }}({{ type }} value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, value); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### else {# .......................................................................... #} ### set comment /* * Builder setter - array of scalars * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const {{ type }} *values, fbb_size_t count) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_ = values; msg->wire.{{ var }}_count_ = count; } ### endset ### set cxxfunc inline void set_{{ var }}(const {{ type }} *values, fbb_size_t count) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, values, count); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder setter - array of scalars (C++) * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const std::vector<{{ type }}>& values) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_ = values.data(); msg->wire.{{ var }}_count_ = values.size(); } ### endset ### set cxxfunc inline void set_{{ var }}(const std::vector<{{ type }}>& values) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, values); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### else ### if quant in [REQUIRED, OPTIONAL] ### if type == STRING {# .......................................................................... #} ### set comment /* * Builder setter - required or optional string with length * {{ type }} {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_length({{ NS }}_Builder_{{ msg }} *msg, const char *value, fbb_size_t len) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); #ifdef FB_EXTRA_DEBUG assert(value == NULL || strlen(value) == len); /* if len is specified, it must be the correct value */ #endif msg->{{ var }}_ = value; msg->wire.{{ var }}_len_ = len; } ### endset ### set cxxfunc inline void set_{{ var }}_with_length(const char *value, fbb_size_t len) { {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_length(this, value, len); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder setter - required or optional string * {{ type }} {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const char *value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_length(msg, value, value ? strlen(value) : 0); } ### endset ### set cxxfunc inline void set_{{ var }}(const char *value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, value); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder setter - required or optional string (C++) * {{ type }} {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const std::string& value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_length(msg, value.c_str(), value.length()); } ### endset ### set cxxfunc inline void set_{{ var }}(const std::string& value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, value); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### else {# .......................................................................... #} ### set comment /* * Builder setter - required or optional FBB * {{ type }} {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const {{ NS }}_Builder *value) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_ = value; } ### endset ### set cxxfunc inline void set_{{ var }}(const {{ NS }}_Builder *value) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, value); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### else {# .......................................................................... #} ### set comment /* * Builder setter - array of strings or FBBs with item count * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_count({{ NS }}_Builder_{{ msg }} *msg, {{ ctype }} const *values, fbb_size_t count) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_how_ = {{ NS }}_{{ type|upper }}_INPUT_FORMAT_ARRAY; msg->{{ var }}_.c_array = values; msg->wire.{{ var }}_count_ = count; } ### endset ### set cxxfunc inline void set_{{ var }}_with_count({{ ctype }} const *values, fbb_size_t count) { {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_count(this, values, count); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder setter - array of strings or FBBs * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, {{ ctype }} const *values) { fbb_size_t count = 0; if (values != NULL) { while (values[count] != NULL) count++; } {{ ns }}_builder_{{ msg }}_set_{{ var }}_with_count(msg, values, count); } ### endset ### set cxxfunc inline void set_{{ var }}({{ ctype }} const *values) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, values); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type == STRING {# .......................................................................... #} ### set comment /* * Builder setter - array of strings as cstring_view * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}_cstring_views({{ NS }}_Builder_{{ msg }} *msg, const cstring_view *values, fbb_size_t count) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_how_ = {{ NS }}_STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY; msg->{{ var }}_.cstring_view_array = values; msg->wire.{{ var }}_count_ = count; } ### endset ### set cxxfunc inline void set_{{ var }}_cstring_views(const cstring_view *values, fbb_size_t count) { {{ ns }}_builder_{{ msg }}_set_{{ var }}_cstring_views(this, values, count); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder setter - array of strings as vector (C++) * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const std::vector& values) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_how_ = {{ NS }}_STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY; msg->{{ var }}_.cstring_view_array = values.data(); msg->wire.{{ var }}_count_ = values.size(); } ### endset ### set cxxfunc inline void set_{{ var }}(const std::vector& values) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, values); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif {# .......................................................................... #} ### set comment /* * Builder setter - array of strings or FBBs (C++) * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const std::vector<{{ ctype }}>& values) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_how_ = {{ NS }}_{{ type|upper }}_INPUT_FORMAT_ARRAY; msg->{{ var }}_.c_array = values.data(); msg->wire.{{ var }}_count_ = values.size(); } ### endset ### set cxxfunc inline void set_{{ var }}(const std::vector<{{ ctype }}>& values) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, values); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type == STRING {# .......................................................................... #} ### set comment /* * Builder setter - array of strings as vector (C++) * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}({{ NS }}_Builder_{{ msg }} *msg, const std::vector& values) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_how_ = {{ NS }}_STRING_INPUT_FORMAT_CXX_STRING_ARRAY; msg->{{ var }}_.cxx_string_array = values.data(); msg->wire.{{ var }}_count_ = values.size(); } ### endset ### set cxxfunc inline void set_{{ var }}(const std::vector& values) { {{ ns }}_builder_{{ msg }}_set_{{ var }}(this, values); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif {# .......................................................................... #} ### set comment /* * Builder setter - array of strings or FBBs as an item getter function * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline void {{ ns }}_builder_{{ msg }}_set_{{ var }}_item_fn({{ NS }}_Builder_{{ msg }} *msg, fbb_size_t count, {{ ctype }} (* item_fn) (int idx, const void *user_data{% if type == STRING %}, fbb_size_t *len_out{% endif %}), const void *user_data) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); msg->{{ var }}_how_ = {{ NS }}_{{ type|upper }}_INPUT_FORMAT_CALLBACK; msg->{{ var }}_.callback.item_fn = item_fn; msg->{{ var }}_.callback.user_data = user_data; msg->wire.{{ var }}_count_ = count; } ### endset ### set cxxfunc inline void set_{{ var }}_item_fn(fbb_size_t count, {{ ctype }} (* item_fn) (int idx, const void *user_data{% if type == STRING %}, fbb_size_t *len_out{% endif %}), const void *user_data) { {{ ns }}_builder_{{ msg }}_set_{{ var }}_item_fn(this, count, item_fn, user_data); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### endif ### endfor {# # Builder getters #} ### for (quant, type, var, dbgfn) in fields {% set ctype = "const char *" if type == STRING else "const " + NS + "_Builder *" if type == FBB else type %} ### if quant == OPTIONAL {# .......................................................................... #} ### set comment /* * Builder getter - check if optional field is set * {{ type }} {{ var }} */ ### endset ### set cfunc static inline bool {{ ns }}_builder_{{ msg }}_has_{{ var }}(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); ### if type in [STRING, FBB] return msg->{{ var }}_ != NULL; ### else return msg->wire.has_{{ var }}_; ### endif } ### endset ### set cxxfunc inline bool has_{{ var }}() const { return {{ ns }}_builder_{{ msg }}_has_{{ var }}(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### if quant in [REQUIRED, OPTIONAL] ### if type not in [STRING, FBB] {# .......................................................................... #} ### set comment /* * Builder getter - required or optional scalar * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ type }} {{ ns }}_builder_{{ msg }}_get_{{ var }}(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); ### if quant == OPTIONAL assert(msg->wire.has_{{ var }}_); ### endif return msg->wire.{{ var }}_; } ### endset ### set cxxfunc inline {{ type }} get_{{ var }}() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder getter - pointer to required or optional scalar * {{ type }} {{ var }} */ ### endset ### set cfunc static inline const {{ type }} *{{ ns }}_builder_{{ msg }}_get_{{ var }}_ptr(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); ### if quant == OPTIONAL if (!msg->wire.has_{{ var }}_) { return NULL; } ### endif return &msg->wire.{{ var }}_; } ### endset ### set cxxfunc inline const {{ type }} *get_{{ var }}_ptr() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_ptr(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if quant == OPTIONAL {# .......................................................................... #} ### set comment /* * Builder getter - optional scalar with fallback default * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ type }} {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_fallback(const {{ NS }}_Builder_{{ msg }} *msg, {{ type }} fallback) { return msg->wire.has_{{ var }}_ ? msg->wire.{{ var }}_ : fallback; } ### endset ### set cxxfunc inline {{ type }} get_{{ var }}_with_fallback({{ type }} fallback) const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_fallback(this, fallback); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### else {# .......................................................................... #} ### set comment /* * Builder getter - required or optional string or FBB * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_builder_{{ msg }}_get_{{ var }}(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); return msg->{{ var }}_; } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type == STRING {# .......................................................................... #} ### set comment /* * Builder getter - required or optional string's length * {{ type }} {{ var }} */ ### endset ### set cfunc static inline fbb_size_t {{ ns }}_builder_{{ msg }}_get_{{ var }}_len(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); return msg->wire.{{ var }}_len_; } ### endset ### set cxxfunc inline fbb_size_t get_{{ var }}_len() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_len(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder getter - required or optional string along with its length * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_len(const {{ NS }}_Builder_{{ msg }} *msg, fbb_size_t *len_out) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); *len_out = {{ ns }}_builder_{{ msg }}_get_{{ var }}_len(msg); return {{ ns }}_builder_{{ msg }}_get_{{ var }}(msg); } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}_with_len(fbb_size_t *len_out) const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_len(this, len_out); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder getter - required or optional string (C++, not async-signal-safe) * {{ type }} {{ var }} */ ### endset ### set cfunc static inline std::string {{ ns }}_builder_{{ msg }}_get_{{ var }}_as_string(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(msg->{{ var }}_ != NULL); return std::string(msg->{{ var }}_, msg->wire.{{ var }}_len_); } ### endset ### set cxxfunc inline std::string get_{{ var }}_as_string() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_as_string(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### endif ### else {# .......................................................................... #} ### set comment /* * Builder getter - array item count * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline fbb_size_t {{ ns }}_builder_{{ msg }}_get_{{ var }}_count(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); return msg->wire.{{ var }}_count_; } ### endset ### set cxxfunc inline fbb_size_t get_{{ var }}_count() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_count(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type not in [STRING, FBB] {# .......................................................................... #} ### set comment /* * Builder getter - array of scalars * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline const {{ type }} *{{ ns }}_builder_{{ msg }}_get_{{ var }}(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); return msg->{{ var }}_; } ### endset ### set cxxfunc inline const {{ type }} *get_{{ var }}() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif {# .......................................................................... #} ### set comment /* * Builder getter - one item from an array * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_builder_{{ msg }}_get_{{ var }}_at(const {{ NS }}_Builder_{{ msg }} *msg, fbb_size_t idx) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->wire.{{ var }}_count_); ### if type not in [STRING, FBB] return msg->{{ var }}_[idx]; ### elif type == STRING switch (msg->{{ var }}_how_) { case {{ NS }}_STRING_INPUT_FORMAT_ARRAY: return msg->{{ var }}_.c_array[idx]; case {{ NS }}_STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY: return msg->{{ var }}_.cstring_view_array[idx].c_str; #ifdef __cplusplus case {{ NS }}_STRING_INPUT_FORMAT_CXX_STRING_ARRAY: return msg->{{ var }}_.cxx_string_array[idx].c_str(); #endif case {{ NS }}_STRING_INPUT_FORMAT_CALLBACK: return (msg->{{ var }}_.callback.item_fn)(idx, msg->{{ var }}_.callback.user_data, NULL); } assert(0); return NULL; ### else switch (msg->{{ var }}_how_) { case {{ NS }}_FBB_INPUT_FORMAT_ARRAY: return msg->{{ var }}_.c_array[idx]; case {{ NS }}_FBB_INPUT_FORMAT_CALLBACK: return (msg->{{ var }}_.callback.item_fn)(idx, msg->{{ var }}_.callback.user_data); } assert(0); return NULL; ### endif } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}_at(fbb_size_t idx) const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_at(this, idx); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type == STRING {# .......................................................................... #} ### set comment /* * Builder getter - one item's length from a string array * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline fbb_size_t {{ ns }}_builder_{{ msg }}_get_{{ var }}_len_at(const {{ NS }}_Builder_{{ msg }} *msg, fbb_size_t idx) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->wire.{{ var }}_count_); switch (msg->{{ var }}_how_) { case {{ NS }}_STRING_INPUT_FORMAT_ARRAY: /* This is costly, the requested length is not readily available so we have to compute it. */ return strlen(msg->{{ var }}_.c_array[idx]); case {{ NS }}_STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY: return msg->{{ var }}_.cstring_view_array[idx].length; #ifdef __cplusplus case {{ NS }}_STRING_INPUT_FORMAT_CXX_STRING_ARRAY: return msg->{{ var }}_.cxx_string_array[idx].length(); #endif case {{ NS }}_STRING_INPUT_FORMAT_CALLBACK: { fbb_size_t len; (msg->{{ var }}_.callback.item_fn)(idx, msg->{{ var }}_.callback.user_data, &len); return len; } } assert(0); return 0; } ### endset ### set cxxfunc inline fbb_size_t get_{{ var }}_len_at(fbb_size_t idx) const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_len_at(this, idx); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Builder getter - one item from a string array along with its length * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_len_at(const {{ NS }}_Builder_{{ msg }} *msg, fbb_size_t idx, fbb_size_t *len_out) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->wire.{{ var }}_count_); switch (msg->{{ var }}_how_) { case {{ NS }}_STRING_INPUT_FORMAT_ARRAY: /* This is costly, the requested length is not readily available so we have to compute it. */ *len_out = strlen(msg->{{ var }}_.c_array[idx]); return msg->{{ var }}_.c_array[idx]; case {{ NS }}_STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY: *len_out = msg->{{ var }}_.cstring_view_array[idx].length; return msg->{{ var }}_.cstring_view_array[idx].c_str; #ifdef __cplusplus case {{ NS }}_STRING_INPUT_FORMAT_CXX_STRING_ARRAY: *len_out = msg->{{ var }}_.cxx_string_array[idx].length(); return msg->{{ var }}_.cxx_string_array[idx].c_str(); #endif case {{ NS }}_STRING_INPUT_FORMAT_CALLBACK: return (msg->{{ var }}_.callback.item_fn)(idx, msg->{{ var }}_.callback.user_data, len_out); } assert(0); return NULL; } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}_with_len_at(fbb_size_t idx, fbb_size_t *len_out) const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_with_len_at(this, idx, len_out); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif {# .......................................................................... #} ### set comment /* * Builder getter - array (C++, not async-signal-safe) * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline std::vector<{{ "std::string" if type == STRING else ctype }}> {{ ns }}_builder_{{ msg }}_get_{{ var }}_as_vector(const {{ NS }}_Builder_{{ msg }} *msg) { assert(msg->wire.{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); std::vector<{{ "std::string" if type == STRING else ctype }}> ret; ret.reserve(msg->wire.{{ var }}_count_); for (fbb_size_t idx = 0; idx < msg->wire.{{ var }}_count_; idx++) ret.emplace_back({{ ns }}_builder_{{ msg }}_get_{{ var }}_at(msg, idx)); return ret; } ### endset ### set cxxfunc inline std::vector<{{ "std::string" if type == STRING else ctype }}> get_{{ var }}_as_vector() const { return {{ ns }}_builder_{{ msg }}_get_{{ var }}_as_vector(this); } ### endset ### do builder_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### endfor {# # Serialized getters #} ### for (quant, type, var, dbgfn) in fields {% set ctype = "const char *" if type == STRING else "const " + NS + "_Serialized *" if type == FBB else type %} ### if quant == OPTIONAL {# .......................................................................... #} ### set comment /* * Serialized getter - check if optional field is set * {{ type }} {{ var }} */ ### endset ### set cfunc static inline bool {{ ns }}_serialized_{{ msg }}_has_{{ var }}(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); ### if type in [STRING, FBB] const {{ NS }}_Relptrs_{{ msg }} *relptrs = (const {{ NS }}_Relptrs_{{ msg }} *) ((const {{ NS }}_Serialized_{{ msg }} *) &msg[1]); /* the area immediately followed by the {{ NS }}_Serialized_{{ msg }} structure */ return relptrs->{{ var }}_relptr_ != 0; ### else return msg->has_{{ var }}_; ### endif } ### endset ### set cxxfunc inline bool has_{{ var }}() const { return {{ ns }}_serialized_{{ msg }}_has_{{ var }}(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### if quant in [REQUIRED, OPTIONAL] ### if type not in [STRING, FBB] {# .......................................................................... #} ### set comment /* * Serialized getter - required or optional scalar * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ type }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); ### if quant == OPTIONAL assert(msg->has_{{ var }}_); ### endif return msg->{{ var }}_; } ### endset ### set cxxfunc inline {{ type }} get_{{ var }}() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Serialized getter - pointer to required or optional scalar * {{ type }} {{ var }} */ ### endset ### set cfunc static inline const {{ type }} *{{ ns }}_serialized_{{ msg }}_get_{{ var }}_ptr(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); ### if quant == OPTIONAL if (!msg->has_{{ var }}_) { return NULL; } ### endif return &msg->{{ var }}_; } ### endset ### set cxxfunc inline const {{ type }} *get_{{ var }}_ptr() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_ptr(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if quant == OPTIONAL {# .......................................................................... #} ### set comment /* * Serialized getter - optional scalar with fallback default * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ type }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}_with_fallback(const {{ NS }}_Serialized_{{ msg }} *msg, {{ type }} fallback) { return msg->has_{{ var }}_ ? msg->{{ var }}_ : fallback; } ### endset ### set cxxfunc inline {{ type }} get_{{ var }}_with_fallback({{ type }} fallback) const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_with_fallback(this, fallback); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### else {# .......................................................................... #} ### set comment /* * Serialized getter - required or optional string or FBB * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); const {{ NS }}_Relptrs_{{ msg }} *relptrs = (const {{ NS }}_Relptrs_{{ msg }} *) ((const {{ NS }}_Serialized_{{ msg }} *) &msg[1]); /* the area immediately followed by the {{ NS }}_Serialized_{{ msg }} structure */ if (relptrs->{{ var }}_relptr_ == 0) { ### if quant == REQUIRED assert(relptrs->{{ var }}_relptr_ != 0); ### else return NULL; ### endif } const char *ret = (const char *)msg + relptrs->{{ var }}_relptr_; return ({{ ctype }}) ret; } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type == STRING {# .......................................................................... #} ### set comment /* * Serialized getter - required or optional string's length * {{ type }} {{ var }} */ ### endset ### set cfunc static inline fbb_size_t {{ ns }}_serialized_{{ msg }}_get_{{ var }}_len(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); return msg->{{ var }}_len_; } ### endset ### set cxxfunc inline fbb_size_t get_{{ var }}_len() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_len(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Serialized getter - required or optional string along with its length * {{ type }} {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}_with_len(const {{ NS }}_Serialized_{{ msg }} *msg, fbb_size_t *len_out) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); *len_out = {{ ns }}_serialized_{{ msg }}_get_{{ var }}_len(msg); return {{ ns }}_serialized_{{ msg }}_get_{{ var }}(msg); } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}_with_len(fbb_size_t *len_out) const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_with_len(this, len_out); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Serialized getter - required or optional string (C++, not async-signal-safe) * {{ type }} {{ var }} */ ### endset ### set cfunc static inline std::string {{ ns }}_serialized_{{ msg }}_get_{{ var }}_as_string(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); const char *c_str = {{ ns }}_serialized_{{ msg }}_get_{{ var }}(msg); assert(c_str != NULL); return std::string(c_str, msg->{{ var }}_len_); } ### endset ### set cxxfunc inline std::string get_{{ var }}_as_string() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_as_string(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### endif ### else {# .......................................................................... #} ### set comment /* * Serialized getter - array item count * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline fbb_size_t {{ ns }}_serialized_{{ msg }}_get_{{ var }}_count(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); return msg->{{ var }}_count_; } ### endset ### set cxxfunc inline fbb_size_t get_{{ var }}_count() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_count(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type not in [STRING, FBB] {# .......................................................................... #} ### set comment /* * Serialized getter - array of scalars * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline const {{ type }} *{{ ns }}_serialized_{{ msg }}_get_{{ var }}(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); const {{ NS }}_Relptrs_{{ msg }} *relptrs = (const {{ NS }}_Relptrs_{{ msg }} *) ((const {{ NS }}_Serialized_{{ msg }} *) &msg[1]); /* the area immediately followed by the {{ NS }}_Serialized_{{ msg }} structure */ const char *array = (const char *)msg + relptrs->{{ var }}_relptr_; return (const {{ type }} *) array; } ### endset ### set cxxfunc inline const {{ type }} *get_{{ var }}() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Serialized getter - one item from an array of scalars * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline {{ type }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}_at(const {{ NS }}_Serialized_{{ msg }} *msg, fbb_size_t idx) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->{{ var }}_count_); const {{ NS }}_Relptrs_{{ msg }} *relptrs = (const {{ NS }}_Relptrs_{{ msg }} *) ((const {{ NS }}_Serialized_{{ msg }} *) &msg[1]); /* the area immediately followed by the {{ NS }}_Serialized_{{ msg }} structure */ const void *array_void = (const char *)msg + relptrs->{{ var }}_relptr_; const {{ type }} *array = (const {{ type }} *)array_void; return array[idx]; } ### endset ### set cxxfunc inline {{ type }} get_{{ var }}_at(fbb_size_t idx) const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_at(this, idx); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### else {# .......................................................................... #} ### set comment /* * Serialized getter - one item from an array of strings or FBBs * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}_at(const {{ NS }}_Serialized_{{ msg }} *msg, fbb_size_t idx) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->{{ var }}_count_); /* double jump */ const {{ NS }}_Relptrs_{{ msg }} *relptrs = (const {{ NS }}_Relptrs_{{ msg }} *) ((const {{ NS }}_Serialized_{{ msg }} *) &msg[1]); /* the area immediately followed by the {{ NS }}_Serialized_{{ msg }} structure */ const void *second_relptrs_void = (const char *)msg + relptrs->{{ var }}_relptr_; const fbb_size_t *second_relptrs = (const fbb_size_t *)second_relptrs_void; const char *ret = (const char *)msg + second_relptrs[{{ "2 * " if type == STRING }}idx]; return ({{ ctype }}) ret; } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}_at(fbb_size_t idx) const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_at(this, idx); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### if type == STRING {# .......................................................................... #} ### set comment /* * Serialized getter - one item's length from an array of strings * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline fbb_size_t {{ ns }}_serialized_{{ msg }}_get_{{ var }}_len_at(const {{ NS }}_Serialized_{{ msg }} *msg, fbb_size_t idx) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->{{ var }}_count_); const {{ NS }}_Relptrs_{{ msg }} *relptrs = (const {{ NS }}_Relptrs_{{ msg }} *) ((const {{ NS }}_Serialized_{{ msg }} *) &msg[1]); /* the area immediately followed by the {{ NS }}_Serialized_{{ msg }} structure */ const void *second_relptrs_void = (const char *)msg + relptrs->{{ var }}_relptr_; const fbb_size_t *second_relptrs = (const fbb_size_t *)second_relptrs_void; return second_relptrs[2 * idx + 1]; } ### endset ### set cxxfunc inline fbb_size_t get_{{ var }}_len_at(fbb_size_t idx) const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_len_at(this, idx); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} {# .......................................................................... #} ### set comment /* * Serialized getter - one item along with its length from an array of strings * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline {{ ctype }} {{ ns }}_serialized_{{ msg }}_get_{{ var }}_with_len_at(const {{ NS }}_Serialized_{{ msg }} *msg, fbb_size_t idx, fbb_size_t *len_out) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); assert(idx < msg->{{ var }}_count_); *len_out = {{ ns }}_serialized_{{ msg }}_get_{{ var }}_len_at(msg, idx); return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_at(msg, idx); } ### endset ### set cxxfunc inline {{ ctype }} get_{{ var }}_with_len_at(fbb_size_t idx, fbb_size_t *len_out) const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_with_len_at(this, idx, len_out); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### endif {# .......................................................................... #} ### set comment /* * Serialized getter - array (C++, not async-signal-safe) * {{ type }}[] {{ var }} */ ### endset ### set cfunc static inline std::vector<{{ "std::string" if type == STRING else ctype }}> {{ ns }}_serialized_{{ msg }}_get_{{ var }}_as_vector(const {{ NS }}_Serialized_{{ msg }} *msg) { assert(msg->{{ ns }}_tag_ == {{ NS }}_TAG_{{ msg }}); std::vector<{{ "std::string" if type == STRING else ctype }}> ret; ret.reserve( msg->{{ var }}_count_); for (fbb_size_t idx = 0; idx < msg->{{ var }}_count_; idx++) { ret.emplace_back({{ ns }}_serialized_{{ msg }}_get_{{ var }}_at(msg, idx) {% if type == STRING %}, (size_t){{ ns }}_serialized_{{ msg }}_get_{{ var }}_len_at(msg, idx) {% endif %}); } return ret; } ### endset ### set cxxfunc inline std::vector<{{ "std::string" if type == STRING else ctype }}> get_{{ var }}_as_vector() const { return {{ ns }}_serialized_{{ msg }}_get_{{ var }}_as_vector(this); } ### endset ### do serialized_funcs.append((comment, cfunc, cxxfunc, 'c++')) {# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #} ### endif ### endfor {# # We've built up "builder_funcs" and "serialized_funcs" for all the fields of the current message # tag. Time to generate some output. #} /* Forward declaration of the functions of the C-style API for this message tag. */ static inline void {{ ns }}_builder_{{ msg }}_init({{ NS }}_Builder_{{ msg }} *msg); ### for (comment, cfunc, cxxfunc, lang) in builder_funcs + serialized_funcs ### if lang == 'c++' #ifdef __cplusplus ### endif {{ comment }} {{ cfunc.split(" {")[0] }}; ### if lang == 'c++' #endif ### endif ### endfor ### set jinjans = namespace(has_relptr=False) ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY or type in [STRING, FBB] ### set jinjans.has_relptr = True ### endif ### endfor /***** Generic *****/ /* * Wire buffer, common to the Builder as well */ #ifdef __cplusplus struct {{ NS }}_Serialized_{{ msg }} { #else typedef struct _{{ NS }}_Serialized_{{ msg }} { #endif /* It's important that the tag is the very first field */ int {{ ns }}_tag_; /* Required and optional scalar fields */ ### for (quant, type, var, dbgfn) in fields ### if quant != ARRAY and type not in [STRING, FBB] {{ type }} {{ var }}_; ### endif ### endfor /* Required and optional string and FBB fields */ ### for (quant, type, var, dbgfn) in fields ### if quant != ARRAY and type in [STRING, FBB] ### if type == STRING fbb_size_t {{ var }}_len_; ### endif ### endif ### endfor /* Arrays of anything */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY fbb_size_t {{ var }}_count_; ### endif ### endfor /* Whether optional scalars have been set */ ### for (quant, type, var, dbgfn) in fields ### if quant == OPTIONAL and type not in [STRING, FBB] bool has_{{ var }}_ : 1; ### endif ### endfor #ifdef __cplusplus /* C++-style convenience member methods. They all just call their equivalent C counterpart. */ ### for (comment, cfunc, cxxfunc, lang) in serialized_funcs {{ comment | indent(2) }} {{ cxxfunc | indent(2) }} ### endfor #endif /* __cplusplus */ #ifdef __cplusplus }; #else } {{ NS }}_Serialized_{{ msg }}; #endif /* __cplusplus */ #ifdef __cplusplus /* Make sure the layout is the same in C and C++. */ static_assert(std::is_standard_layout_v<{{ NS }}_Serialized_{{ msg }}>); #endif /* * Placed in the serialized format after {{ NS }}_Serialized_{{ msg }}, * containing the direct relptrs and the first hops of the indirect relptrs */ ### if jinjans.has_relptr typedef struct _{{ NS }}_Relptrs_{{ msg }} { ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY or type in [STRING, FBB] fbb_size_t {{ var }}_relptr_; ### endif ### endfor } {{ NS }}_Relptrs_{{ msg }}; ### else /* Empty {{ NS }}_Relptrs_{{ msg }} not defined becase C and C++ would disagree on its size */ ### endif /* * Builder */ #ifdef __cplusplus struct {{ NS }}_Builder_{{ msg }} { #else typedef struct _{{ NS }}_Builder_{{ msg }} { #endif /* The part of the message that's common with the serialized format */ {{ NS }}_Serialized_{{ msg }} wire; /* Arrays of scalars (pointers only, owned by the caller) */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY and type not in [STRING, FBB] const {{ type }} *{{ var }}_; ### endif ### endfor /* Single strings and FBBs (pointers only, owned by the caller */ ### for (quant, type, var, dbgfn) in fields ### if quant != ARRAY ### if type == STRING const char *{{ var }}_; ### elif type == FBB const {{ NS }}_Builder *{{ var }}_; ### endif ### endif ### endfor /* Arrays of strings and FBBs (pointers only, owned by the caller) */ ### for (quant, type, var, dbgfn) in fields ### if quant == ARRAY ### if type == STRING /* In what format do we have the strings */ {{ NS }}_String_Input_Format {{ var }}_how_; union { /* For STRING_INPUT_FORMAT_C_ARRAY */ const char * const *c_array; /* For STRING_INPUT_FORMAT_CSTRING_VIEW_ARRAY */ const cstring_view *cstring_view_array; #ifdef __cplusplus /* For STRING_INPUT_FORMAT_CXX_STRING_ARRAY */ const std::string *cxx_string_array; #endif /* For STRING_INPUT_FORMAT_CALLBACK */ struct { /* Function to get the Nth item of the string array */ const char * (*item_fn) (int idx, const void *user_data, fbb_size_t *len_out); /* Arbitrary pointer passed to item_fn */ const void *user_data; } callback; } {{ var }}_; ### elif type == FBB /* In what format do we have the strings */ {{ NS }}_FBB_Input_Format {{ var }}_how_; union { /* For FBB_INPUT_FORMAT_ARRAY */ const {{ NS }}_Builder * const *c_array; /* For FBB_INPUT_FORMAT_CALLBACK */ struct { /* Function to get the Nth item of the FBB array */ const {{ NS }}_Builder * (*item_fn) (int idx, const void *user_data); /* Arbitrary pointer passed to item_fn */ const void *user_data; } callback; } {{ var }}_; ### endif ### endif ### endfor #ifdef FB_EXTRA_DEBUG /* Whether required scalars have been set */ ### for (quant, type, var, dbgfn) in fields ### if type not in [STRING, FBB] and quant == REQUIRED bool has_{{ var }}_ : 1; ### endif ### endfor #endif #ifdef __cplusplus /* C++-style convenience member methods. Except for the constructor, they all just call their equivalent C counterpart. */ /* Constructor that also initializes the object. */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Weffc++" /* suppress warning about not initializing the member fields, we'll memset() them */ {{ NS }}_Builder_{{ msg }}() { init(); } #pragma GCC diagnostic pop /* Standalone init, you need it in case you allocated the memory area independently and cast it to a Builder object. */ inline void init() { {{ ns }}_builder_{{ msg }}_init(this); } ### for (comment, cfunc, cxxfunc, lang) in builder_funcs {{ comment | indent(2) }} {{ cxxfunc | indent(2) }} ### endfor #endif /* __cplusplus */ #ifdef __cplusplus }; #else } {{ NS }}_Builder_{{ msg }}; #endif /* __cplusplus */ /* * Builder: Initialize, set tag */ static inline void {{ ns }}_builder_{{ msg }}_init({{ NS }}_Builder_{{ msg }} *msg) { /* Zero out even the padding / unused bits to avoid random garbage over the wire or in stored values. * FIXME This should be followed by value-initializing to 0 for types where the value 0 isn't represented * by all-zero bits, e.g. float/double, but theoretically also integer 0 and nullptr on some architectures. */ /* Casting to suppress -Wclass-memaccess. */ memset((void *) msg, 0, sizeof(*msg)); msg->wire.{{ ns }}_tag_ = {{ NS }}_TAG_{{ msg }}; } ### for (comment, cfunc, cxxfunc, lang) in builder_funcs + serialized_funcs ### if lang == 'c++' #ifdef __cplusplus ### endif {{ comment }} {{ cfunc }} ### if lang == 'c++' #endif ### endif ### endfor ### endfor #pragma GCC diagnostic pop /* -Wcast-align */ #endif /* {{ NS }}_H */ firebuild-0.8.2/src/common/fbb/tpl_decode.c000066400000000000000000000050101447164520700205520ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template to generate {{ ns }}_decode.c. #} {# ------------------------------------------------------------------ #} /* Auto-generated by generate_fbb, do not edit */ {# Well, not here, #} {# this is the manually edited template file, #} {# placing this message in the output. #} #include #include #include #include #include #include #include #include #include "./{{ ns }}.h" int main(int argc, const char *argv[]) { if (argc != 2) { fprintf(stderr, "Usage: %s filename\n", argv[0]); exit(1); } int fd = open(argv[1], O_RDONLY); if (fd < 0) { fprintf(stderr, "open failed: %s\n", strerror(errno)); exit(1); } struct stat st; if (fstat(fd, &st) < 0) { fprintf(stderr, "fstat failed: %s\n", strerror(errno)); exit(1); } if (st.st_size == 0) { fprintf(stderr, "input file is empty\n"); exit(1); } void *p = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0); if (p == MAP_FAILED) { fprintf(stderr, "mmap failed: %s\n", strerror(errno)); exit(1); } {{ ns }}_serialized_debug(stderr, p); return 0; } firebuild-0.8.2/src/common/fbbcomm.def000066400000000000000000000704351447164520700176550ustar00rootroot00000000000000# Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # This is the definition of the FBB format used for communication # between the interceptor and the supervisor. # This is a Python dictionary, to be read and processed by "generate_fbb". { "types_with_custom_debugger": ["mode_t"], "varnames_with_custom_debugger": ["error_no"], "extra_c": """ /* Debugger method for mode_t fields. */ static void fbbcomm_debug_mode_t(FILE *f, mode_t mode) { fprintf(f, "\\""); debug_mode_t(f, mode); fprintf(f, "\\""); } /* Debugger method for error_no values. */ static void fbbcomm_debug_error_no(FILE *f, int error_no) { fprintf(f, "\\""); debug_error_no(f, error_no); fprintf(f, "\\""); } /* Debugger method for fields that represent wstatus values. */ static void fbbcomm_debug_wstatus(FILE *f, int wstatus, bool is_serialized, const void *fbb) { (void)is_serialized; /* unused */ (void)fbb; /* unused */ fprintf(f, "\\""); debug_wstatus(f, wstatus); fprintf(f, "\\""); } /* Debugger method for int fields that represent open()'s O_* flags. */ static void fbbcomm_debug_open_flags(FILE *f, int flags, bool is_serialized, const void *fbb) { (void)is_serialized; /* unused */ (void)fbb; /* unused */ fprintf(f, "\\""); debug_open_flags(f, flags); fprintf(f, "\\""); } /* Debugger method for int fields that represent AT_* flags. */ static void fbbcomm_debug_at_flags(FILE *f, int flags, bool is_serialized, const void *fbb) { (void)is_serialized; /* unused */ (void)fbb; /* unused */ fprintf(f, "\\""); debug_at_flags(f, flags); fprintf(f, "\\""); } /* Debugger method for int fields that represent fcntl()'s F_* command. */ static void fbbcomm_debug_fcntl_cmd(FILE *f, int cmd, bool is_serialized, const void *fbb) { (void)is_serialized; /* unused */ (void)fbb; /* unused */ fprintf(f, "\\""); debug_fcntl_cmd(f, cmd); fprintf(f, "\\""); } /* Debugger method for int fields that represent fcntl()'s argument or non-error return value, * which could be FD_* or O_* or other flags depending on the command. */ static void fbbcomm_debug_fcntl_arg_or_ret(FILE *f, int arg_or_ret, bool is_serialized, const void *fbb) { int cmd = is_serialized ? fbbcomm_serialized_fcntl_get_cmd((FBBCOMM_Serialized_fcntl *)fbb) : fbbcomm_builder_fcntl_get_cmd((FBBCOMM_Builder_fcntl *)fbb); fprintf(f, "\\""); debug_fcntl_arg_or_ret(f, cmd, arg_or_ret); fprintf(f, "\\""); } /* Debugger method for int fields that represent CLONE_* flags. */ static void fbbcomm_debug_clone_flags(FILE *f, int flags, bool is_serialized, const void *fbb) { (void)is_serialized; /* unused */ (void)fbb; /* unused */ fprintf(f, "\\""); debug_clone_flags(f, flags); fprintf(f, "\\""); } """, "extra_h": """ #include "common/debug_sysflags.h" """, "tags": [ # interceptor library queries Firebuild supervisor if it can shortcut execution of the process ("scproc_query", [ # process id (OPTIONAL, "pid_t", "pid"), # parent pid (OPTIONAL, "pid_t", "ppid"), # working dir process started in (OPTIONAL, STRING, "cwd"), # only argv, sending argc would be redundant (ARRAY, STRING, "arg"), # environment variables in unprocessed NAME=value form (ARRAY, STRING, "env_var"), # umask (REQUIRED, "mode_t", "umask"), # full path of the binary, it is expected to be in absolute and canonical form # fds (2: [, ]) likely being used by GNU Make's jobserver (ARRAY, "int", "jobserver_fds"), # full path of the binary (OPTIONAL, STRING, "executable"), # original executed_path converted to canonical and absolute path # set only when the original executed path is different from executable (OPTIONAL, STRING, "executed_path"), # pathname used to execute the program # set only if it is relative or not canonical path, i.e. it differs from executed_path (OPTIONAL, STRING, "original_executed_path"), # loaded shared libraries in the beginning (ARRAY, STRING, "libs"), # interceptor's version (OPTIONAL, STRING, "version"), ]), # Submessage within "scproc_resp". Contains the list of client-side fds (dups of each other) # where to reopen one particular attached fd. # Does not contain "flags": the supervisor sets up fcntl(..., F_SETFL, ...) correctly and the # interceptor knows the desired CLOEXEC state. ("scproc_resp_reopen_fd", [ (ARRAY, "int", "fds"), ]), # Firebuild supervisor's response with details of shortcutting. # Unlike most others, this message type is used in the supervisor->interceptor direction of the # communication. Nevertheless, it resides in the same namespace. # Ancillary data (SCM_RIGHTS) contains the fds to reopen, excatly as many as the number of items # in "reopen_fds". ("scproc_resp", [ (REQUIRED, "bool", "shortcut"), (OPTIONAL, "int", "exit_status"), # disable interception and remove libfirebuild from LD_PRELOAD (OPTIONAL, "bool", "dont_intercept"), # makes sense only for shortcut = false (OPTIONAL, "int32_t", "debug_flags"), # Client-side fds to reopen to, messages of type "scproc_resp_reopen_fd". # Each item in this array corresponds to one ancillary fd. (ARRAY, FBB, "reopen_fds"), # The inherited seekable fds that were appended to while shortcutting; # the interceptor needs to seek forward in them. (ARRAY, "int", "fds_appended_to"), # File backed seekable fds that could be seeked to the end or not. # If they are not at the end the interceptor must send back an inherited_fd_offset message. (ARRAY, "int", "seekable_fds"), # Size of the backed seekable fds as the supervisor knows it. (ARRAY, "int64_t", "seekable_fds_size"), ]), # The inherited fd's offset at the start of the process ("inherited_fd_offset", [ (REQUIRED, "int", "fd"), (REQUIRED, "int64_t", "offset"), ]), # Those function calls are not handled specially in interceptor lib and # are reported once per process to supervisor ("gen_call", [ # function name (REQUIRED, STRING, "call"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), # Firebuild error ("fb_error", [ # error message (REQUIRED, STRING, "msg"), ]), # Firebuild debugging message ("fb_debug", [ # error message (REQUIRED, STRING, "msg"), ]), ("fcntl", [ # file descriptor (OPTIONAL, "int", "fd"), # command (OPTIONAL, "int", "cmd", "fbbcomm_debug_fcntl_cmd"), # arg, if present (OPTIONAL, "int", "arg", "fbbcomm_debug_fcntl_arg_or_ret"), # return value, depends on cmd (OPTIONAL, "int", "ret", "fbbcomm_debug_fcntl_arg_or_ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("ioctl", [ # file descriptor (OPTIONAL, "int", "fd"), # command (OPTIONAL, "unsigned long", "cmd"), # return value, depends on cmd (OPTIONAL, "int", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("pre_open", [ # dir file descriptor for openat() (OPTIONAL, "int", "dirfd"), # file path (OPTIONAL, STRING, "pathname"), ]), ("open", [ # dir file descriptor for openat() (OPTIONAL, "int", "dirfd"), # file path (OPTIONAL, STRING, "pathname"), # flags, decoding is left for Firebuild supervisor (REQUIRED, "int", "flags", "fbbcomm_debug_open_flags"), # mode if (flags & O_CREAT), decoding is left for Firebuild supervisor (OPTIONAL, "mode_t", "mode"), # return value, the file descriptor if != -1 (OPTIONAL, "int", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), # pre_open message has been sent before this message (REQUIRED, "bool", "pre_open_sent"), (OPTIONAL, "bool", "tmp_file"), ]), ("freopen", [ # file path, can be NULL (OPTIONAL, STRING, "pathname"), # flags, decoding is left for Firebuild supervisor (OPTIONAL, "int", "flags", "fbbcomm_debug_open_flags"), # file descriptor associated to the stream to be reopened (OPTIONAL, "int", "oldfd"), # return value, the file descriptor if != -1 (OPTIONAL, "int", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), # pre_open message has been sent before this message (REQUIRED, "bool", "pre_open_sent"), ]), ("chdir", [ # directory path (OPTIONAL, STRING, "pathname"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("fchdir", [ # directory fd (OPTIONAL, "int", "fd"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("readlink", [ # dir file descriptor for readlinkat() (OPTIONAL, "int", "dirfd"), # path name (OPTIONAL, STRING, "pathname"), # buffer size (OPTIONAL, "size_t", "bufsiz"), # returned path (OPTIONAL, STRING, "ret_target"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("rename", [ # old dir file descriptor for renameat[2]() (OPTIONAL, "int", "olddirfd"), # path to old file (OPTIONAL, STRING, "oldpath"), # new dir file descriptor for renameat[2]() (OPTIONAL, "int", "newdirfd"), # path to new file (OPTIONAL, STRING, "newpath"), # flags for renmaeat2() (OPTIONAL, "unsigned int", "flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("faccessat", [ # dir file descriptor (OPTIONAL, "int", "dirfd"), # path to file (OPTIONAL, STRING, "pathname"), # access mode (NOT related to "mode_t") (REQUIRED, "int", "mode"), # flags (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("fstatat", [ # fstat()'s fd, or fstatat()'s dirfd (OPTIONAL, "int", "fd"), # path to file, except for fstat() (OPTIONAL, STRING, "pathname"), # it could be lstat() encoded as AT_SYMLINK_NOFOLLOW or fstatat(..., flags) (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # Returned file type and mode (OPTIONAL, "mode_t", "st_mode"), # Returned file size (OPTIONAL, "off_t", "st_size"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("mkdir", [ # dir file descriptor for mkdirat() (OPTIONAL, "int", "dirfd"), # dir path (OPTIONAL, STRING, "pathname"), # mode (REQUIRED, "mode_t", "mode"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), (OPTIONAL, "bool", "tmp_dir"), ]), ("rmdir", [ # dir path (OPTIONAL, STRING, "pathname"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), # pre_open message has been sent before this message (REQUIRED, "bool", "pre_open_sent"), ]), ("close", [ # file descriptor (OPTIONAL, "int", "fd"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("closefrom", [ (REQUIRED, "int", "lowfd"), ]), ("close_range", [ (REQUIRED, "unsigned int", "first"), (REQUIRED, "unsigned int", "last"), (REQUIRED, "int", "flags"), (OPTIONAL, "int", "error_no"), ]), ("umask", [ # The new mask (REQUIRED, "mode_t", "mask"), # The old mask (REQUIRED, "mode_t", "ret"), ]), ("fchmodat", [ # fchmod()'s fd, or fchmodat()'s dirfd (OPTIONAL, "int", "fd"), # file path, except for fchmod() (OPTIONAL, STRING, "pathname"), # mode (REQUIRED, "mode_t", "mode"), # flags for fchmodat(), AT_SYMLINK_NOFOLLOW for lchmod() (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("fchownat", [ # fchown()'s fd, or fchownat()'s dirfd (OPTIONAL, "int", "fd"), # file path (OPTIONAL, STRING, "pathname"), # uid (OPTIONAL, "uid_t", "owner"), # gid (OPTIONAL, "gid_t", "group"), # flags for fchownat(), AT_SYMLINK_NOFOLLOW for lchown() (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("unlink", [ # dir file descriptor for unlinkat() (OPTIONAL, "int", "dirfd"), # path name (OPTIONAL, STRING, "pathname"), # flags for unlinkat() (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), # pre_open message has been sent before this message (REQUIRED, "bool", "pre_open_sent"), ]), ("link", [ # old dir file descriptor for linkat() (OPTIONAL, "int", "olddirfd"), # old file path (OPTIONAL, STRING, "oldpath"), # new dir file descriptor for linkat() (OPTIONAL, "int", "newdirfd"), # new file path (OPTIONAL, STRING, "newpath"), # flags for linkat() (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("symlink", [ # old file path (OPTIONAL, STRING, "target"), # new dir file descriptor for symlinkat() (OPTIONAL, "int", "newdirfd"), # new file path (OPTIONAL, STRING, "newpath"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("lockf", [ # file fd (OPTIONAL, "int", "fd"), # lock command (OPTIONAL, "int", "cmd"), # file range (OPTIONAL, "off_t", "len"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("clock_gettime", [ # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("utime", [ # ..at(), like utimensat (OPTIONAL, "int", "dirfd"), # file name (OPTIONAL, STRING, "pathname"), # all timestamps should be set to current time (REQUIRED, "bool", "all_utime_now"), # flags for utimensat(), AT_SYMLINK_NOFOLLOW for lutimes() (OPTIONAL, "int", "flags", "fbbcomm_debug_at_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("futime", [ # file fd (REQUIRED, "int", "fd"), # all timestamps should be set to current time (REQUIRED, "bool", "all_utime_now"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), # Step 1/3 of intercepting a pipe() or pipe2() call. See #656 for design rationale. # Request the supervisor to create an intercepted unnamed pipe for the interceptor. ("pipe_request", [ (OPTIONAL, "int", "flags", "fbbcomm_debug_open_flags"), ]), # Step 2/3 of intercepting a pipe() or pipe2() call. This message is sent from the supervisor # to the interceptor, with the two fds attached as ancillary data (SCM_RIGHTS). ("pipe_created", [ # if an error occurred in the supervisor while creating the pipe (OPTIONAL, "int", "error_no"), ]), # Step 3/3 of intercepting a pipe() or pipe2() call. # Notify the supervisor about the fd numbers in the interceptor. ("pipe_fds", [ # The fd used for reading (REQUIRED, "int", "fd0"), # The fd used for writing (REQUIRED, "int", "fd1"), ]), # for dup ("dup", [ # old file fd (REQUIRED, "int", "oldfd"), # new fd (REQUIRED, "int", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), # for dup2 and dup3 ("dup3", [ # old file fd (REQUIRED, "int", "oldfd"), # new file fd (REQUIRED, "int", "newfd"), # flags (OPTIONAL, "int", "flags", "fbbcomm_debug_open_flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("dlopen", [ # file path (OPTIONAL, STRING, "filename"), # flag, decoding is left for Firebuild supervisor (OPTIONAL, "int", "flag"), # absolute filename where (if) the library was found (OPTIONAL, STRING, "absolute_filename"), # dlopen() does not set errno. Also, as per #920, we cannot get # the error string in a simple way without altering the # intercepted process's behavior and without nasty hacks. So let's # just go with a simple boolean denoting success vs. failure. (REQUIRED, "bool", "error"), ]), ("memfd_create", [ # name (REQUIRED, STRING, "name"), # flags, decoding is left for Firebuild supervisor (REQUIRED, "int", "flags"), # return value, the file descriptor if != -1 (REQUIRED, "int", "ret"), ]), ("timerfd_create", [ (REQUIRED, "int", "ret"), # flags, decoding is left for Firebuild supervisor (REQUIRED, "int", "flags"), ]), ("epoll_create", [ # flags for epoll_create1(), decoding is left for Firebuild supervisor (OPTIONAL, "int", "flags"), (REQUIRED, "int", "ret"), ]), ("eventfd", [ # flags, decoding is left for Firebuild supervisor (REQUIRED, "int", "flags"), (REQUIRED, "int", "ret"), ]), ("signalfd", [ # old file fd (REQUIRED, "int", "fd"), # flags, decoding is left for Firebuild supervisor (REQUIRED, "int", "flags"), (REQUIRED, "int", "ret"), ]), ("seccomp", [ # always EINVAL at the moment, since the original call is not called (REQUIRED, "int", "error_no"), ]), ("exec", [ # file to execute (OPTIONAL, STRING, "file"), # file fd to execute, in case of fexecve() (OPTIONAL, "int", "fd"), # dir fd to execute, in case of execveat() (OPTIONAL, "int", "dirfd"), # argv[] (ARRAY, STRING, "arg"), # envp[] (ARRAY, STRING, "env"), # true, in case of execvp()/execvpe() (OPTIONAL, "bool", "with_p"), # PATH, or confstr(_CS_PATH) if PATH is not set (OPTIONAL, STRING, "path"), # user CPU time in microseconds since last exec() (REQUIRED, "int64_t", "utime_u"), # system CPU time in microseconds since laste exec() (REQUIRED, "int64_t", "stime_u"), ]), ("exec_failed", [ # error no., when ret = -1 (REQUIRED, "int", "error_no"), ]), # system(3) ("system", [ # command, only SystemRet is sent when command was NULL (OPTIONAL, STRING, "cmd"), # return value is sent in SystemRet ]), ("system_ret", [ # command, not present when it was NULL (OPTIONAL, STRING, "cmd"), # return value (REQUIRED, "int", "ret", "fbbcomm_debug_wstatus"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), # popen(3) ("popen", [ # command (OPTIONAL, STRING, "cmd"), # type, represented as flags (REQUIRED, "int", "type_flags", "fbbcomm_debug_open_flags"), ]), ("popen_parent", [ # return value (REQUIRED, "int", "fd"), ]), # This message goes from the supervisor to the interceptor, ACKing the "popen_parent" message. # The FBB message is empty, but there's the pipe's reopened fd attached as ancillary data. ("popen_fd", [ ]), ("popen_failed", [ # command, to let the supervisor remove it from expected_children (OPTIONAL, STRING, "cmd"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("pclose", [ # file descriptor (REQUIRED, "int", "fd"), # return value (REQUIRED, "int", "ret", "fbbcomm_debug_wstatus"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), # This is not used as a toplevel message, but in the "file_actions" field of a # "posix_spawn_parent" message, corresponding to an earlier posix_spawn_file_actions_addopen() call. ("posix_spawn_file_action_open", [ (REQUIRED, "int", "fd"), # Note: path is not absolute! (REQUIRED, STRING, "pathname"), (REQUIRED, "int", "flags", "fbbcomm_debug_open_flags"), (REQUIRED, "mode_t", "mode"), ]), # This is not used as a toplevel message, but in the "file_actions" field of a # "posix_spawn_parent" message, corresponding to an earlier posix_spawn_file_actions_addclose() call. ("posix_spawn_file_action_close", [ (REQUIRED, "int", "fd"), ]), # This is not used as a toplevel message, but in the "file_actions" field of a # "posix_spawn_parent" message, corresponding to an earlier posix_spawn_file_actions_addclosefrom_np() call. ("posix_spawn_file_action_closefrom", [ (REQUIRED, "int", "lowfd"), ]), # This is not used as a toplevel message, but in the "file_actions" field of a # "posix_spawn_parent" message, corresponding to an earlier posix_spawn_file_actions_adddup2() call. ("posix_spawn_file_action_dup2", [ (REQUIRED, "int", "oldfd"), (REQUIRED, "int", "newfd"), ]), # This is not used as a toplevel message, but in the "file_actions" field of a # "posix_spawn_parent" message, corresponding to an earlier posix_spawn_file_actions_addchdir_np() call. ("posix_spawn_file_action_chdir", [ (REQUIRED, STRING, "pathname"), ]), # This is not used as a toplevel message, but in the "file_actions" field of a # "posix_spawn_parent" message, corresponding to an earlier posix_spawn_file_actions_addfchdir_np() call. ("posix_spawn_file_action_fchdir", [ (REQUIRED, "int", "fd"), ]), # posix_spawn[p](3) ("posix_spawn", [ # command (OPTIONAL, STRING, "file"), # only argv, sending argc would be redundant (ARRAY, STRING, "arg"), # environment variables in unprocessed NAME=value form (ARRAY, STRING, "env"), # posix_spawn_file_actions_t, each action is an FBB message of one of # posix_spawn_file_action_* (ARRAY, FBB, "file_actions"), # spawn or spawnp (REQUIRED, "bool", "is_spawnp"), ]), ("posix_spawn_parent", [ # command args repeated, to let the supervisor remove it from expected_child (ARRAY, STRING, "arg"), # posix_spawn_file_actions_t, each action is an FBB message of one of # posix_spawn_file_action_* (ARRAY, FBB, "file_actions"), # child's process id (REQUIRED, "pid_t", "pid"), ]), ("posix_spawn_failed", [ # command args repeated, to let the supervisor remove it from expected_child (ARRAY, STRING, "arg"), # posix_spawn_file_actions_t, each action is an FBB message of one of # posix_spawn_file_action_* (ARRAY, FBB, "file_actions"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("wait", [ # pid whose termination the parent has successfully waited for (REQUIRED, "pid_t", "pid"), # not set for waitid (OPTIONAL, "int", "wstatus", "fbbcomm_debug_wstatus"), # only set for waitid (OPTIONAL, "int", "si_status"), # only set for waitid (OPTIONAL, "int", "si_code"), ]), ("sysconf", [ # name (OPTIONAL, "int", "name"), # value (OPTIONAL, "long", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("gethostname", [ # name (OPTIONAL, STRING, "name"), # value (OPTIONAL, "size_t", "len"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("getdomainname", [ # name (OPTIONAL, STRING, "name"), # value (OPTIONAL, "size_t", "len"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("truncate", [ # name (REQUIRED, STRING, "pathname"), (REQUIRED, "long", "length"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("pathconf", [ # path name (OPTIONAL, STRING, "path"), # option name (OPTIONAL, "int", "name"), # option value (OPTIONAL, "long", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("fpathconf", [ # file fd (OPTIONAL, "int", "fd"), # option name (OPTIONAL, "int", "name"), # option value (OPTIONAL, "long", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), # The first time the process attempts to read (including the recv() family) from the given inherited fd. # Also re-sent once with is_pread=true if that value was false the first time. ("read_from_inherited", [ # file fd (REQUIRED, "int", "fd"), # Whether the read happened at an explicit offset with pread() and friends, # rather than at the file's current offset in a way that advances this offset. # Also false if preadv2() with offset == -1 mimics plain old sequential read(). (REQUIRED, "bool", "is_pread"), ]), # The first time the process attempts to write (including the send() family) to the given inherited fd. # Also re-sent once with is_pwrite=true if that value was false the first time. ("write_to_inherited", [ # file fd (REQUIRED, "int", "fd"), # Whether the write happened at an explicit offset with pwrite() and friends, # rather than at the file's current offset in a way that advances this offset. # Also false if pwritev2() with offset == -1 mimics plain old sequential write(). (REQUIRED, "bool", "is_pwrite"), ]), # The first time the process attempts to query or change the seek offset. # Also re-sent once with modify_offset=true if that value was false the first time. ("seek_in_inherited", [ # file fd (REQUIRED, "int", "fd"), # Whether the operation requested to change the offset (REQUIRED, "bool", "modify_offset"), ]), # Received some file descriptors via a recv[m]msg() and SCM_RIGHTS. ("recvmsg_scm_rights", [ (REQUIRED, "bool", "cloexec"), (ARRAY, "int", "fds"), ]), # Resource usage collected at exit. The exit status will be collected by the wait() in the parent ("rusage", [ # user CPU time in microseconds (REQUIRED, "int64_t", "utime_u"), # system CPU time in microseconds (REQUIRED, "int64_t", "stime_u"), ]), # fork()'s child ("fork_child", [ # process id (REQUIRED, "pid_t", "pid"), # process parent id (REQUIRED, "pid_t", "ppid"), ]), # process that called fork() ("fork_parent", [ ]), # disables interception and shortcutting ("clone", [ (REQUIRED, "int", "flags", "fbbcomm_debug_clone_flags"), ]), # reading from /dev/urandom is allowed (GRND_RANDOM flag not set) ("getrandom", [ (OPTIONAL, "int", "flags"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("socket", [ (REQUIRED, "int", "domain"), (REQUIRED, "int", "type"), (REQUIRED, "int", "protocol"), # return value, the file descriptor if != -1 (OPTIONAL, "int", "ret"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("socketpair", [ (REQUIRED, "int", "domain"), (REQUIRED, "int", "type"), (REQUIRED, "int", "protocol"), (OPTIONAL, "int", "fd0"), (OPTIONAL, "int", "fd1"), # error no., when ret = -1 (OPTIONAL, "int", "error_no"), ]), ("statfs", [ (OPTIONAL, STRING, "pathname"), (OPTIONAL, "int", "error_no"), ]), ] } firebuild-0.8.2/src/common/firebuild_common.c000066400000000000000000000160201447164520700212370ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include "common/firebuild_common.h" #ifdef __cplusplus extern "C" { #endif void cstring_view_array_init(cstring_view_array *array) { memset(array, 0, sizeof(*array)); } /* Does NOT deep copy the string */ void cstring_view_array_append(cstring_view_array *array, char *s) { if (array->size_alloc == 0) { array->size_alloc = 16 /* whatever */; array->p = malloc(sizeof(cstring_view) * array->size_alloc); } else if (array->len == array->size_alloc) { array->size_alloc *= 2; array->p = realloc(array->p, sizeof(cstring_view) * array->size_alloc); } array->p[array->len].c_str = s; array->p[array->len].length = strlen(s); array->len++; } static int cmp_cstring_view(const void *p1, const void *p2) { return strcmp(((const cstring_view * const)p1)->c_str, ((const cstring_view * const)p2)->c_str); } void cstring_view_array_sort(cstring_view_array *array) { if (array->p) { qsort(array->p, array->len, sizeof(cstring_view), cmp_cstring_view); } } /* The string array needs to allocate more space to append a new entry. */ bool is_cstring_view_array_full(cstring_view_array *array) { return !array || array->len == array->size_alloc; } /* Does NOT deep copy the string */ void cstring_view_array_append_noalloc(cstring_view_array *array, char *s) { assert(array->size_alloc > 0); assert(array->len < array->size_alloc); array->p[array->len].c_str = s; array->p[array->len].length = strlen(s); array->len++; } void cstring_view_array_deep_free(cstring_view_array *array) { for (int i = 0; i < array->len; i++) free((/* non-const */ char *)array->p[i].c_str); free(array->p); } bool is_in_sorted_cstring_view_array(const char *str, const ssize_t len, const cstring_view_array *array) { for (int i = 0; i < array->len; i++) { const char * const entry = array->p[i].c_str; const ssize_t entry_len = array->p[i].length; if (len != entry_len) { continue; } const int memcmp_res = memcmp(entry, str, len); if (memcmp_res < 0) { continue; } else { return memcmp_res == 0; } } return false; } void voidp_array_init(voidp_array *array) { memset(array, 0, sizeof(*array)); } /* Does NOT deep copy whatever is behind voidp - obviously */ void voidp_array_append(voidp_array *array, void *p) { if (array->size_alloc == 0) { array->size_alloc = 16 /* whatever */; array->p = malloc(sizeof(void *) * array->size_alloc); } else if (array->len + 1 == array->size_alloc) { array->size_alloc *= 2; array->p = realloc(array->p, sizeof(void *) * array->size_alloc); } array->p[array->len++] = p; array->p[array->len] = NULL; } void voidp_array_deep_free(voidp_array *array, void (*fn_free)(void *)) { if (fn_free != NULL) { for (int i = 0; i < array->len; i++) { (*fn_free)(array->p[i]); } } free(array->p); } void voidp_set_init(voidp_set *set) { memset(set, 0, sizeof(*set)); } void voidp_set_clear(voidp_set *set) { set->len = 0; } bool voidp_set_contains(const voidp_set *set, const void *p) { for (int i = 0; i < set->len; i++) { if (set->p[i] == p) { return true; } } return false; } /* Does NOT deep copy whatever is behind voidp - obviously */ void voidp_set_insert(voidp_set *set, const void *p) { if (!voidp_set_contains(set, p)) { if (set->size_alloc == 0) { set->size_alloc = 16 /* whatever */; set->p = malloc(sizeof(void *) * set->size_alloc); } else if (set->len == set->size_alloc) { set->size_alloc *= 2; set->p = realloc(set->p, sizeof(void *) * set->size_alloc); } set->p[set->len++] = p; } } /* Does NOT deep free whatever is behind voidp - obviously */ void voidp_set_erase(voidp_set *set, const void *p) { for (int i = 0; i < set->len; i++) { if (set->p[i] == p) { set->p[i] = set->p[set->len - 1]; set->len--; break; } } } /** * Checks if a path semantically begins with one of the given sorted subpaths. * * Does string operations only, does not look at the file system. */ bool is_path_at_locations(const char * const path, const ssize_t len, const cstring_view_array *location_array) { const ssize_t path_len = len >= 0 ? len : (ssize_t)strlen(path); for (int i = 0; i < location_array->len; i++) { const char * const location = location_array->p[i].c_str; ssize_t location_len = location_array->p[i].length; while (location_len > 0 && location[location_len - 1] == '/') { location_len--; } if (path_len < location_len) { continue; } if (path[location_len] != '/' && path_len > location_len) { continue; } #if 0 // TODO(rbalint) enforce alignment of cstring_view_array entries to make this safe and quick /* Try comparing only the first 8 bytes to potentially save a call to memcmp */ if (location_len >= sizeof(int64_t) && *(const int64_t*)location != *(const int64_t*)path) { /* Does not break the loop if path_ > location->name_ */ // TODO(rbalint) maybe the loop could be broken making this function even faster continue; } #endif const int memcmp_res = memcmp(location, path, location_len); if (memcmp_res < 0) { continue; } else if (memcmp_res > 0) { return false; } if (path_len == location_len) { return true; } if (path[location_len] == '/') { return true; } } return false; } /** * Checks if the file name is canonical, i.e.: * - does not start with "./" * - does not end with "/" or "/." * - does not contain "//" or "/./" * - can contain "/../", since they might point elsewhere if a symlink led to its containing * directory. * See #401 for further details and gotchas. * * Returns if the path is in canonical form */ bool is_canonical(const char * const path, const size_t length) { if (path[0] == '\0') return true; if ((path[0] == '.' && path[1] == '/') || (length >= 2 && path[length - 1] == '/') || (length >= 2 && path[length - 2] == '/' && path[length - 1] == '.') || strstr(path, "//") || strstr(path, "/./")) { return false; } return true; } #ifdef __cplusplus } /* extern "C" */ #endif firebuild-0.8.2/src/common/firebuild_common.h000066400000000000000000000236461447164520700212600ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef COMMON_FIREBUILD_COMMON_H_ #define COMMON_FIREBUILD_COMMON_H_ #include #include #include #include #include #include "common/cstring_view.h" #ifdef __cplusplus extern "C" { #endif /** Buffer size for paths. */ #define FB_PATH_BUFSIZE 4096 /* This structure's size needs to be a multiple of 8 bytes, so that reads from the serialized FBB * message, which follows this structure in memory, are properly aligned. */ typedef struct msg_header_ { /* message payload size (without the header or the attached fds), in bytes */ uint32_t msg_size; /* ack_id, or 0 if unused */ uint16_t ack_id; /* the number of fds attached as ancillary data (SCM_RIGHTS) */ uint16_t fd_count; } msg_header; /** * cstring_view_array allows to conveniently build up an array of strings (i.e. NULL-terminated char**). */ typedef struct { cstring_view *p; int len; /* excluding the trailing NULL */ int size_alloc; /* including the room for the trailing NULL */ } cstring_view_array; void cstring_view_array_init(cstring_view_array *array); void cstring_view_array_append(cstring_view_array *array, char *s); void cstring_view_array_sort(cstring_view_array *array); void cstring_view_array_deep_free(cstring_view_array *array); bool is_cstring_view_array_full(cstring_view_array *array); void cstring_view_array_append_noalloc(cstring_view_array *array, char *s); bool is_in_sorted_cstring_view_array(const char *str, const ssize_t len, const cstring_view_array *array); #define STATIC_CSTRING_VIEW_ARRAY(name, size) \ cstring_view name##_ptrs[size] = {0}; \ cstring_view_array name = {name##_ptrs, 0, size} /** * voidp_array allows to conveniently build up an array of pointers (i.e. NULL-terminated void**). */ typedef struct { void **p; int len; /* excluding the trailing NULL */ int size_alloc; /* including the room for the trailing NULL */ } voidp_array; void voidp_array_init(voidp_array *array); void voidp_array_append(voidp_array *array, void *p); void voidp_array_deep_free(voidp_array *array, void (*fn_free)(void *)); /** * voidp_set allows to conveniently build up an unordered set of pointers. */ typedef struct { const void **p; int len; int size_alloc; } voidp_set; void voidp_set_init(voidp_set *set); void voidp_set_clear(voidp_set *set); bool voidp_set_contains(const voidp_set *set, const void *p); void voidp_set_insert(voidp_set *set, const void *p); void voidp_set_erase(voidp_set *set, const void *p); bool is_path_at_locations(const char *path, const ssize_t len, const cstring_view_array *prefix_array); /** * Checks if the file name is canonical, i.e.: * - does not start with "./" * - does not end with "/" or "/." * - does not contain "//" or "/./" * - can contain "/../", since they might point elsewhere if a symlink led to its containing * directory. * See #401 for further details and gotchas. * * Returns if the path is in canonical form */ bool is_canonical(const char * const path, const size_t length); static inline bool is_rdonly(int flags) { return ((flags & O_ACCMODE) == O_RDONLY); } static inline bool is_wronly(int flags) { return ((flags & O_ACCMODE) == O_WRONLY); } static inline bool is_rdwr(int flags) { return ((flags & O_ACCMODE) == O_RDWR); } // static inline bool is_read(int flags) { return (is_rdonly(flags) || is_rdwr(flags)); } static inline bool is_write(int flags) { return (is_wronly(flags) || is_rdwr(flags)); } /** * wrapper for read() retrying on recoverable errors * * It is implemented differently in supervisor and interceptor */ ssize_t fb_read(int fd, void *buf, size_t count); /** * wrapper for write() retrying on recoverable errors * * It is implemented differently in supervisor and interceptor */ ssize_t fb_write(int fd, const void *buf, size_t count); /** * wrapper for writev() retrying on recoverable errors * * It is implemented differently in supervisor and interceptor */ ssize_t fb_writev(int fd, struct iovec *iov, int iovcnt); #ifdef __cplusplus } /* extern "C" */ #endif #ifndef TEMP_FAILURE_RETRY #define TEMP_FAILURE_RETRY(expression) ({ \ ssize_t ret; \ do { \ ret = (expression); \ } while (ret == -1 && errno == EINTR); \ ret; \ }) #endif #ifndef __OPEN_NEEDS_MODE #define __OPEN_NEEDS_MODE(flags) (((flags) & O_CREAT) != 0) #endif /** Wrapper macro for read() or write() retrying on recoverable errors * (EINTR and short read/write). */ #define FB_READ_WRITE(op, fd, buf, count) \ { \ ssize_t ret; \ size_t remaining = count; \ do { \ ret = (op)(fd, buf, remaining); \ if (ret == -1) { \ if (errno == EINTR) { \ continue; \ } else { \ return ret; \ } \ } else if (ret == 0) { \ return ret; \ } else { \ remaining -= ret; \ buf = ((char *) buf) + ret; \ } \ } while (remaining > 0); \ return count; \ } /** * Wrapper macro for readv() or writev(), with the following differences: * * - retries/continues on recoverable errors (EINTR and short read/write); * * - iov/iovcnt can be arbitrarily large (if it's larger than IOV_MAX then * the operation will not be atomic though); * * - in order to implement the previous two without having to copy the * entire iov array, iov isn't const. */ #define FB_READV_WRITEV(op, fd, iov, iovcnt) \ { \ ssize_t ret; \ ssize_t written = 0; \ ssize_t remaining = 0; \ for (int i = 0; i < iovcnt; i++) { \ remaining += iov[i].iov_len; \ } \ while (1) { \ ret = (op)(fd, iov, iovcnt <= IOV_MAX ? iovcnt : IOV_MAX); \ if (ret == remaining) { \ /* completed */ \ return written + ret; \ } else if (ret == -1) { \ if (errno == EINTR) { \ /* retry on signal */ \ continue; \ } else { \ /* bail out on permanent error */ \ return ret; \ } \ } else { \ /* some, but not all bytes have been written */ \ written += ret; \ remaining -= ret; \ while ((size_t) ret >= iov[0].iov_len) { \ /* skip the fully written chunks */ \ ret -= iov[0].iov_len; \ iov++; \ iovcnt--; \ } \ /* adjust the partially written chunk */ \ iov[0].iov_base = ((char *) iov[0].iov_base) + ret; \ iov[0].iov_len -= ret; \ } \ } \ } #endif // COMMON_FIREBUILD_COMMON_H_ firebuild-0.8.2/src/common/platform.h000066400000000000000000000130011447164520700175470ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef COMMON_PLATFORM_H_ #define COMMON_PLATFORM_H_ #include #include #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #include #include /* Convenience macro because #if defined (__GLIBC_PREREQ) && __GLIBC_PREREQ (x, y) does not work. */ #ifdef __GLIBC_PREREQ #define FB_GLIBC_PREREQ __GLIBC_PREREQ #else #define FB_GLIBC_PREREQ(major, minor) 0 #endif #ifdef __has_include #if __has_include() #include #endif #endif #ifndef CLOSE_RANGE_CLOEXEC #define CLOSE_RANGE_CLOEXEC (1U << 2) #endif #ifndef CLONE_PIDFD #define CLONE_PIDFD 0x00001000 #endif #ifndef __linux__ #define loff_t off_t #endif #ifdef __APPLE__ #define off64_t off_t #endif #ifdef __linux__ #ifndef STATX_TYPE #define STATX_TYPE 0x0001U #define STATX_MODE 0x0002U #define STATX_SIZE 0x0200U struct statx_timestamp { __s64 tv_sec; __u32 tv_nsec; }; struct statx { __u32 stx_mask; __u32 stx_blksize; __u64 stx_attributes; __u32 stx_nlink; __u32 stx_uid; __u32 stx_gid; __u16 stx_mode; __u64 stx_ino; __u64 stx_size; __u64 stx_blocks; __u64 stx_attributes_mask; struct statx_timestamp stx_atime; struct statx_timestamp stx_btime; struct statx_timestamp stx_ctime; struct statx_timestamp stx_mtime; __u32 stx_rdev_major; __u32 stx_rdev_minor; __u32 stx_dev_major; __u32 stx_dev_minor; }; #endif // STATX_TYPE #endif // __linux__ #ifdef __APPLE__ #define stat64 stat #define lstat64 lstat #define fstat64 fstat #define st_mtim st_mtimespec #endif #ifdef __APPLE__ #define PRImode "uh" #else #define PRImode "u" #endif #if (defined(SIZE_WIDTH) && (SIZE_WIDTH == 64)) \ || (defined(__SIZE_WIDTH__) && (__SIZE_WIDTH__ == 64)) #define PRIsize "lu" #define PRIssize "ld" #else #define PRIsize "u" #define PRIssize "d" #endif #ifdef __APPLE__ #define PRIoff "lld" #define SCNoff "lld" #else #define PRIoff "ld" #define SCNoff "ld" #endif #if __WORDSIZE == 64 #define PRIoff64 "ld" #else #define PRIoff64 "lld" #endif #ifdef __APPLE__ #define PRIloff "lld" #else #define PRIloff PRIoff64 #endif #ifdef __APPLE__ #define sighandler_t sig_t #endif static inline bool path_is_absolute(const char * p) { #ifdef _WIN32 return !PathIsRelative(p); #else if (p[0] == '/') { return true; } else { return false; } #endif } /** * Check if fd1 and fd2 point to the same place. * kcmp() is not universally available, so in its absence do a back-n-forth fcntl() on one and see * if it drags the other with it. * See https://unix.stackexchange.com/questions/191967. * @return 0 if they point to the same place, -1 or 1 if fd1 sorts lower or higher than fd2 in an * arbitrary ordering to help using fdcmp for sorting */ static inline int fdcmp(int fd1, int fd2) { #ifdef __linux__ pid_t pid = getpid(); switch (syscall(SYS_kcmp, pid, pid, KCMP_FILE, fd1, fd2)) { case 0: return 0; case 1: return -1; case 2: return 1; case 3: return fd1 < fd2 ? -1 : 1; case -1: { #endif /* TODO(rbalint) this may not be safe for shim, but I have no better idea */ int flags1 = fcntl(fd1, F_GETFL); int flags2a = fcntl(fd2, F_GETFL); fcntl(fd1, F_SETFL, flags1 ^ O_NONBLOCK); int flags2b = fcntl(fd2, F_GETFL); fcntl(fd1, F_SETFL, flags1); return (flags2a != flags2b) ? 0 : (fd1 < fd2 ? -1 : 1); #ifdef __linux__ } default: assert(0 && "not reached"); abort(); } #endif } /** Makes a directory hierarchy, like the mkdirhier(1) command */ static inline int mkdirhier(const char *pathname, const mode_t mode) { if (mkdir(pathname, mode) == 0) { return 0; } else { switch (errno) { case EEXIST: return 0; case ENOENT: { const char *last_slash = strrchr(pathname, '/'); if (last_slash) { ssize_t len = last_slash - pathname; char* parent = (char*)alloca(len + 1); memcpy(parent, pathname, len); parent[len] = '\0'; if (mkdirhier(parent, mode) == 0) { return mkdir(pathname, mode); } else { return -1; } } else { return -1; } } default: return -1; } } } inline int fb_pipe2(int pipefd[2], int flags) { #ifdef __APPLE__ if (pipe(pipefd) == -1) { return -1; } else if (fcntl(pipefd[0], F_SETFL, flags) == -1 || fcntl(pipefd[1], F_SETFL, flags) == -1) { int saved_errno = errno; close(pipefd[0]); close(pipefd[1]); errno = saved_errno; return -1; } else { return 0; } #else return pipe2(pipefd, flags); #endif } #endif // COMMON_PLATFORM_H_ firebuild-0.8.2/src/firebuild/000077500000000000000000000000001447164520700162345ustar00rootroot00000000000000firebuild-0.8.2/src/firebuild/.gitignore000066400000000000000000000000641447164520700202240ustar00rootroot00000000000000fbbfp.? fbbfp_decode.c fbbstore.? fbbstore_decode.c firebuild-0.8.2/src/firebuild/CMakeLists.txt000066400000000000000000000063141447164520700210000ustar00rootroot00000000000000 # GCC raises warning about the GCC-optimized code set_source_files_properties(debug.cc execed_process.cc file_name.cc obj_cache.cc utils.cc PROPERTIES COMPILE_FLAGS "-Wno-strict-overflow") # FBB structures and placing them aligned in buffers ensure that members will be aligned properly, too set_source_files_properties(execed_process_cacher.cc PROPERTIES COMPILE_FLAGS "-Wno-cast-align -Wno-strict-overflow") set_source_files_properties(message_processor.cc PROPERTIES COMPILE_FLAGS "-DFIREBUILD_VERSION='\"${FIREBUILD_VERSION}\"' -Wno-cast-align") set_source_files_properties(firebuild.cc PROPERTIES COMPILE_FLAGS "-DFIREBUILD_VERSION='\"${FIREBUILD_VERSION}\"'") find_package(LibConfig REQUIRED) find_package(PkgConfig REQUIRED) # xxh128's algorithm changed and became officially supported in 0.8.0 pkg_check_modules(XXHASH REQUIRED libxxhash>=0.8.0) if (WITH_JEMALLOC) pkg_check_modules(JEMALLOC jemalloc) endif() find_package(tsl-hopscotch-map REQUIRED) if (SANITIZE) set(SANITIZE_SUPERVISOR_LINK_OPTIONS -static-libasan -fsanitize=address -fsanitize=undefined) endif() add_custom_command( OUTPUT fbbfp.cc fbbfp.h fbbfp_decode.c DEPENDS fbbfp.def ../common/fbb/generate_fbb ../common/fbb/tpl.c ../common/fbb/tpl.h WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ../common/fbb/generate_fbb fbbfp ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating fbbfp files") add_custom_target(fbbfp_gen_files ALL DEPENDS fbbfp.cc fbbfp.h fbbfp_decode.c) add_custom_command( OUTPUT fbbstore.cc fbbstore.h fbbstore_decode.c DEPENDS fbbstore.def ../common/fbb/generate_fbb ../common/fbb/tpl.c ../common/fbb/tpl.h WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ../common/fbb/generate_fbb fbbstore ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating fbbstore files") add_custom_target(fbbstore_gen_files ALL DEPENDS fbbstore.cc fbbstore.h fbbstore_decode.c) add_executable(firebuild-bin base64.cc config.cc debug.cc epoll.cc file_name.cc firebuild.cc pipe.cc pipe_recorder.cc process.cc exe_matcher.cc execed_process.cc execed_process_cacher.cc execed_process_env.cc forked_process.cc message_processor.cc process_factory.cc process_tree.cc hash.cc hash_cache.cc file_fd.cc file_info.cc file_usage.cc file_usage_update.cc blob_cache.cc obj_cache.cc report.cc sigchild_callback.cc utils.cc fbbfp.cc fbbstore.cc $ $) target_link_libraries(firebuild-bin ${LIBCONFIGPP_LIBRARY} ${JEMALLOC_LIBRARIES} ${XXHASH_LIBRARIES}) target_link_options(firebuild-bin PUBLIC -Wno-array-bounds -Wno-strict-overflow ${SANITIZE_SUPERVISOR_LINK_OPTIONS}) set_target_properties(firebuild-bin PROPERTIES OUTPUT_NAME firebuild) # GCC 9's LTO implementation seem to have a bug we hit, but did not fully triage yet # https://github.com/Tessil/hopscotch-map/issues/55 if ((NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10)) set_target_properties(firebuild-bin PROPERTIES INTERPROCEDURAL_OPTIMIZATION True) endif() add_dependencies(firebuild-bin fbbcomm_gen_files fbbfp_gen_files fbbstore_gen_files) install(TARGETS firebuild-bin DESTINATION bin) firebuild-0.8.2/src/firebuild/CPPLINT.cfg000066400000000000000000000000541447164520700200250ustar00rootroot00000000000000exclude_files=(.*_generated\.h|fbb.*\.[ch]) firebuild-0.8.2/src/firebuild/ascii_hash.h000066400000000000000000000036041447164520700205030ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_ASCII_HASH_H_ #define FIREBUILD_ASCII_HASH_H_ #include #include #include #include #include "firebuild/base64.h" #include "firebuild/hash.h" namespace firebuild { class AsciiHash { public: AsciiHash() = default; explicit AsciiHash(const char * const str) { #ifdef FB_EXTRA_DEBUG assert(Hash::valid_ascii(str)); #endif for (size_t i = 0; i < Hash::kAsciiLength + 1; i++) { str_[i] = str[i]; } } bool operator==(const AsciiHash& other) const { return memcmp(str_, other.str_, Hash::kAsciiLength) == 0; } const char * c_str() const { return str_; } private: char str_[Hash::kAsciiLength + 1]; }; inline std::string d(const AsciiHash& ascii_hash, const int level = 0) { return d(ascii_hash.c_str(), level); } } /* namespace firebuild */ namespace std { template <> class hash { public: size_t operator()(const firebuild::AsciiHash &a) const { return XXH3_64bits(a.c_str(), firebuild::Hash::kAsciiLength); } }; } #endif // FIREBUILD_ASCII_HASH_H_ firebuild-0.8.2/src/firebuild/base64.cc000066400000000000000000000066201447164520700176330ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include namespace firebuild { bool Base64::valid_ascii(const char* const str, const int length) { int i; /* The last character could be from a limited set, depending on the length of the input * that's encoded in this base64-like ASCII representation. This is not checked here * because the ASCII representation is never decoded to get the input bits back in this * program. */ for (i = 0; i < length; i++) { if ((str[i] >= 'A' && str[i] <= 'Z') || (str[i] >= 'a' && str[i] <= 'z') || (str[i] >= '0' && str[i] <= '9') || str[i] == '+' || str[i] == '^') { continue; } else { return false; } } if (str[i] != '\0') { return false; } return true; } /** * Helper method of encode(). * * Convert 3 input bytes (part of the binary representation) into 4 output bytes (part of the base64 * ASCII representation) according to base64 encoding. */ void Base64::encode_3byte_block(const unsigned char *in, char *out) { uint32_t val = (in[0] << 16) | (in[1] << 8) | (in[2]); out[0] = kEncodeMap[ val >> 18 ]; out[1] = kEncodeMap[(val >> 12) & 0x3f]; out[2] = kEncodeMap[(val >> 6) & 0x3f]; out[3] = kEncodeMap[ val & 0x3f]; } /** Similar to the previous, but for 2 input bytes (2 byte of the binary -> 3 ASCII characters */ void Base64::encode_2byte_block(const unsigned char *in, char *out) { uint16_t val = (in[0] << 8) | (in[1]); out[0] = kEncodeMap[(val >> 10) & 0x3f]; out[1] = kEncodeMap[(val >> 4) & 0x3f]; out[2] = kEncodeMap[(val << 2) & 0x3f]; } /** Similar to the previous, but for 1 input byte (1 byte of the binary -> 2 ASCII characters */ void Base64::encode_1byte_block(const unsigned char *in, char *out) { uint8_t val = in[0]; out[0] = kEncodeMap[ val >> 2 ]; out[1] = kEncodeMap[(val << 4) & 0x3f]; } void Base64::encode(const unsigned char* in, char *out, int in_length) { if (in_length == 16) { encode_3byte_block(&in[ 0], out); encode_3byte_block(&in[ 3], out + 4); encode_3byte_block(&in[ 6], out + 8); encode_3byte_block(&in[ 9], out + 12); encode_3byte_block(&in[12], out + 16); encode_1byte_block(&in[15], out + 20); out[22] = '\0'; } else if (in_length == 8) { encode_3byte_block(&in[0], out); encode_3byte_block(&in[3], out + 4); encode_2byte_block(&in[6], out + 8); out[11] = '\0'; } else { // TODO(rbalint) support other lengths, maybe drop those hand-unrolled loops if // something is faster abort(); } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/base64.h000066400000000000000000000037031447164520700174740ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_BASE64_H_ #define FIREBUILD_BASE64_H_ namespace firebuild { /** * Base64 variant with encoding only. * * The two non-alphanumeric characters of our base64 alphabet are '+' and '^' and * none of the characters are at their usual position. * The characters are reordered compared to the original base64 mapping. * They are ordered by increasing ASCII code to have a set of hash values * and their ASCII representation sort the same way. * No trailing '=' signs to denote the partial block. */ class Base64 { public: static bool valid_ascii(const char* const str, const int length); static void encode(const unsigned char* in, char* out, int in_length); private: static void encode_3byte_block(const unsigned char *in, char *out); static void encode_2byte_block(const unsigned char *in, char *out); static void encode_1byte_block(const unsigned char *in, char *out); /* Subkey's sorting relies on the characters being in ASCII order. */ static constexpr char kEncodeMap[] = "+0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ^abcdefghijklmnopqrstuvwxyz"; }; } /* namespace firebuild */ #endif // FIREBUILD_BASE64_H_ firebuild-0.8.2/src/firebuild/blob_cache.cc000066400000000000000000000413211447164520700206050ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/blob_cache.h" #include #ifdef __APPLE__ #include #endif #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #include "firebuild/ascii_hash.h" #include "firebuild/execed_process_cacher.h" #include "firebuild/debug.h" #include "firebuild/file_name.h" #include "firebuild/hash.h" #include "firebuild/utils.h" namespace firebuild { /* singleton */ BlobCache *blob_cache; BlobCache::BlobCache(const std::string &base_dir) : base_dir_(base_dir) { mkdir(base_dir_.c_str(), 0700); } /* * Copy the contents from an open file descriptor to another, * preferring advanced technologies like copy on write. * Might skip the beginning of the input file. * Might append to the target file instead of replacing its contents. O_APPEND should _not_ be set * on fd_dst because then the fast copy_file_range() method doesn't work. */ static bool copy_file(int fd_src, loff_t src_skip_bytes, int fd_dst, bool append, const struct stat64 *src_stat_ptr = NULL) { /* Try CoW first. */ if (src_skip_bytes == 0 && !append) { #ifdef __APPLE__ if (fcopyfile(fd_src, fd_dst, nullptr, COPYFILE_DATA) == 0) { #else if (ioctl(fd_dst, FICLONE, fd_src) == 0) { #endif /* CoW succeeded. Moo! */ return true; } } else { // FIXME try FICLONERANGE } /* Try copy_file_range(). Gotta get the source file's size, and in append mode also the * destination file's size */ struct stat64 src_st_local; if (!src_stat_ptr && fstat64(fd_src, &src_st_local) == -1) { fb_perror("fstat"); assert(0); return false; } const struct stat64 *src_st = src_stat_ptr ? src_stat_ptr : &src_st_local; if (!S_ISREG(src_st->st_mode)) { FB_DEBUG(FB_DEBUG_CACHING, "not a regular file"); return false; } struct stat64 dst_st_local; const struct stat64 *dst_st = nullptr; if (append) { if (fstat64(fd_dst, &dst_st_local) == -1) { fb_perror("fstat"); assert(0); return false; } dst_st = &dst_st_local; if (!S_ISREG(dst_st->st_mode)) { FB_DEBUG(FB_DEBUG_CACHING, "not a regular file"); return false; } } off_t len = src_st->st_size >= src_skip_bytes ? src_st->st_size - src_skip_bytes : 0; loff_t dst_skip_bytes = append ? dst_st->st_size : 0; return fb_copy_file_range(fd_src, &src_skip_bytes, fd_dst, &dst_skip_bytes, len, 0) == len; } /* /x/xx/ */ static size_t kBlobCachePathLength = 1 + 1 + 1 + 2 + 1 + Hash::kAsciiLength; /* * Constructs the filename where the cached file is to be stored, or * read from. Optionally creates the necessary subdirectories within the * cache's base directory. * * Example: with base="base", key's ASCII representation being "key", and * create_dirs=true, it creates the directories "base/k" and "base/k/ke" * and returns "base/k/ke/key". */ static void construct_cached_file_name(const std::string &base, const Hash &key, bool create_dirs, char* path) { char ascii[Hash::kAsciiLength + 1]; key.to_ascii(ascii); char *end = path; memcpy(end, base.c_str(), base.length()); end += base.length(); *end++ = '/'; *end++ = ascii[0]; if (create_dirs) { *end = '\0'; mkdir(path, 0700); } *end++ = '/'; *end++ = ascii[0]; *end++ = ascii[1]; if (create_dirs) { *end = '\0'; mkdir(path, 0700); } *end++ = '/'; memcpy(end, ascii, sizeof(ascii)); } bool BlobCache::store_file(const FileName *path, int max_writers, int fd_src, loff_t src_skip_bytes, loff_t size, Hash *key_out) { TRACK(FB_DEBUG_CACHING, "path=%s, max_writers=%d, fd_src=%d, skip=%" PRIloff ", size=%" PRIloff, D(path), max_writers, fd_src, src_skip_bytes, size); FB_DEBUG(FB_DEBUG_CACHING, "BlobCache: storing blob " + d(path)); if (path->writers_count() > max_writers) { /* The file could be written while saving the file, don't take that risk. */ FB_DEBUG(FB_DEBUG_CACHING, "file is opened for writing by some other process"); return false; } bool close_fd_src = false; if (fd_src == -1) { fd_src = open(path->c_str(), O_RDONLY); if (fd_src == -1) { fb_perror("Failed opening file to be stored in cache"); assert(0); return false; } close_fd_src = true; } /* In order to save an fstat64() call in copy_file() and set_from_fd(), create a "fake" stat * result here. We know it's a regular file, we know its size, and the rest are irrelevant. */ struct stat64 src_st; src_st.st_mode = S_IFREG; src_st.st_size = size; /* Copy the file to a temporary one under the cache */ char *tmpfile; if (asprintf(&tmpfile, "%s/new.XXXXXX", base_dir_.c_str()) < 0) { fb_perror("asprintf"); assert(0); return false; } int fd_dst = mkstemp(tmpfile); /* opens with O_RDWR */ if (fd_dst == -1) { fb_perror("Failed mkstemp() during storing file"); assert(0); if (close_fd_src) { close(fd_src); } free(tmpfile); return false; } if (!copy_file(fd_src, src_skip_bytes, fd_dst, false, &src_st)) { FB_DEBUG(FB_DEBUG_CACHING, "failed to copy file"); if (close_fd_src) { close(fd_src); } close(fd_dst); unlink(tmpfile); free(tmpfile); return false; } if (close_fd_src) { close(fd_src); } /* Copying complete. Compute checksum on the copy, to prevent cache * corruption if someone is modifying the original file. */ Hash key; /* In order to save an fstat64() call in set_from_fd(), create a "fake" stat result here. We * know that it's a regular file, we know its size, and the rest are irrelevant. */ struct stat64 dst_st; dst_st.st_mode = S_IFREG; dst_st.st_size = src_st.st_size >= src_skip_bytes ? src_st.st_size - src_skip_bytes : 0; if (!key.set_from_fd(fd_dst, &dst_st, NULL)) { FB_DEBUG(FB_DEBUG_CACHING, "failed to compute hash"); close(fd_dst); unlink(tmpfile); free(tmpfile); return false; } close(fd_dst); char* path_dst = reinterpret_cast(alloca(base_dir_.length() + kBlobCachePathLength + 1)); construct_cached_file_name(base_dir_, key, true, path_dst); if (fb_renameat2(AT_FDCWD, tmpfile, AT_FDCWD, path_dst, RENAME_NOREPLACE) == -1) { if (errno == EEXIST) { FB_DEBUG(FB_DEBUG_CACHING, "blob is already stored"); unlink(tmpfile); } else { fb_perror("Failed renaming file while storing it"); assert(0); free(tmpfile); return false; } } else { execed_process_cacher->update_cached_bytes(dst_st.st_size); } free(tmpfile); if (FB_DEBUGGING(FB_DEBUG_CACHING)) { FB_DEBUG(FB_DEBUG_CACHING, " => " + d(key)); } if (FB_DEBUGGING(FB_DEBUG_CACHE)) { /* Place meta info in the cache, for easier debugging. */ std::string path_debug = std::string(path_dst) + kDebugPostfix; std::string txt(pretty_timestamp() + " Copied from " + d(path) + "\n"); int debugfd = open(path_debug.c_str(), O_CREAT|O_WRONLY|O_APPEND, 0600); if (write(debugfd, txt.c_str(), txt.size()) < 0) { fb_perror("BlobCache::store_file"); assert(0); } execed_process_cacher->update_cached_bytes(txt.size()); close(debugfd); } if (key_out != NULL) { *key_out = key; } return true; } bool BlobCache::move_store_file(const std::string &path, int fd, loff_t size, Hash *key_out) { TRACK(FB_DEBUG_CACHING, "path=%s, fd=%d, size=%" PRIloff, D(path), fd, size); FB_DEBUG(FB_DEBUG_CACHING, "BlobCache: storing blob by moving " + path); Hash key; /* In order to save an fstat64() call in set_from_fd(), create a "fake" stat result here. * We know that it's a regular file, we know its size, and the rest are irrelevant. */ struct stat64 st; st.st_mode = S_IFREG; st.st_size = size; if (!key.set_from_fd(fd, &st, NULL)) { FB_DEBUG(FB_DEBUG_CACHING, "failed to compute hash"); close(fd); unlink(path.c_str()); return false; } close(fd); char* path_dst = reinterpret_cast(alloca(base_dir_.length() + kBlobCachePathLength + 1)); construct_cached_file_name(base_dir_, key, true, path_dst); if (fb_renameat2(AT_FDCWD, path.c_str(), AT_FDCWD, path_dst, RENAME_NOREPLACE) == -1) { if (errno == EEXIST) { FB_DEBUG(FB_DEBUG_CACHING, "blob is already stored"); unlink(path.c_str()); } else { fb_perror("Failed renaming file to cache"); assert(0); unlink(path.c_str()); return false; } } else { execed_process_cacher->update_cached_bytes(size); } if (FB_DEBUGGING(FB_DEBUG_CACHING)) { FB_DEBUG(FB_DEBUG_CACHING, " => " + key.to_ascii()); } if (FB_DEBUGGING(FB_DEBUG_CACHE)) { /* Place meta info in the cache, for easier debugging. */ std::string path_debug = std::string(path_dst) + kDebugPostfix; std::string txt(pretty_timestamp() + " Moved from " + path + "\n"); int debugfd = open(path_debug.c_str(), O_CREAT|O_WRONLY|O_APPEND, 0600); ssize_t written = write(debugfd, txt.c_str(), txt.size()); if (written < 0) { fb_perror("BlobCache::move_store_file"); assert(0); close(debugfd); return false; } execed_process_cacher->update_cached_bytes(written); close(debugfd); } if (key_out != NULL) { *key_out = key; } return true; } bool BlobCache::retrieve_file(int blob_fd, const FileName *path_dst, bool append) { TRACK(FB_DEBUG_CACHING, "blob_fd=%d, path_dst=%s, append=%s", blob_fd, D(path_dst), D(append)); int flags = append ? O_WRONLY : (O_WRONLY|O_CREAT|O_TRUNC); int fd_dst = open(path_dst->c_str(), flags, 0666); if (fd_dst == -1) { if (append) { fb_perror("Failed opening file to be recreated from cache"); assert(0); } return false; } if (!copy_file(blob_fd, 0, fd_dst, append)) { FB_DEBUG(FB_DEBUG_CACHING, "Copying file from cache failed"); assert(0); close(blob_fd); close(fd_dst); if (!append) { unlink(path_dst->c_str()); } return false; } close(fd_dst); return true; } int BlobCache::get_fd_for_file(const Hash &key) { if (FB_DEBUGGING(FB_DEBUG_CACHING)) { FB_DEBUG(FB_DEBUG_CACHING, "BlobCache: getting fd for blob " + key.to_ascii()); } char* path_src = reinterpret_cast(alloca(base_dir_.length() + kBlobCachePathLength + 1)); construct_cached_file_name(base_dir_, key, false, path_src); return open(path_src, O_RDONLY); } void BlobCache::delete_entries(const std::string& path, const std::vector& entries, const std::string& debug_postfix, off_t* debug_bytes) { struct stat st; for (const auto& entry : entries) { const std::string absolute_entry = path + "/" + entry; if (fstatat(AT_FDCWD, absolute_entry.c_str(), &st, AT_SYMLINK_NOFOLLOW) != 0) { fb_perror(entry.c_str()); } else { if (unlink(absolute_entry.c_str()) == 0) { execed_process_cacher->update_cached_bytes(-st.st_size); } else { fb_perror("unlink"); } } if (FB_DEBUGGING(FB_DEBUG_CACHE)) { const std::string absolute_debug_entry = absolute_entry + debug_postfix; if (fstatat(AT_FDCWD, absolute_debug_entry.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0) { /* All debugging entries were kept in the previous round. * Delete the ones related to entries to be deleted. */ if (unlink(absolute_debug_entry.c_str()) == 0) { execed_process_cacher->update_cached_bytes(-st.st_size); /* The size of this debug file has already been added to debug_bytes, reverse that. */ *debug_bytes -= st.st_size; } else { fb_perror("unlink"); } } } } } off_t BlobCache::gc_collect_total_blobs_size() { return recursive_total_file_size(base_dir_); } void BlobCache::gc_blob_cache_dir(const std::string& path, const tsl::hopscotch_set& referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes) { DIR * dir = opendir(path.c_str()); if (dir == NULL) { return; } /* Visit dirs recursively and check all the files. */ struct dirent *dirent; std::vector entries_to_delete; std::vector subdirs_to_visit; while ((dirent = readdir(dir)) != NULL) { const char* name = dirent->d_name; if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { continue; } switch (fixed_dirent_type(dirent, dir, path)) { case DT_DIR: { subdirs_to_visit.push_back(name); break; } case DT_REG: { if (Hash::valid_ascii(name)) { if (referenced_blobs.find(AsciiHash(name)) == referenced_blobs.end()) { /* Not referenced, can be cleaned up*/ entries_to_delete.push_back(name); } else { /* Good, keeping the referenced blob. */ *cache_bytes += file_size(dir, name); } break; } else { /* Regular file, but not named as expected for a cache blob. */ const char* debug_postfix = nullptr; if ((debug_postfix = strstr(name, kDebugPostfix))) { /* Files for debugging blobs.*/ if (FB_DEBUGGING(FB_DEBUG_CACHE)) { const size_t name_len = debug_postfix - name; assert_cmp(name_len, <, FB_PATH_BUFSIZE); char related_name[FB_PATH_BUFSIZE]; memcpy(related_name, name, name_len); related_name[name_len] = '\0'; struct stat st; if (fstatat(dirfd(dir), related_name, &st, 0) == 0) { /* Keeping debugging file that has related blob. If the object gets removed * the debugging file will be removed with it, too. In that case debug_bytes * needs to be adjusted again. */ *debug_bytes += file_size(dir, name); } else { /* Removing old debugging file later to not break next readdir(). */ entries_to_delete.push_back(name); } } else { /* Removing old debugging file later to not break next readdir(). */ entries_to_delete.push_back(name); } } else { fb_error("Regular file among cache blobs has unexpected name, keeping it: " + path + "/" + d(name)); *unexpected_file_bytes += file_size(dir, name); } } break; } default: fb_error("File's type is unexpected, it is not a directory nor a regular file: " + path + "/" + d(name)); } } delete_entries(path, entries_to_delete, kDebugPostfix, debug_bytes); for (const auto& subdir : subdirs_to_visit) { gc_blob_cache_dir(path + "/" + subdir, referenced_blobs, cache_bytes, debug_bytes, unexpected_file_bytes); } /* Remove empty directory. */ rewinddir(dir); bool has_valid_entries = false; while ((dirent = readdir(dir)) != NULL) { const char* name {dirent->d_name}; /* skip "." and ".." */ if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { continue; } has_valid_entries = true; break; } if (!has_valid_entries && path != base_dir_) { /* The directory is now empty. It can be removed. */ rmdir(path.c_str()); } closedir(dir); } void BlobCache::gc(const tsl::hopscotch_set& referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes) { gc_blob_cache_dir(base_dir_, referenced_blobs, cache_bytes, debug_bytes, unexpected_file_bytes); } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/blob_cache.h000066400000000000000000000144311447164520700204510ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_BLOB_CACHE_H_ #define FIREBUILD_BLOB_CACHE_H_ #include #include #include #include #include #include #include "firebuild/ascii_hash.h" #include "firebuild/file_name.h" #include "firebuild/hash.h" namespace firebuild { class BlobCache { public: explicit BlobCache(const std::string &base_dir); ~BlobCache(); /** * Store the given regular file in the blob cache, with its hash as the key. * Uses advanced technologies, such as copy on write, if available. * * If fd >= 0 then that is used as the data source, the path is only used for debugging. * * @param path The file to place in the cache * @param max_writers Maximum allowed number of writers to this file * @param fd_src Optionally the opened file descriptor to copy * @param src_skip_bytes Number of bytes to omit from the beginning of the input file * @param size The file's size (including the bytes to be skipped) * @param key_out Optionally store the key (hash) here * @return Whether succeeded */ bool store_file(const FileName *path, int max_writers, int fd_src, loff_t src_skip_bytes, loff_t size, Hash *key_out); /** * Store the given regular file in the blob cache, with its hash as the key. * * The file is moved from its previous location. It is assumed that * no one modifies it during checksum computation, that is, the * intercepted processes have no direct access to it. * * The file handle is closed. * * The hash_cache is not queried or updated. * * This API is designed for PipeRecorder in order to place the recorded data in the cache. * * @param path The file to move to the cache * @param fd A fd referring to this file * @param size The file's size * @param key_out Optionally store the key (hash) here * @return Whether succeeded */ bool move_store_file(const std::string &path, int fd, loff_t size, Hash *key_out); /** * Retrieve the given file from the blob cache. * * In non-append mode the file doesn't have to exist. If it doesn't exist, it's created with the * default permissions, according to the current umask. If it already exists, its contents will be * replaced, the permissions will be left unchanged. * * In append mode the file must already exist, the cache entry will be appended to it. * * Uses advanced technologies, such as copy on write, if available. * * @param blob_fd opened file descriptor of the blob to be used * @param path_dst Where to place the file * @param append Whether to use append mode * @return Whether succeeded */ bool retrieve_file(int blob_fd, const FileName *path_dst, bool append); /** * Get a read-only fd for a given entry in the cache. * * This is comfy when shortcutting a process and replaying what it wrote to a pipe. * * @param key The key (the file's hash) * @return A read-only fd, or -1 */ int get_fd_for_file(const Hash &key); /** * Garbage collect the blob cache * @param referenced_blobs blobs referenced from the object cache, they won't be deleted * @param[in,out] cache_bytes increased by every found and kept blob's size * @param[in,out] debug_bytes increased by every found and kept debug file's size * @param[in,out] unexpected_file_bytes increased by every found and kept file's size that has unexpected name, i.e. it is not used as a blob, nor a debug file */ void gc(const tsl::hopscotch_set& referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes); /** * Delete entries on the the specified path also deleting the debug entries related to the entries * to delete. * @param path path where the entries reside * @param entries entries to delete * @param debug_postfix string to prepend to entries to get the related debug entries * @param[in,out] debug_bytes decremented when removing a debug entry */ static void delete_entries(const std::string& path, const std::vector& entries, const std::string& debug_postfix, off_t* debug_bytes); /** Returns total size of all stored blob files including debug and invalid entries. */ off_t gc_collect_total_blobs_size(); private: /** * Garbage collect a blob cache directory * @param path blob cache directory's absolute path * @param referenced_blobs blobs referenced from the object cache, they won't be deleted * @param[in,out] cache_bytes increased by every found and kept blob's size * @param[in,out] debug_bytes increased by every found and kept debug file's size * @param[in,out] unexpected_file_bytes increased by every found and kept file's size that has unexpected name, i.e. it is not used as a blob, nor a debug file */ void gc_blob_cache_dir(const std::string& path, const tsl::hopscotch_set& referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes); /* Including the "blobs" subdir. */ std::string base_dir_; static constexpr char kDebugPostfix[] = "_debug.txt"; }; /* singleton */ extern BlobCache *blob_cache; } /* namespace firebuild */ #endif // FIREBUILD_BLOB_CACHE_H_ firebuild-0.8.2/src/firebuild/config.cc000066400000000000000000000423301447164520700200120ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/firebuild_common.h" #include "firebuild/debug.h" #include "firebuild/exe_matcher.h" #include "firebuild/file_name.h" #define GLOBAL_CONFIG "/etc/firebuild.conf" #define USER_CONFIG ".firebuild.conf" #define XDG_CONFIG "firebuild/firebuild.conf" namespace firebuild { libconfig::Config * cfg = nullptr; cstring_view_array ignore_locations {nullptr, 0, 0}; cstring_view_array read_only_locations {nullptr, 0, 0}; ExeMatcher* shortcut_allow_list_matcher = nullptr; ExeMatcher* dont_shortcut_matcher = nullptr; ExeMatcher* dont_intercept_matcher = nullptr; ExeMatcher* skip_cache_matcher = nullptr; tsl::hopscotch_set* shells = nullptr; bool ccache_disabled = false; /** Store results of processes consuming more CPU time (system + user) in microseconds than this. */ int64_t min_cpu_time_u = 0; int shortcut_tries = 0; int64_t max_cache_size = 0; uint64_t max_entry_size = 0; int quirks = 0; /** * Parse configuration file * * If custom_cfg_file is non-NULL, use that. * Otherwise try ./firebuild.conf, ~/.firebuild.conf, $XDG_CONFIG_HOME/firebuild/firebuild.conf, * /etc/firebuild.conf in that order. */ static void parse_cfg_file(libconfig::Config *cfg, const char *custom_cfg_file) { std::vector cfg_files; if (custom_cfg_file != NULL) { cfg_files = {custom_cfg_file}; } else { cfg_files = {".firebuild.conf"}; char *homedir = getenv("HOME"); if (homedir != NULL) { std::string user_cfg_file = homedir + std::string("/" USER_CONFIG); cfg_files.push_back(user_cfg_file); } char *xdg_config_home = getenv("XDG_CONFIG_HOME"); if (xdg_config_home != NULL) { std::string user_cfg_file = xdg_config_home + std::string("/" XDG_CONFIG); cfg_files.push_back(user_cfg_file); } cfg_files.push_back(GLOBAL_CONFIG); } for (size_t i = 0; i < cfg_files.size(); i++) { try { cfg->readFile(cfg_files[i].c_str()); } catch (const libconfig::FileIOException &fioex) { if (i == cfg_files.size() - 1) { std::cerr << "Could not read configuration file " << cfg_files[i] << std::endl; exit(EXIT_FAILURE); } else { continue; } } catch (const libconfig::ParseException &pex) { std::cerr << "Parse error at " << pex.getFile() << ":" << pex.getLine() << " - " << pex.getError() << std::endl; exit(EXIT_FAILURE); } } } /** Modify configuration * * str is one of: * key = value: * Create or replace the existing `key` to contain the scalar `value` * key += value * Append the scalar `value` to the existing array `key` * key -= value * Remove the scalar `value` from the existing array `key`, if found * key = "[]": * Clear an array * * E.g. str = "processes.dont_shortcut += \"myapp\"" * * Currently only strings are supported, but it's easy to add TypeBoolean, * TypeInt, TypeInt64 and TypeFloat too, if required. */ static void modify_config(libconfig::Config *cfg, const std::string& str) { bool append = false; bool remove = false; size_t eq_pos = str.find('='); if (eq_pos == std::string::npos) { std::cerr << "-o requires an equal sign" << std::endl; exit(EXIT_FAILURE); } size_t name_end_pos = eq_pos; if (name_end_pos > 0) { if (str[name_end_pos - 1] == '+') { name_end_pos--; append = true; } else if (str[name_end_pos - 1 ] == '-') { name_end_pos--; remove = true; } } while (name_end_pos > 0 && str[name_end_pos - 1] == ' ') { name_end_pos--; } std::string name = str.substr(0, name_end_pos); /* We support operations with scalars (string, int, bool...). * Libconfig doesn't provide a direct method to parse or even * get the type of the argument. So create and parse a mini config. */ std::string mini_config_str = "x = " + str.substr(eq_pos + 1); libconfig::Config *mini_config = new libconfig::Config(); mini_config->readString(mini_config_str); libconfig::Setting& x = mini_config->getRoot()["x"]; libconfig::Setting::Type type = x.getType(); if (append) { /* Append scalar value to an existing array. */ try { libconfig::Setting& array = cfg->lookup(name); libconfig::Setting& adding = array.add(type); /* Unfortunately there's no operator= to assign from another Setting. */ switch (type) { case libconfig::Setting::TypeString: { std::string x_str(x.c_str()); adding = x_str.c_str(); break; } default: std::cerr << "This type is not supported" << std::endl; exit(EXIT_FAILURE); } } catch(libconfig::SettingNotFoundException&) { std::cerr << "Setting not found: " << name << std::endl; exit(EXIT_FAILURE); } } else if (remove) { /* Remove all occurrences of a scalar value from an existing array. */ libconfig::Setting& array = cfg->lookup(name); for (int i = 0; i < array.getLength(); i++) { libconfig::Setting& item = array[i]; /* Unfortunately there's no operator== to compare with another Setting. */ switch (type) { case libconfig::Setting::TypeString: { std::string item_str(item.c_str()); std::string x_str(x.c_str()); if (item_str == x_str) { array.remove(i); i--; } break; } default: std::cerr << "This type is not supported" << std::endl; exit(EXIT_FAILURE); } } } else { if (type == libconfig::Setting::TypeArray) { if (x.getLength() > 0) { std::cerr << "Arrays can only be reset" << std::endl; exit(EXIT_FAILURE); } try { libconfig::Setting& array = cfg->lookup(name); const std::string setting_name = array.getName(); libconfig::Setting& parent = array.getParent(); parent.remove(setting_name); parent.add(setting_name, type); } catch(libconfig::SettingNotFoundException&) { std::cerr << "Setting not found" << std::endl; exit(EXIT_FAILURE); } delete mini_config; return; } /* Set a given value, overwriting the previous value if necessary. */ try { cfg->getRoot().remove(name); } catch(libconfig::SettingNotFoundException&) {} libconfig::Setting& adding = cfg->getRoot().add(name, type); /* Unfortunately there's no operator= to assign from another Setting. */ switch (type) { case libconfig::Setting::TypeString: { std::string x_str(x.c_str()); adding = x_str.c_str(); break; } case libconfig::Setting::TypeFloat: { float x_float = x; adding = x_float; break; } case libconfig::Setting::TypeInt: { int x_int = x; adding = x_int; break; } default: std::cerr << "This type is not supported" << std::endl; exit(EXIT_FAILURE); } } delete mini_config; } static void init_locations(cstring_view_array* locations, const libconfig::Config *cfg, const char* locations_setting) { cstring_view_array_init(locations); try { const libconfig::Setting& items = cfg->getRoot()[locations_setting]; for (int i = 0; i < items.getLength(); i++) { cstring_view_array_append(locations, strdup(items[i].c_str())); } } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } cstring_view_array_sort(locations); } static void init_matcher(ExeMatcher **matcher, const libconfig::Config *cfg, const char* matcher_setting) { assert(!*matcher); *matcher = new ExeMatcher(); try { const libconfig::Setting& items = cfg->getRoot()["processes"][matcher_setting]; for (int i = 0; i < items.getLength(); i++) { (*matcher)->add(items[i].c_str()); } } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } } void read_config(libconfig::Config *cfg, const char *custom_cfg_file, const std::list &config_strings) { parse_cfg_file(cfg, custom_cfg_file); cfg->setAutoConvert(true); for (auto s : config_strings) { modify_config(cfg, s); } if (FB_DEBUGGING(FB_DEBUG_CONFIG)) { fprintf(stderr, "--- Config:\n"); cfg->write(stderr); fprintf(stderr, "--- End of config.\n"); } /* Save portions of the configuration to separate variables for faster access. */ if (cfg->exists("min_cpu_time")) { libconfig::Setting& min_cpu_time_cfg = cfg->getRoot()["min_cpu_time"]; if (min_cpu_time_cfg.isNumber()) { float min_cpu_time_s = min_cpu_time_cfg; min_cpu_time_u = 1000000.0 * min_cpu_time_s; } } if (cfg->exists("shortcut_tries")) { libconfig::Setting& shortcut_tries_cfg = cfg->getRoot()["shortcut_tries"]; if (shortcut_tries_cfg.isNumber()) { shortcut_tries = shortcut_tries_cfg; } } if (cfg->exists("max_cache_size")) { libconfig::Setting& max_cache_size_cfg = cfg->getRoot()["max_cache_size"]; if (max_cache_size_cfg.isNumber()) { double max_cache_size_gb = max_cache_size_cfg; max_cache_size = max_cache_size_gb * 1000000000; if (max_cache_size < 0) { /* Fix up negative numbers. */ max_cache_size = 0; } } } if (cfg->exists("max_entry_size")) { libconfig::Setting& max_entry_size_cfg = cfg->getRoot()["max_entry_size"]; if (max_entry_size_cfg.isNumber()) { double max_entry_size_mb = max_entry_size_cfg; if (max_entry_size_mb < 0) { /* Fix up negative numbers. */ max_entry_size_mb = 0; } max_entry_size = max_entry_size_mb * 1000000; } } assert(FileName::isDbEmpty()); init_locations(&ignore_locations, cfg, "ignore_locations"); init_locations(&read_only_locations, cfg, "read_only_locations"); /* The read_only_locations setting used to be called system_locations. */ try { const libconfig::Setting& items = cfg->getRoot()["system_locations."]; for (int i = 0; i < items.getLength(); i++) { cstring_view_array_append(&read_only_locations, strdup(items[i].c_str())); } cstring_view_array_sort(&read_only_locations); } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } init_matcher(&shortcut_allow_list_matcher, cfg, "shortcut_allow_list"); if (shortcut_allow_list_matcher->empty()) { delete(shortcut_allow_list_matcher); shortcut_allow_list_matcher = nullptr; } init_matcher(&dont_shortcut_matcher, cfg, "dont_shortcut"); init_matcher(&dont_intercept_matcher, cfg, "dont_intercept"); init_matcher(&skip_cache_matcher, cfg, "skip_cache"); shells = new tsl::hopscotch_set(); try { libconfig::Setting& shells_cfg = cfg->getRoot()["processes"]["shells"]; for (int i = 0; i < shells_cfg.getLength(); i++) { shells->emplace(shells_cfg[i]); } } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } if (cfg->exists("quirks")) { const libconfig::Setting& items = cfg->getRoot()["quirks"]; for (int i = 0; i < items.getLength(); i++) { std::string quirk(items[i]); if (quirk == "ignore-tmp-listing") { quirks |= FB_QUIRK_IGNORE_TMP_LISTING; } else if (quirk == "lto-wrapper") { quirks |= FB_QUIRK_LTO_WRAPPER; } else if (quirk == "ignore-time-queries") { quirks |= FB_QUIRK_IGNORE_TIME_QUERIES; } else if (quirk == "ignore-statfs") { quirks |= FB_QUIRK_IGNORE_STATFS; } else if (quirk == "guess-file-params") { quirks |= FB_QUIRK_GUESS_FILE_PARAMS; } else { if (FB_DEBUGGING(FB_DEBUG_CONFIG)) { std::cerr <<"Ignoring unknown quirk: " + quirk << std::endl; } } } } } static void export_sorted(const libconfig::Setting& setting, const std::string env_var_name, std::map* env) { std::vector entries; for (int i = 0; i < setting.getLength(); i++) { entries.emplace_back(setting[i].c_str()); } if (entries.size() > 0) { std::sort(entries.begin(), entries.end()); std::string entries_appended; for (auto entry : entries) { if (entries_appended.length() == 0) { entries_appended.append(entry); } else { entries_appended.append(":" + entry); } } (*env)[env_var_name] = std::string(entries_appended); FB_DEBUG(FB_DEBUG_PROC, " " + env_var_name + "=" + (*env)[env_var_name]); } } static void export_sorted_locations(libconfig::Config *cfg, const char* configuration_name, const std::string env_var_name, std::map* env) { const libconfig::Setting& root = cfg->getRoot(); try { const libconfig::Setting& locations_setting = root[configuration_name]; export_sorted(locations_setting, env_var_name, env); } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } } char** get_sanitized_env(libconfig::Config *cfg, const char *fb_conn_string, bool insert_trace_markers) { const libconfig::Setting& root = cfg->getRoot(); std::map env; FB_DEBUG(FB_DEBUG_PROC, "Passing through environment variables:"); try { const libconfig::Setting& pass_through = root["env_vars"]["pass_through"]; for (int i = 0; i < pass_through.getLength(); i++) { std::string pass_through_env(pass_through[i].c_str()); char * got_env = getenv(pass_through_env.c_str()); if (got_env != NULL) { env[pass_through_env] = std::string(got_env); FB_DEBUG(FB_DEBUG_PROC, " " + std::string(pass_through_env) + "=" + env[pass_through_env]); } } FB_DEBUG(FB_DEBUG_PROC, ""); } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } FB_DEBUG(FB_DEBUG_PROC, "Setting preset environment variables:"); try { const libconfig::Setting& preset = root["env_vars"]["preset"]; for (int i = 0; i < preset.getLength(); i++) { std::string str(preset[i].c_str()); size_t eq_pos = str.find('='); if (eq_pos == std::string::npos) { fb_error("Invalid present environment variable: " + str); abort(); } else { const std::string var_name = str.substr(0, eq_pos); env[var_name] = str.substr(eq_pos + 1); if (str == "CCACHE_DISABLE=1") { ccache_disabled = true; } FB_DEBUG(FB_DEBUG_PROC, " " + var_name + "=" + env[var_name]); } } } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } export_sorted_locations(cfg, "read_only_locations", "FB_READ_ONLY_LOCATIONS", &env); export_sorted_locations(cfg, "ignore_locations", "FB_IGNORE_LOCATIONS", &env); try { const libconfig::Setting& locations_setting = root["processes"]["jobserver_users"]; export_sorted(locations_setting, "FB_JOBSERVER_USERS", &env); } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } const char *ld_preload_value = getenv(LD_PRELOAD); if (ld_preload_value) { env[LD_PRELOAD] = LIBFIREBUILD_SO ":" + std::string(ld_preload_value); } else { env[LD_PRELOAD] = LIBFIREBUILD_SO; } FB_DEBUG(firebuild::FB_DEBUG_PROC, " " LD_PRELOAD "=" + env[LD_PRELOAD]); #ifdef __APPLE__ env["DYLD_FORCE_FLAT_NAMESPACE"] = "0"; FB_DEBUG(firebuild::FB_DEBUG_PROC, " DYLD_FORCE_FLAT_NAMESPACE=" + env["DYLD_FORCE_FLAT_NAMESPACE"]); #endif env["FB_SOCKET"] = fb_conn_string; FB_DEBUG(FB_DEBUG_PROC, " FB_SOCKET=" + env["FB_SOCKET"]); FB_DEBUG(FB_DEBUG_PROC, ""); #ifdef FB_EXTRA_DEBUG if (insert_trace_markers) { env["FB_INSERT_TRACE_MARKERS"] = "1"; } #else (void)insert_trace_markers; #endif char ** ret_env = static_cast(malloc(sizeof(char*) * (env.size() + 1))); auto it = env.begin(); int i = 0; while (it != env.end()) { ret_env[i] = strdup(std::string(it->first + "=" + it->second).c_str()); it++; i++; } ret_env[i] = NULL; return ret_env; } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/config.h000066400000000000000000000054061447164520700176570ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_CONFIG_H_ #define FIREBUILD_CONFIG_H_ #include #include #include #include #include "common/firebuild_common.h" #include "firebuild/exe_matcher.h" #include "firebuild/file_name.h" namespace firebuild { /** global configuration */ extern libconfig::Config * cfg; extern cstring_view_array ignore_locations; extern cstring_view_array read_only_locations; extern ExeMatcher* shortcut_allow_list_matcher; extern ExeMatcher* dont_shortcut_matcher; extern ExeMatcher* dont_intercept_matcher; extern ExeMatcher* skip_cache_matcher; extern tsl::hopscotch_set* shells; extern bool ccache_disabled; /** Store results of processes consuming more CPU time (system + user) in microseconds than this. */ extern int64_t min_cpu_time_u; /** * Give up after shortcut_tries and run the process without shortcutting it. * Value of 0 means trying all candidates. */ extern int shortcut_tries; /** * Maximum size of the files stored in the cache, in bytes. */ extern int64_t max_cache_size; /** * Maximum size of a single cache entry including the referenced objs. */ extern uint64_t max_entry_size; /** Enabled quirks represented as flags. See "quirks" in etc/firebuild.conf. */ extern int quirks; #define FB_QUIRK_IGNORE_TMP_LISTING 0x01 #define FB_QUIRK_LTO_WRAPPER 0x02 #define FB_QUIRK_GUESS_FILE_PARAMS 0x04 #define FB_QUIRK_IGNORE_TIME_QUERIES 0x08 #define FB_QUIRK_IGNORE_STATFS 0x10 void read_config(libconfig::Config *cfg, const char *custom_cfg_file, const std::list& config_strings); /** * Construct a NULL-terminated array of "NAME=VALUE" environment variables * for the build command. The returned stings and array must be free()-d. * * TODO: detect duplicates */ char** get_sanitized_env(libconfig::Config *cfg, const char* fb_conn_string, bool insert_trace_markers); } /* namespace firebuild */ #endif // FIREBUILD_CONFIG_H_ firebuild-0.8.2/src/firebuild/connection_context.h000066400000000000000000000053231447164520700223130ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /** * Context of an interceptor connection. */ #ifndef FIREBUILD_CONNECTION_CONTEXT_H_ #define FIREBUILD_CONNECTION_CONTEXT_H_ #include #include #include "firebuild/cxx_lang_utils.h" #include "firebuild/debug.h" #include "firebuild/epoll.h" #include "firebuild/execed_process.h" #include "firebuild/message_processor.h" #include "firebuild/linear_buffer.h" #include "firebuild/process.h" #include "firebuild/process_tree.h" extern firebuild::Epoll *epoll; namespace firebuild { class ConnectionContext { public: explicit ConnectionContext(int conn) : buffer_(), conn_(conn) {} ~ConnectionContext() { if (proc) { auto exec_child_sock = proc_tree->Pid2ExecChildSock(proc->pid()); if (exec_child_sock) { auto exec_child = exec_child_sock->incomplete_child; exec_child->set_fds(proc->pass_on_fds()); MessageProcessor::accept_exec_child(exec_child, exec_child_sock->sock); proc_tree->DropQueuedExecChild(proc->pid()); } proc->finish(); } assert(conn_ >= 0); epoll->maybe_del_fd(conn_); close(conn_); conn_ = -1; } LinearBuffer& buffer() {return buffer_;} Process * proc = nullptr; private: /** Partial interceptor message including the FBB header */ LinearBuffer buffer_; int conn_; DISALLOW_COPY_AND_ASSIGN(ConnectionContext); }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ inline std::string d(const ConnectionContext& ctx, const int level = 0) { (void)level; /* unused */ return "{ConnectionContext proc=" + d(ctx.proc) + "}"; } inline std::string d(const ConnectionContext *ctx, const int level = 0) { if (ctx) { return d(*ctx, level); } else { return "{ConnectionContext NULL}"; } } } /* namespace firebuild */ #endif // FIREBUILD_CONNECTION_CONTEXT_H_ firebuild-0.8.2/src/firebuild/cxx_lang_utils.h000066400000000000000000000023741447164520700214360ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_CXX_LANG_UTILS_H__ #define FIREBUILD_CXX_LANG_UTILS_H__ // From http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml // A macro to disallow the copy constructor and operator= functions // This should be used in the private: declarations for a class #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ TypeName(const TypeName&); \ void operator=(const TypeName&) #endif // FIREBUILD_CXX_LANG_UTILS_H__ firebuild-0.8.2/src/firebuild/debug.cc000066400000000000000000000142231447164520700176330ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/debug.h" #include #include #include #include #include "common/debug_sysflags.h" #include "firebuild/exe_matcher.h" #include "firebuild/process.h" namespace firebuild { ExeMatcher* debug_filter {nullptr}; __thread bool debug_suppressed {false}; int32_t debug_flags = 0; void fb_error(const std::string &msg) { fprintf(stderr, "FIREBUILD ERROR: %s\n", msg.c_str()); } void fb_debug(const std::string &msg) { if (!debug_suppressed) { fprintf(stderr, "FIREBUILD: %s\n", msg.c_str()); } } std::string d(const std::string& str, const int level) { (void)level; /* unused */ std::string ret = "\""; for (unsigned char c : str) { if (c < 0x20 || c >= 0x7f) { const char *hex = "0123456789ABCDEF"; ret += "\\x"; ret += hex[c / 16]; ret += hex[c % 16]; } else if (c == '\\' || c == '"') { ret += "\\"; ret += c; } else { ret += c; } } ret += "\""; return ret; } std::string d(const char *str, const int level) { if (str) { return d(std::string(str), level); } else { return "NULL"; } } std::string d(const struct stat64& st, const int level) { (void)level; /* unused */ char *modestring_ptr = NULL; size_t modestring_sizeloc = -1; FILE *f = open_memstream(&modestring_ptr, &modestring_sizeloc); debug_mode_t(f, st.st_mode); fclose(f); std::string ret = std::string("{stat mode=") + modestring_ptr + " size=" + d(st.st_size) + "}"; free(modestring_ptr); return ret; } std::string d(const struct stat64 *st, const int level) { if (st) { return d(*st, level); } else { return "{stat NULL}"; } } std::string pretty_timestamp() { struct timeval tv; gettimeofday(&tv, NULL); time_t t = tv.tv_sec; struct tm local; localtime_r(&t, &local); int abs_diff_min = std::abs(local.tm_gmtoff) / 60; char buf[64]; /* Note: strftime() doesn't support sub-seconds. */ #ifdef __APPLE__ snprintf(buf, sizeof(buf), "%d-%02d-%02d %02d:%02d:%02d.%06d %c%02d%02d", #else snprintf(buf, sizeof(buf), "%d-%02d-%02d %02d:%02d:%02d.%06ld %c%02d%02d", #endif 1900 + local.tm_year, 1 + local.tm_mon, local.tm_mday, local.tm_hour, local.tm_min, local.tm_sec, tv.tv_usec, local.tm_gmtoff >= 0 ? '+' : '-', abs_diff_min / 60, abs_diff_min % 60); return std::string(buf); } struct flag { const char *name; int32_t value; }; /* Keep this in sync with debug.h! */ static struct flag available_flags[] = { { "config", FB_DEBUG_CONFIG }, { "proc", FB_DEBUG_PROC }, { "proctree", FB_DEBUG_PROCTREE }, { "communication", FB_DEBUG_COMM }, { "comm", FB_DEBUG_COMM }, { "filesystem", FB_DEBUG_FS }, { "fs", FB_DEBUG_FS }, { "hash", FB_DEBUG_HASH }, { "cache", FB_DEBUG_CACHE }, { "deterministic-cache", FB_DEBUG_DETERMINISTIC_CACHE }, { "caching", FB_DEBUG_CACHING }, { "shortcut", FB_DEBUG_SHORTCUT }, { "pipe", FB_DEBUG_PIPE }, { "function", FB_DEBUG_FUNC }, { "func", FB_DEBUG_FUNC }, { "time", FB_DEBUG_TIME }, { NULL, 0 } }; #define SEPARATORS ",:" int32_t parse_debug_flags(const std::string& str) { int32_t flags = 0; bool all = false; size_t pos = 0; while (pos < str.length()) { size_t start = str.find_first_not_of(SEPARATORS, pos); if (start == std::string::npos) { break; } size_t end = str.find_first_of(SEPARATORS, start); if (end == std::string::npos) { end = str.length(); } std::string flag_str = str.substr(start, end - start); pos = end; bool found = false; if (flag_str == "all") { all = true; found = true; } else if (flag_str == "help") { fprintf(stderr, "Firebuild: available debug flags are:"); int id = 0; while (available_flags[id].name != NULL) { if (id > 0 && available_flags[id].value == available_flags[id - 1].value) { fprintf(stderr, " or "); } else { fprintf(stderr, "\n "); } fprintf(stderr, "%s", available_flags[id].name); id++; } fprintf(stderr, "\n all\n"); exit(EXIT_SUCCESS); } else { int id = 0; while (available_flags[id].name != NULL) { if (flag_str == available_flags[id].name) { flags |= available_flags[id].value; found = true; break; } id++; } } if (!found) { fprintf(stderr, "Firebuild: Unrecognized debug flag %s\n", flag_str.c_str()); } } if (all) { flags ^= 0xFFFF; } return flags; } void init_debug_filter(const std::string commands) { /* Allow passing debug filter to the firebuild binary multiple times. */ if (debug_filter) { delete(debug_filter); } debug_filter = new firebuild::ExeMatcher(); size_t pos = 0; while (pos < commands.length()) { size_t start = commands.find_first_not_of(",", pos); if (start == std::string::npos) { break; } size_t end = commands.find_first_of(",", start); if (end == std::string::npos) { end = commands.length(); } debug_filter->add(commands.substr(start, end - start)); pos = end; } } #ifdef FB_EXTRA_DEBUG std::vector fd_ages; int method_tracker_level = 0; #endif } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/debug.h000066400000000000000000000333261447164520700175020ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_DEBUG_H_ #define FIREBUILD_DEBUG_H_ #include #include #include #include #include #include #include #include "firebuild/cxx_lang_utils.h" namespace firebuild { class ExeMatcher; extern ExeMatcher* debug_filter; extern __thread bool debug_suppressed; class Process; /** Print error message */ void fb_error(const std::string &msg); /** Possible debug flags. Keep in sync with debug.cc! */ enum { /* Firebuild's configuration */ FB_DEBUG_CONFIG = 1 << 0, /* Events with one process, e.g. shortcut, exit */ FB_DEBUG_PROC = 1 << 1, /* How processes are organized into ProcTree */ FB_DEBUG_PROCTREE = 1 << 2, /* Communication */ FB_DEBUG_COMM = 1 << 3, /* File system */ FB_DEBUG_FS = 1 << 4, /* Checksum computation */ FB_DEBUG_HASH = 1 << 5, /* The data stored in the cache */ FB_DEBUG_CACHE = 1 << 6, /* Sort the records in the cache */ FB_DEBUG_DETERMINISTIC_CACHE = 1 << 7, /* Placing in / retrieving from the cache */ FB_DEBUG_CACHING = 1 << 8, /* Shortcutting */ FB_DEBUG_SHORTCUT = 1 << 9, /* Emulating pipes */ FB_DEBUG_PIPE = 1 << 10, /* Entering and leaving functions */ FB_DEBUG_FUNC = 1 << 11, /* Similar to bash's "time" */ FB_DEBUG_TIME = 1 << 12, }; /** * Test if debugging this kind of events is enabled. */ #define FB_DEBUGGING(flag) (((firebuild::debug_flags) & flag) && !firebuild::debug_suppressed) /** * Print debug message if the given debug flag is enabled. */ #define FB_DEBUG(flag, msg) if (FB_DEBUGGING(flag)) \ firebuild::fb_debug(msg) /** Print debug message */ void fb_debug(const std::string &msg); /** Current debugging flags */ extern int32_t debug_flags; /** * Parse the debug flags similarly to GLib's g_parse_debug_string(). * * Currently case-sensitive (i.e. all lowercase is expected). */ int32_t parse_debug_flags(const std::string& str); void init_debug_filter(const std::string commands); static inline std::string d(int value, const int level = 0) { (void)level; /* unused */ return std::to_string(value); } static inline std::string d(long value, const int level = 0) { /* NOLINT(runtime/int) */ (void)level; /* unused */ return std::to_string(value); } static inline std::string d(long long value, const int level = 0) { /* NOLINT(runtime/int) */ (void)level; /* unused */ return std::to_string(value); } static inline std::string d(unsigned int value, const int level = 0) { (void)level; /* unused */ return std::to_string(value); } static inline std::string d(unsigned long value, const int level = 0) { /* NOLINT(runtime/int) */ (void)level; /* unused */ return std::to_string(value); } static inline std::string d(unsigned long long value, /* NOLINT(runtime/int) */ const int level = 0) { (void)level; /* unused */ return std::to_string(value); } static inline std::string d(bool value, const int level = 0) { (void)level; /* unused */ return value ? "true" : "false"; } /** * Get a human friendly representation of a string, inside double * quotes, for debugging purposes. */ std::string d(const std::string& str, const int level = 0); /** * Get a human friendly representation of a C string, inside double * quotes (unless NULL), for debugging purposes. */ std::string d(const char *str, const int level = 0); /** * Get a human friendly representation of a struct stat, for debugging purposes. */ std::string d(const struct stat64& st, const int level = 0); /** * Get a human friendly representation of a struct stat, for debugging purposes. */ std::string d(const struct stat64 *st, const int level = 0); /** * Get a human friendly representation of an array of anything that is d()-debuggable, * enclosed in square brackets, separated by commas, like: * * [item1, item2, item3] */ template static inline std::string d(const std::vector& arr, const int level = 0) { std::string res = "["; bool first_val = true; unsigned int repeats = 1; const T* prev_val; for (const T& val : arr) { if (!first_val) { if (*prev_val == val) { repeats += 1; continue; } else { if (repeats == 1) { res += ", "; } else { res += " /* times " + std::to_string(repeats) + " */, "; repeats = 1; } } } res += d(val, level); prev_val = &val; first_val = false; } if (repeats != 1) { res += " /* times " + std::to_string(repeats) + " */"; } res += "]"; return res; } template static inline std::string d(const std::vector *arr, const int level = 0) { if (arr) { return d(*arr, level); } else { return "{std::vector NULL}"; } } /** * Debug a shared_ptr of anything that is d()-debuggable. */ template static inline std::string d(const std::shared_ptr& ptr, const int level = 0) { return d(ptr.get(), level); } /* Convenience wrapper around our various d(...) debugging functions. * Instead of returning a std::string, as done by d(), this gives the raw C char* pointer * which is valid only inside the expression where D() is called. */ #define D(var) firebuild::d(var).c_str() #ifdef FB_EXTRA_DEBUG /* The age of each fd, for debugging purposes. */ extern std::vector fd_ages; #endif /* Increase the "age" of a given fd. */ static inline void bump_fd_age(int fd) { (void)fd; /* unused in non-debug build */ #ifdef FB_EXTRA_DEBUG if (fd >= static_cast(fd_ages.size())) { fd_ages.resize(fd + 1); } fd_ages[fd]++; #endif } /* Debug a file descriptor number. * If its age hasn't been bumped then report the number only, e.g. "7". * If its age has been bumped then report the fd number and with its age, e.g. "7.1", "7.2" etc. */ static inline std::string d_fd(int fd) { #ifdef FB_EXTRA_DEBUG if (fd >= 0 && fd < static_cast(fd_ages.size()) && fd_ages[fd] > 0) { return std::to_string(fd) + "." + std::to_string(fd_ages[fd]); } #endif return std::to_string(fd); } /* Convenience wrapper around our d_fd(). * Instead of returning a std::string, as done by d_fd(), this gives the raw C char* pointer * which is valid only inside the expression where D_FD() is called. */ #define D_FD(fd) firebuild::d_fd(fd).c_str() /** * Get a human friendly representation of the current local time, for * debugging purposes. * * The format was chosen as a compromise between standards, common * practices, best readability, and best accuracy. It currently looks * like: * * 2019-12-31 23:59:59.999999 +0100 */ std::string pretty_timestamp(); #ifndef FB_EXTRA_DEBUG #define TRACK(...) #define TRACKX(...) #else /* Global, shared across all MethodTrackers, for nice indentation */ extern int method_tracker_level; /** * Track entering and leaving a function (or any brace-block of code). * Print some variables when entering. * * @param flag Do the logging if FB_DEBUG_FUNC or any of these debug flags specified here is enabled * @param fmt printf format string, may be empty * @param ... The additional parameters for printf */ #define TRACK(flag, fmt, ...) \ firebuild::MethodTracker method_tracker(__func__, __FILE__, __LINE__, flag, 0, 0, \ "", NULL, NULL, fmt, ##__VA_ARGS__) /** * Track entering and leaving a function (or any brace-block of code). * Print one variable both when entering and leaving. * Print some more variables when entering. * * The variable to be printed when leaving the block (obj_ptr) has to have a corresponding global * std::string d(classname *obj_ptr, int level) debugging method, which will be used to format it. * The variable is remembered via its pointer, so if you reassign it in the function body then you * can't print it upon exit. * * Having to pass the classsname of obj_ptr is a technical necessity, remove it if you know how to. * * @param flag Do the logging if FB_DEBUG_FUNC or any of these debug flags specified here is enabled * @param print_obj_on_enter Whether to log obj_ptr when entering the block * @param print_obj_on_leave Whether to log obj_ptr when leaving the block * @param classname The classname of obj_ptr, without 'const', or '*' for pointer * @param obj_ptr The object to print on entering of leaving, of type 'classname *'. * @param fmt printf format string for the additional variables to log when entering, may be empty * @param ... The additional parameters for printf */ #define TRACKX(flag, print_obj_on_enter, print_obj_on_leave, classname, obj_ptr, fmt, ...) \ /* Find the address of the correct ovedloaded d() method belonging to obj_ptr. */ \ /* Needs to happen in the context of the macro's caller, because debug.h doesn't see the */ \ /* specific d() method, so d() inside MethodTracker() would pick the one taking a boolean. */ \ std::string (*resolved_d)(const classname *, int) = &firebuild::d; \ firebuild::MethodTracker method_tracker(__func__, __FILE__, __LINE__, flag, \ print_obj_on_enter, print_obj_on_leave, \ #obj_ptr, obj_ptr, resolved_d, \ fmt, ##__VA_ARGS__) template class MethodTracker { public: MethodTracker(const char *func, const char *file, int line, int flag, bool print_obj_on_enter, bool print_obj_on_leave, const char *obj_name, const T *obj_ptr, std::string (*resolved_d)(const T *, int), const char *fmt, ...) __attribute__((format(printf, 11, 12))) : func_(func), file_(file), line_(line), flag_(flag | FB_DEBUG_FUNC), print_obj_on_leave_(print_obj_on_leave), obj_name_(obj_name), obj_ptr_(obj_ptr), resolved_d_(resolved_d) { if (FB_DEBUGGING(flag_)) { const char *last_slash = strrchr(file_, '/'); if (last_slash) { file_ = last_slash + 1; } char buf[1024]; size_t offset = snprintf(buf, sizeof(buf), "%*s-> %s() (%s:%d)%s", 2 * method_tracker_level, "", func_, file_, line_, print_obj_on_enter || fmt[0] ? " " : ""); if (print_obj_on_enter && offset < sizeof(buf)) { offset += snprintf(buf + offset, sizeof(buf) - offset, "%s=%s%s", obj_name_, ((*resolved_d_)(obj_ptr_, 0)).c_str(), fmt[0] ? ", " : ""); } if (offset < sizeof(buf)) { va_list ap; va_start(ap, fmt); vsnprintf(buf + offset, sizeof(buf) - offset, fmt, ap); va_end(ap); } FB_DEBUG(flag_, buf); method_tracker_level++; } } ~MethodTracker() { if (FB_DEBUGGING(flag_)) { method_tracker_level--; char buf[1024]; size_t offset = snprintf(buf, sizeof(buf), "%*s<- %s() (%s:%d)", 2 * method_tracker_level, "", func_, file_, line_); if (print_obj_on_leave_ && offset < sizeof(buf)) { snprintf(buf + offset, sizeof(buf) - offset, " %s=%s", obj_name_, ((*resolved_d_)(obj_ptr_, 0)).c_str()); } FB_DEBUG(flag_, buf); } } private: const char *func_; const char *file_; int line_; int flag_; bool print_obj_on_leave_; const char *obj_name_; const T *obj_ptr_; std::string (*resolved_d_)(const T *, int); DISALLOW_COPY_AND_ASSIGN(MethodTracker); }; #endif /* NDEBUG */ #ifndef NDEBUG /* * Like an "assert(a op b)" statement, "assert_cmp(a, op, b)" makes sure that the "a op b" condition * is true. Note the required commas. Example: "assert_cmp(foo, >=, 0)". * * In case of failure, prints both values. * * Based on the idea of GLib's g_assert_cmp*(). With C++'s overloading we can do better, though. * * The two values can be of any type that's printable using d() and comparable, and accordingly, * they are indeed printed using d() if the comparison fails. * * Note: because d(NULL) doesn't work, you can't do "assert_cmp(p, ==, NULL)" or * "assert_cmp(p, !=, NULL)". For the former, use our "assert_null(p)". For the latter, use the * standard "assert(p)". */ #define assert_cmp(a, op, b) do { \ if (!(a op b)) { \ std::string source = #a " " #op " " #b; \ std::string actual = firebuild::d(a) + " " + #op + " " + firebuild::d(b); \ fprintf(stderr, "Assertion `%s': `%s' failed.\n", source.c_str(), actual.c_str()); \ assert(0 && "see previous message"); \ } \ } while (0) /* * Like an assert(p == NULL), but if fails then prints the value using d(). */ #define assert_null(p) do { \ if (p != NULL) { \ std::string source = #p " != NULL"; \ std::string actual = firebuild::d(p) + " != NULL"; \ fprintf(stderr, "Assertion `%s': `%s' failed.\n", source.c_str(), actual.c_str()); \ assert(0 && "see previous message"); \ } \ } while (0) #else #define assert_cmp(a, op, b) #define assert_null(p) #endif /* NDEBUG */ } /* namespace firebuild */ #endif // FIREBUILD_DEBUG_H_ firebuild-0.8.2/src/firebuild/epoll.cc000066400000000000000000000211261447164520700176600ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/epoll.h" #include #ifdef __APPLE__ #include #include #include #else #include #endif #include #include #include #include "common/firebuild_common.h" #include "firebuild/debug.h" #include "firebuild/utils.h" namespace firebuild { /* singleton */ Epoll *epoll = nullptr; void Epoll::delete_closed_fd_context(int fd) { if (fd_contexts_[fd].callback_user_data) { #ifdef __APPLE__ struct kevent fake_event; EV_SET(&fake_event, fd, EVFILT_READ|EVFILT_WRITE, EV_EOF, 0, 0, 0); #else struct epoll_event fake_event {EPOLLHUP, {}}; #endif set_event_fd(&fake_event, fd); (*fd_contexts_[fd].callback)(&fake_event, fd_contexts_[fd].callback_user_data); } } Epoll::~Epoll() { close(main_fd_); for (size_t fd = 0; fd < fd_contexts_.size(); fd++) { if (fd_contexts_[fd].callback != nullptr) { /* This fd is still open while firebuild is quitting. This may be connected to a * orphan process. Simulate the termination of the process by closing the fd and letting * the callback to act on it and free the user data. */ close(fd); delete_closed_fd_context(fd); } } } void Epoll::ensure_room_fd(int fd) { if (fd >= static_cast(fd_contexts_.size())) { fd_contexts_.resize(fd + 1); } } bool Epoll::is_added_fd(int fd) { return fd < static_cast(fd_contexts_.size()) && fd_contexts_[fd].callback != nullptr; } int Epoll::remap_to_not_added_fd(int fd) { assert(fd_contexts_[fd].callback); std::vector close_fds = {fd}; do { int ret = dup(fd); if (is_added_fd(ret)) { close_fds.push_back(ret); } else { for (int close_fd : close_fds) { closed_context_fds_.push(close_fd); } return ret; } } while (0); return -1; } void Epoll::add_fd(int fd, uint32_t events, void (*callback)(const struct epoll_event* event, void *callback_user_data), void *callback_user_data) { ensure_room_fd(fd); assert(fd_contexts_[fd].callback == nullptr); fd_contexts_[fd].callback = callback; fd_contexts_[fd].callback_user_data = callback_user_data; fds_++; #ifdef __APPLE__ timespec ts = {0, 0}; struct kevent ke = {static_cast(fd), static_cast((events & EPOLLIN) ? EVFILT_READ : EVFILT_WRITE), EV_ADD | KEVENT_FLAG_IMMEDIATE | EV_RECEIPT, 0, reinterpret_cast(nullptr), &ts}; int kevent_ret = kevent(main_fd_, &ke, 1, nullptr, 0, nullptr); if (kevent_ret == -1) { perror("kevent"); abort(); } assert(kevent_ret == 0); #else struct epoll_event ee; memset(&ee, 0, sizeof(ee)); ee.events = events; set_event_fd(&ee, fd); if (epoll_ctl(main_fd_, EPOLL_CTL_ADD, fd, &ee) == -1) { fb_perror("Error adding epoll fd"); abort(); } #endif } void Epoll::del_fd(int fd) { ensure_room_fd(fd); assert(fd_contexts_[fd].callback != nullptr); fd_contexts_[fd].callback = nullptr; assert_cmp(fds_, >, 0); fds_--; #ifdef __APPLE__ timespec ts = {0, 0}; struct kevent ke = {static_cast(fd), EV_DELETE, EV_ADD | KEVENT_FLAG_IMMEDIATE | EV_RECEIPT, 0, reinterpret_cast(nullptr), &ts}; int kevent_ret = kevent(main_fd_, &ke, 1, nullptr, 0, nullptr); /* With closing the fd the monitored events are automatically cleared. */ if (kevent_ret == -1 && errno != EINVAL) { perror("kevent"); abort(); } #else epoll_ctl(main_fd_, EPOLL_CTL_DEL, fd, NULL); #endif /* When deleting an fd, make sure to also delete it from the yet unprocessed part of * epoll_wait()'s returned events. Do this by setting .data.fd to -1. * * Example: epoll_wait() returns a set of two events, one for fd1, one for fd2. The callback of * fd1 might remove fd2 from the epoll set, or might even close fd2, and might even open another * file which happens to receive the same file descriptor. Calling fd2's registered callback in * the next iteration of process_all_events()'s loop could result in uncontrollable bad * consequences. */ for (int i = event_current_ + 1; i < event_count_; i++) { if (event_fd(&events_[i]) == fd) { set_event_fd(&events_[i], -1); break; } } } void Epoll::maybe_del_fd(int fd) { /* Note: if fd is not added to the epoll set then there's no way it could be present anywhere in * events_. So in that case it's okay to skip the tricky loop of del_fd(), too. */ if (is_added_fd(fd)) { del_fd(fd); } } int Epoll::add_timer(int ms, void (*callback)(void *callback_user_data), void *callback_user_data) { /* Find the first empty slot. */ int timer_id; for (timer_id = 0; timer_id <= largest_timer_id_; timer_id++) { if (timer_contexts_[timer_id].callback == nullptr) { break; } } if (timer_id > largest_timer_id_) { largest_timer_id_++; if (static_cast(timer_id) == timer_contexts_.size()) { timer_contexts_.resize(timer_id + 1); } } /* Set the callback. */ timer_contexts_[timer_id].callback = callback; timer_contexts_[timer_id].callback_user_data = callback_user_data; /* Compute when to fire. */ struct timespec now, delay; clock_gettime(CLOCK_MONOTONIC, &now); delay.tv_sec = ms / 1000; delay.tv_nsec = (ms % 1000) * 1000 * 1000; timespecadd(&now, &delay, &timer_contexts_[timer_id].when); /* Update next_timer_ to point to the timeout that will elapse next. */ if (next_timer_ < 0 || timespeccmp(&timer_contexts_[timer_id].when, &timer_contexts_[next_timer_].when, <)) { next_timer_ = timer_id; } return timer_id; } void Epoll::del_timer(int timer_id) { assert(timer_contexts_[timer_id].callback != nullptr); timer_contexts_[timer_id].callback = nullptr; /* Cap largest_timer_id to point to the new largest used timer slot. */ while (largest_timer_id_ >= 0 && timer_contexts_[largest_timer_id_].callback == nullptr) { largest_timer_id_--; } /* Update next_timer_. */ if (timer_id == next_timer_) { /* Iterate over the array to find the closest in time, skipping deleted entries. */ next_timer_ = -1; for (int i = 0; i <= largest_timer_id_; i++) { if (timer_contexts_[i].callback != nullptr && (next_timer_ < 0 || timespeccmp(&timer_contexts_[i].when, &timer_contexts_[next_timer_].when, <))) { next_timer_ = i; } } } /* Note that the trick we do in del_fd() is not necessary here. If the callback of a timer deletes * another timer, or creates a new one, maybe even occupying a deleted one's id, the "worst" that * can happen is that the ongoing process_all_events() will already execute that timer if it has * already elapsed. */ } void Epoll::wait() { #ifndef __APPLE__ int timeout_ms = -1; #else struct timespec diff; #endif if (next_timer_ >= 0) { struct timespec now; #ifndef __APPLE__ struct timespec diff; #endif clock_gettime(CLOCK_MONOTONIC, &now); if (timespeccmp(&timer_contexts_[next_timer_].when, &now, <)) { /* The next timer should already fire, calling timespecsub() would give a negative diff. Save the epoll_wait() system call and return immediately to process the timers. */ event_count_ = 0; return; } timespecsub(&timer_contexts_[next_timer_].when, &now, &diff); #ifndef __APPLE__ timeout_ms = diff.tv_sec * 1000 + diff.tv_nsec / (1000 * 1000); #endif } #ifdef __APPLE__ event_count_ = TEMP_FAILURE_RETRY( kevent(main_fd_, nullptr, 0, events_, sizeof(events_) / sizeof(events_[0]), (next_timer_ >= 0) ? &diff : nullptr)); #else event_count_ = TEMP_FAILURE_RETRY( epoll_wait(main_fd_, events_, sizeof(events_) / sizeof(events_[0]), timeout_ms)); #endif } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/epoll.h000066400000000000000000000175261447164520700175330ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_EPOLL_H_ #define FIREBUILD_EPOLL_H_ #ifdef __APPLE__ #include #include #include // TODO(rbalint) this is hackish, we should rather have a wrapper class #define epoll_event kevent #define EPOLLIN 0x001 #define EPOLLOUT 0x004 #else #include #endif #include #include #include #include #include #include "firebuild/utils.h" namespace firebuild { typedef struct fd_context_ { /* The callback to call for this fd, passing the struct epoll_event as returned by epoll_wait(), * as well callback_user_data. Non-null if and only if the given fd is added to epollfd. */ #ifdef SET_EV void (*callback)(const struct kevent* event, void *callback_user_data); #else void (*callback)(const struct epoll_event* event, void *callback_user_data); #endif /* User data, as usual for callbacks. */ void *callback_user_data; } fd_context; typedef struct timer_context_ { /* The callback to call for this timer, passing callback_user_data. Non-null if and only if the * given timer exists. */ void (*callback)(void *callback_user_data); /* User data, as usual for callbacks. */ void *callback_user_data; /* When to fire this, according to CLOCK_MONOTONIC. */ struct timespec when; } timer_context; class Epoll { public: Epoll() { #ifdef __APPLE__ main_fd_ = kqueue(); if (main_fd_ == -1) { perror("kqueue"); abort(); } #else main_fd_ = epoll_create1(EPOLL_CLOEXEC); #endif } ~Epoll(); /** Whether we've added an fd to epollfd (according to our own bookkeeping */ bool is_added_fd(int fd); /** * Dup already added fd to an fd that's not added yet to epollfd. * Also close fd. */ int remap_to_not_added_fd(int fd); /** Thin wrapper around epoll_ctl(). Makes sure that the fd isn't added yet to epollfd * (according to our own bookkeeping) and adds it with the given parameters. */ void add_fd(int fd, uint32_t events, void (*callback)(const struct epoll_event* event, void *callback_user_data), void *callback_user_data); /** Thin wrapper around epoll_ctl(). Makes sure that the fd is already added to epollfd * (according to our own bookkeeping) and removes it. */ void del_fd(int fd); /** Number of added and not removed fds. */ size_t fds() const {return fds_;} /** Thin wrapper around epoll_ctl(). Checks if fd is already added to epollfd * (according to our own bookkeeping) and if so then removes it. */ void maybe_del_fd(int fd); /** Add a one-shot timer, return its id */ int add_timer(int ms, void (*callback)(void *callback_user_data), void *callback_user_data); /** Delete a one-shot timer by its id, before it fires. Make sure NOT to call this after the timer * has fired! Don't even call it from the timer's own callback, the timer will clean up itself * automatically. */ void del_timer(int timer_id); #ifdef __APPLE__ static int event_fd(const struct kevent* event) { return event->ident; } static void set_event_fd(struct kevent* event, int fd) { event->ident = static_cast(fd); } static bool ready_for_read(const struct epoll_event* event) { return event->ident & EVFILT_READ && !(event->ident & EV_EOF); } static bool ready_for_write(const struct epoll_event* event) { return event->ident & EVFILT_WRITE && !(event->ident & EV_EOF); } #else static int event_fd(const struct epoll_event* event) { return event->data.fd; } static void set_event_fd(struct epoll_event* event, int fd) { event->data.fd = fd; } static bool ready_for_read(const struct epoll_event* event) { return event->events & EPOLLIN; } static bool ready_for_write(const struct epoll_event* event) { return event->events & EPOLLOUT; } #endif /** Wrapper around epoll_wait(). Places the result in events_ and event_count_. */ void wait(); /** Call the relevant callback for all the returned events in events_, and all the expired * timers. */ void process_all_events() { /* Loop through the file descriptors for which the close() were missed. */ while (!closed_context_fds_.empty()) { int fd = closed_context_fds_.front(); delete_closed_fd_context(fd); closed_context_fds_.pop(); close(fd); } /* Loop through the file descriptors. * In case of a signal, event_count_ might be -1, but that's fine for this loop. */ for (event_current_ = 0; event_current_ < event_count_; event_current_++) { #ifdef __APPLE__ const int16_t filter = events_[event_current_].filter; if (filter != EVFILT_READ && filter != EVFILT_WRITE) { continue; } #endif int fd = event_fd(&events_[event_current_]); /* fd might be -1, see in del_fd() for explanation. Skip those. * For the rest, call the appropriate callback. */ if (fd >= 0) { assert(fd_contexts_[fd].callback != nullptr); (*fd_contexts_[fd].callback)(&events_[event_current_], fd_contexts_[fd].callback_user_data); } } /* Loop through the timers. Fire the elapsed ones in no particular order. */ if (largest_timer_id_ >= 0) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); for (int i = 0; i <= largest_timer_id_; i++) { /* Skip the inactive entries (callback is null). */ if (timer_contexts_[i].callback != nullptr && timespeccmp(&timer_contexts_[i].when, &now, <)) { (*timer_contexts_[i].callback)(timer_contexts_[i].callback_user_data); del_timer(i); } } } } private: /** Make sure epoll_fd_contexts is large enough to contain fd. */ inline void ensure_room_fd(int fd); /** Clean up context for a closed fd. */ void delete_closed_fd_context(int fd); /* Our main epoll fd. */ int main_fd_ = -1; /* Number of added fds not removed yet. */ size_t fds_ = 0; /* For each fd, tells its current role in epollfd. The entry is "active" (part of epoll's set) * if and only if its callback is non-null. */ std::vector fd_contexts_ {}; /* Closed fds that still have context in fd_contexts_. Those contexts need to be cleared * up before using the fds again with epoll_ctl(). */ std::queue closed_context_fds_ {}; /* For each timer id, tells when to fire and what to call. The entry is "active" if and only its * callback is non-null. */ std::vector timer_contexts_ {}; /* Index to the last active item in timer_contexts_, or -1. */ int largest_timer_id_ = -1; /* Index to the timer that will fire next, or -1. */ int next_timer_ = -1; /* The place where epoll_wait() store the returned events. */ struct epoll_event events_[32]; /* The return value of epoll_wait(), i.e. the number of events placed in events_, or -1 */ int event_count_ = 0; /* The index to the event in events_ we're currently processing. */ int event_current_ = 0; }; /* singleton */ extern Epoll *epoll; } /* namespace firebuild */ #endif // FIREBUILD_EPOLL_H_ firebuild-0.8.2/src/firebuild/exe_matcher.cc000066400000000000000000000034401447164520700210300ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/exe_matcher.h" #include #include #include "firebuild/execed_process.h" #include "firebuild/file_name.h" namespace firebuild { bool ExeMatcher::match(const ExecedProcess* const proc) const { return match(proc->executable(), proc->executed_path(), proc->args().size() > 0 ? proc->args()[0] : ""); } bool ExeMatcher::match(const FileName* exe_file, const FileName* executed_file, const std::string& arg0) const { return match(exe_file->to_string()) || match(arg0) || (executed_file == exe_file ? false : (executed_file ? match(executed_file->to_string()) : false)); } bool ExeMatcher::match(const std::string& exe) const { size_t pos = exe.rfind('/'); const std::string exe_base = exe.substr(pos == std::string::npos ? 0 : pos + 1); return base_names_.find(exe_base) != base_names_.end() || full_names_.find(exe) != full_names_.end(); } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/exe_matcher.h000066400000000000000000000034621447164520700206760ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_EXE_MATCHER_H_ #define FIREBUILD_EXE_MATCHER_H_ #include #include #include "firebuild/file_name.h" namespace firebuild { class ExecedProcess; /** * Class for checking if either exe or arg0 matches any of the base names of full names (paths). */ class ExeMatcher { public: ExeMatcher() : base_names_(), full_names_() {} bool match(const ExecedProcess* const proc) const; bool match(const FileName* exe_file, const FileName* executed_file, const std::string& arg0) const; bool empty() const {return base_names_.empty() && full_names_.empty();} void add(const std::string name) { if (name.find('/') == std::string::npos) { base_names_.insert(name); } else { full_names_.insert(name); } } private: bool match(const std::string& exe) const; tsl::hopscotch_set base_names_; tsl::hopscotch_set full_names_; }; } /* namespace firebuild */ #endif // FIREBUILD_EXE_MATCHER_H_ firebuild-0.8.2/src/firebuild/execed_process.cc000066400000000000000000000477121447164520700215510ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/execed_process.h" #include #include #include #include "common/platform.h" #include "firebuild/config.h" #include "firebuild/execed_process_cacher.h" #include "firebuild/forked_process.h" #include "firebuild/process_debug_suppressor.h" #include "firebuild/process_tree.h" #include "firebuild/utils.h" extern bool generate_report; namespace firebuild { ExecedProcess::ExecedProcess(const int pid, const int ppid, const FileName *initial_wd, const FileName *executable, const FileName *executed_path, char* original_executed_path, const std::vector& args, const std::vector& env_vars, const std::vector& libs, const mode_t umask, Process * parent, const bool debug_suppressed, std::vector>* fds) : Process(pid, ppid, parent ? parent->exec_count() + 1 : 1, initial_wd, umask, parent, fds, debug_suppressed), can_shortcut_(true), was_shortcut_(false), maybe_shortcutable_ancestor_( (parent && parent->exec_point()) ? parent->exec_point()->closest_shortcut_point() : nullptr), initial_wd_(initial_wd), wds_(), failed_wds_(), args_(args), env_vars_(env_vars), executable_(executable), executed_path_(executed_path), original_executed_path_(original_executed_path), libs_(libs), file_usages_(), created_pipes_() { TRACKX(FB_DEBUG_PROC, 0, 1, Process, this, "pid=%d, ppid=%d, initial_wd=%s, executable=%s, umask=%03o, parent=%s", pid, ppid, D(initial_wd), D(executable), umask, D(parent)); if (parent) { assert(parent->state() == FB_PROC_TERMINATED); /* add as exec child of parent */ fork_point_ = parent->fork_point(); parent->set_exec_pending(false); parent->reset_file_fd_pipe_refs(); parent->set_exec_child(this); } } ExecedProcess* ExecedProcess::common_exec_ancestor(ExecedProcess* other) { TRACKX(FB_DEBUG_PROC, 0, 1, Process, this, "other=%s", D(other)); tsl::hopscotch_set my_ancestors; tsl::hopscotch_set others_ancestors; ExecedProcess* curr_this = this; ExecedProcess* curr_other = other; /* Walk up on each branch in parallel collecting the visited nodes in two sets * to find the common ancestor when a newly visited node is in the other branch's set. */ do { if (curr_this) { if (others_ancestors.find(curr_this) != others_ancestors.end()) { return curr_this; } else { my_ancestors.insert(curr_this); curr_this = curr_this->parent_exec_point(); } } if (curr_other) { if (my_ancestors.find(curr_other) != my_ancestors.end()) { return curr_other; } else { others_ancestors.insert(curr_other); curr_other = curr_other->parent_exec_point(); } } } while (curr_this || curr_other); assert(0 && "not reached"); return nullptr; } void ExecedProcess::set_parent(Process *parent) { /* set_parent() is called only on processes which are created by posix_spawn(). */ assert(parent); ExecedProcess* parent_exec_point = parent->exec_point(); assert(parent_exec_point); Process::set_parent(parent); fork_point_ = parent->fork_point(); maybe_shortcutable_ancestor_ = parent_exec_point->closest_shortcut_point(); } void ExecedProcess::initialize() { TRACKX(FB_DEBUG_PROC, 0, 1, Process, this, ""); /* Propagate the opening of the executable and libraries upwards as * regular file open events. */ if (parent_exec_point()) { FileUsageUpdate exe_update = FileUsageUpdate::get_from_open_params(executable(), O_RDONLY, 0, 0, false); parent_exec_point()->register_file_usage_update(executable(), exe_update); for (const auto& lib : libs_) { FileUsageUpdate lib_update = FileUsageUpdate::get_from_open_params(lib, O_RDONLY, 0, 0, false); parent_exec_point()->register_file_usage_update(lib, lib_update); } } /* Find the inherited files. * Group them according to the open file description they point to. * E.g. if fd 1 & 2 are dups of each other, but fd 3 is the same file opened by name separately * then build up [[1, 2], [3]]. * The outer list (according to the lowest fd) and the inner lists are all sorted. */ std::vector inherited_files; /* This iterates over the fds in increasing order. */ for (auto file_fd : *fds()) { if (!file_fd) { continue; } bool found = false; for (inherited_file_t& inherited_file : inherited_files) { if (get_fd(inherited_file.fds[0])->fdcmp(*file_fd) == 0) { inherited_file.fds.push_back(file_fd->fd()); found = true; break; } } if (!found) { inherited_file_t inherited_file; inherited_file.type = file_fd->type(); assert(inherited_file.type != FD_UNINITIALIZED); inherited_file.fds.push_back(file_fd->fd()); inherited_file.filename = file_fd->filename(); inherited_file.flags = file_fd->flags(); inherited_files.push_back(inherited_file); } } for (inherited_file_t& inherited_file : inherited_files) { if (inherited_file.type == FD_FILE) { /* Remember the file size and seek offset. */ FileFD *file_fd = (*fds())[inherited_file.fds[0]].get(); struct stat64 st; if (stat64(file_fd->filename()->c_str(), &st) < 0) { disable_shortcutting_only_this("Failed to stat inherited file"); } else if (S_ISREG(st.st_mode) && is_write(file_fd->flags())) { /* Assume that the file is seeked to the end. Otherwise the interceptor will * report the offset back. */ inherited_file.start_offset = st.st_size; } } } set_inherited_files(inherited_files); if (FB_DEBUGGING(FB_DEBUG_PROC)) { FB_DEBUG(FB_DEBUG_PROC, "Client-side fds are:"); for (const inherited_file_t& inherited_file : inherited_files) { std::string arr = " type=" + std::string(fd_type_to_string(inherited_file.type)) + " ["; bool add_sep = false; for (int fd : inherited_file.fds) { if (add_sep) { arr += ", "; } add_sep = true; arr += std::to_string(fd); } arr += "]"; FB_DEBUG(FB_DEBUG_PROC, arr); } } } void ExecedProcess::set_on_finalized_ack(int id, int fd) { fork_point()->set_on_finalized_ack(id, fd); } bool ExecedProcess::been_waited_for() const { return fork_point()->been_waited_for(); } void ExecedProcess::set_been_waited_for() { fork_point()->set_been_waited_for(); } void ExecedProcess::resource_usage(const int64_t utime_u, const int64_t stime_u) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "utime_u=%" PRId64 ", stime_u=%" PRId64, utime_u, stime_u); /* store resource usage for this process */ Process::resource_usage(utime_u, stime_u); } void ExecedProcess::do_finalize() { ProcessDebugSuppressor debug_suppressor(this); TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); close_fds(); /* store data for shortcutting */ if (!was_shortcut() && can_shortcut() && fork_point()->exit_status() != -1 && aggr_cpu_time_u() >= min_cpu_time_u) { execed_process_cacher->store(this); } /* Propagate resource usage. */ if (parent_exec_point()) { parent_exec_point()->add_children_cpu_time_u(aggr_cpu_time_u()); parent_exec_point()->add_shortcut_cpu_time_ms(shortcut_cpu_time_ms()); } /* Call the base class's method */ Process::do_finalize(); for (const auto& pipe : created_pipes_) { pipe->finish(); } proc_tree->QueueExecProcForGC(this); } bool ExecedProcess::register_file_usage_update(const FileName *name, const FileUsageUpdate& update) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "name=%s, update=%s", D(name), D(update)); /* What the FileUsage was before the update, on the previous (descendant) level. The initial value * should differ from any valid value, including NULL which is valid too (indicating no prior * knowledge about that file), to allow to detect if we're about the perform the same update * operation as we did on the previous (descendant) level. It should also differ from fu_new's * initial value, to allow to detect if the update didn't change anything. */ const FileUsage *fu_old = reinterpret_cast(-1); /* What the FileUsage became after the update, on the previous (descendant) level. */ const FileUsage *fu_new = nullptr; if (name->is_in_ignore_location()) { FB_DEBUG(FB_DEBUG_FS, "Ignoring file usage: " + d(name)); return true; } bool propagated = false; ExecedProcess *proc = this; if (!proc->can_shortcut_ && !generate_report) { /* Register at the first shortcutable ancestor instead. */ proc = proc->next_shortcutable_ancestor(); propagated = true; } if (!proc) { return true; } /* Register (and bubble up) what we implicitly got to know about the parent directory. This call * will in turn call back to us, but it's not a recursion since 'update' will then have * parent_type_ == DONTKNOW. */ if (update.parent_type() != DONTKNOW) { proc->register_parent_directory(name, update.parent_type()); } /* Bubble up to the root, but abort the loop if at a certain level we didn't do anything (fu_new * became the same as fu_old was) because then continuing to bubble up wouldn't do anything new * either. */ while (proc && fu_new != fu_old) { ProcessDebugSuppressor debug_suppressor(proc == this ? nullptr : proc); const FileUsage *fu = nullptr; auto it = proc->file_usages_.find(name); if (it != proc->file_usages_.end()) { fu = it->second; } if (fu == fu_old) { /* Quick path: We need to do the same as we did on the previous level (fu and fu_old can be * NULL or non-NULL, the same quick path logic applies). The new value is fu_new (which can be * the same as fu_old or can differ, again, the same quick path logic applies for both cases), * just as it was on the previous level. Nothing to do here. */ } else { /* Need to merge "update" into "fu". */ fu_old = fu; if (!fu) { fu = FileUsage::Get(DONTKNOW); } else { if (fu->generation() != update.generation() && fu->generation() + 1 != update.generation()) { /* If all file changes were performed by descendants then the generation updates should * always be incremented by one. Otherwise the file could have been changed outside of the * process's subtree wich makes the process not shortcutable. */ proc->disable_shortcutting_only_this( generate_report ? deduplicated_string("A parallel process modified " + d(name)).c_str() : "A parallel process modified the file"); /* Still bubble up to the root because an ancestor may still be shortcutable and also * been updated with the parallel change. */ propagated = true; proc = proc->next_shortcutable_ancestor(); if (proc) { /* There is no merged file usage, it can't be carried in this loop. Recurse instead. */ return proc->register_file_usage_update(name, update); } else { /* No process to bubble up to, file usage registration in finshed. */ return true; } } } /* Note: This can update "update" if some value is computed now and cached there. In that * case, in the next iteration of the loop "update" will already contain this value. */ fu_new = fu->merge(update, propagated); if (!fu_new) { if (FB_DEBUGGING(FB_DEBUG_FS) || generate_report) { std::string reason("Could not merge " + d(name) + " file usage " + d(fu) + " with " + d(update)); FB_DEBUG(FB_DEBUG_FS, reason); disable_shortcutting_bubble_up( "Could not register unsupported file usage combination:", reason); } else { disable_shortcutting_bubble_up("Could not register unsupported file usage combination"); } return false; } } if (it != proc->file_usages_.end()) { /* Update the previous value. */ it.value() = fu_new; } else { /* Insert a new value. */ proc->file_usages_[name] = fu_new; } propagated = true; proc = proc->next_shortcutable_ancestor(); } return true; } bool ExecedProcess::register_parent_directory(const FileName *name, FileType type) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "name=%s", D(name)); const FileName* const parent_dir = FileName::GetParentDir(name->c_str(), name->length()); if (!parent_dir) { return false; } else if (parent_dir->length() == 1 && parent_dir->c_str()[0] == '/') { /* don't bother registering "/" */ return true; } FileUsageUpdate update(parent_dir, type); assert(update.parent_type() == DONTKNOW); return register_file_usage_update(parent_dir, update); } bool ExecedProcess::shortcut(std::vector *fds_appended_to) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (can_shortcut()) { return execed_process_cacher->shortcut(this, fds_appended_to); } else { FB_DEBUG(FB_DEBUG_SHORTCUT, "┌─"); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Shortcutting disabled:"); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ exe = " + d(executable())); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ arg = " + d(args())); /* FB_DEBUG(FB_DEBUG_SHORTCUT, "│ env = " + d(env_vars())); */ FB_DEBUG(FB_DEBUG_SHORTCUT, "└─"); return false; } } const char* ExecedProcess::reason_with_fd(const char* reason, const int fd) const { return deduplicated_string(std::string(reason) + " fd: " + d(fd) + ((get_fd(fd) && get_fd(fd)->filename()) ? (std::string(", name: ") + get_fd(fd)->filename()->c_str()) : "")).c_str(); } void ExecedProcess::disable_shortcutting_bubble_up_to_excl( ExecedProcess *stop, const char* reason, const ExecedProcess *p, ExecedProcess *shortcutable_ancestor, bool shortcutable_ancestor_is_set) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "stop=%s, reason=%s, source=%s", D(stop), D(reason), D(p)); if (this == stop) { return; } if (p == NULL) { p = this; } disable_shortcutting_only_this(reason, p); if (next_shortcutable_ancestor() == nullptr) { /* Shortcutting is already disabled for all transitive exec parents. */ return; } if (!shortcutable_ancestor_is_set) { /* Move maybe_shortcutable_ancestor_ only upwards. */ shortcutable_ancestor = stop; while (shortcutable_ancestor != nullptr && !shortcutable_ancestor->can_shortcut()) { shortcutable_ancestor = shortcutable_ancestor->maybe_shortcutable_ancestor_; } shortcutable_ancestor_is_set = true; } maybe_shortcutable_ancestor_ = shortcutable_ancestor; if (parent_exec_point()) { parent_exec_point()->disable_shortcutting_bubble_up_to_excl( stop, reason, p, shortcutable_ancestor, shortcutable_ancestor_is_set); } } void ExecedProcess::disable_shortcutting_bubble_up_to_excl( ExecedProcess *stop, const char* reason, const int fd, const ExecedProcess *p, ExecedProcess *shortcutable_ancestor, bool shortcutable_ancestor_is_set) { disable_shortcutting_bubble_up_to_excl( stop, generate_report ? reason_with_fd(reason, fd) : reason, p, shortcutable_ancestor, shortcutable_ancestor_is_set); FB_DEBUG(FB_DEBUG_PROC, "fd: " + d(fd)); } void ExecedProcess::disable_shortcutting_bubble_up(const char* reason, const ExecedProcess *p) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "reason=%s, source=%s", D(reason), D(p)); disable_shortcutting_bubble_up_to_excl(NULL, reason, p); } void ExecedProcess::disable_shortcutting_bubble_up(const char* reason, const int fd, const ExecedProcess *p) { disable_shortcutting_bubble_up( generate_report ? reason_with_fd(reason, fd) : reason, p); FB_DEBUG(FB_DEBUG_PROC, "fd: " + d(fd)); } void ExecedProcess::disable_shortcutting_bubble_up(const char* reason, const FileName& file, const ExecedProcess *p) { disable_shortcutting_bubble_up( generate_report ? deduplicated_string(std::string(reason) + " file: " + d(file)).c_str() : reason, p); FB_DEBUG(FB_DEBUG_PROC, "file: " + d(file)); } void ExecedProcess::disable_shortcutting_bubble_up(const char* reason, const std::string& str, const ExecedProcess *p) { disable_shortcutting_bubble_up( generate_report ? deduplicated_string(std::string(reason) + " " + d(str)).c_str() : reason, p); FB_DEBUG(FB_DEBUG_PROC, d(str)); } void ExecedProcess::disable_shortcutting_only_this(const char* reason, const ExecedProcess *p) { ProcessDebugSuppressor debug_suppressor(this); TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "reason=%s, source=%s", D(reason), D(p)); if (can_shortcut_) { can_shortcut_ = false; assert(cant_shortcut_reason_ == nullptr); cant_shortcut_reason_ = reason; assert_null(cant_shortcut_proc_); cant_shortcut_proc_ = p ? p : this; FB_DEBUG(FB_DEBUG_PROC, "Command " + d(executable_->c_str()) + " can't be short-cut due to: " + reason + ", " + d(this)); for (const inherited_file_t& inherited_file : inherited_files_) { if (inherited_file.recorder) { assert(inherited_file.type == FD_PIPE_OUT); inherited_file.recorder->deactivate(); } } } } std::string ExecedProcess::args_to_short_string() const { const int max_len = 65; if (args().size() == 0) { return ""; } size_t slash_pos = args()[0].rfind('/'); std::string str; if (slash_pos == std::string::npos) { str = args()[0]; } else { str = args()[0].substr(slash_pos + 1); } for (size_t i = 1; i < args().size(); i++) { str += " "; str += args()[i]; } if (str.length() <= max_len) { return str; } else { const int one_run = max_len / 2 - 5; return str.substr(0, one_run) + "[...]" + str.substr(str.length() - one_run); } } std::string ExecedProcess::d_internal(const int level) const { if (level > 0) { /* brief */ return Process::d_internal(level); } else { /* verbose */ return "{ExecedProcess " + pid_and_exec_count() + ", " + state_string() + ", " + d(args_to_short_string()) + ", fds=" + d(fds(), level + 1) + "}"; } } ExecedProcess::~ExecedProcess() { TRACKX(FB_DEBUG_PROC, 1, 0, Process, this, ""); if (original_executed_path_ != executed_path_->c_str()) { free(original_executed_path_); } execed_process_cacher->erase_fingerprint(this); } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/execed_process.h000066400000000000000000000400741447164520700214050ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_EXECED_PROCESS_H_ #define FIREBUILD_EXECED_PROCESS_H_ #include #include #include #include #include #include #include #include "firebuild/file_info.h" #include "firebuild/file_name.h" #include "firebuild/file_usage.h" #include "firebuild/file_usage_update.h" #include "firebuild/pipe.h" #include "firebuild/pipe_recorder.h" #include "firebuild/process.h" #include "firebuild/cxx_lang_utils.h" #include "firebuild/debug.h" namespace firebuild { class ExecedProcessCacher; /** * Represents one open file description that this process inherited, along with the list of * corresponding file descriptors. (An open file description might have multiple file descriptors, * as per dup() and friends. They are stored in ascending order. There's at least one fd.) * * The structure always refers to how things were when the process started, * it isn't modified later as the process does various things with its file descriptors. * * Accordingly, for pipes, it does not hold a pointer to the Pipe object, since that one might go * away while we still need to keep this structure. */ typedef struct inherited_file_ { /* Type. */ fd_type type {FD_UNINITIALIZED}; /* The client-side file descriptor numbers, sorted */ std::vector fds {}; /* For FD_PIPE_OUT only: The recorder of the traffic, as seen from this exec point */ std::shared_ptr recorder {}; /* For type FD_FILE. */ const FileName *filename {}; /* Flags. We need to know for regular files if they're opened for writing. */ int flags {-1}; /* For writable FD_FILE: If the O_APPEND flag is not set then it's the seek offset when the * process started up. If O_APPEND is set then it's the file size as of that time. That is, in * either case, assuming no other process writes to the file, and assuming that this process does * sequential writes only, this is the offset from where this process writes its data. */ ssize_t start_offset {-1}; } inherited_file_t; class ExecedProcess : public Process { public: explicit ExecedProcess(const int pid, const int ppid, const FileName *initial_wd, const FileName *executable, const FileName *executed_path, char* original_executed_path, const std::vector& args, const std::vector& env_vars, const std::vector& libs, const mode_t umask, Process * parent, const bool debug_suppressed, std::vector>* fds); virtual ~ExecedProcess(); virtual bool exec_started() const {return true;} ExecedProcess* exec_point() {return this;} const ExecedProcess* exec_point() const {return this;} ExecedProcess* common_exec_ancestor(ExecedProcess* other); ForkedProcess* fork_point() {return fork_point_;} const ForkedProcess* fork_point() const {return fork_point_;} void set_parent(Process *parent); /** * Set jobserver fds if they are reasonably small and fit in the member's type. */ void maybe_set_jobserver_fds(int fd_r, int fd_w) { if (fd_w >= 0 && fd_r <= INT16_MAX) { jobserver_fd_r_ = fd_r; } if (fd_w >= 0 && fd_w <= INT16_MAX) { jobserver_fd_w_ = fd_w; } } int jobserver_fd_r() const {return jobserver_fd_r_;} int jobserver_fd_w() const {return jobserver_fd_w_;} bool been_waited_for() const; void set_been_waited_for(); void add_utime_u(int64_t t) {utime_u_ += t;} int64_t utime_u() const {return utime_u_;} void add_stime_u(int64_t t) {stime_u_ += t;} int64_t stime_u() const {return stime_u_;} int64_t cpu_time_u() const {return utime_u_ + stime_u_;} void add_children_cpu_time_u(const int64_t t) {children_cpu_time_u_ += t;} void add_shortcut_cpu_time_ms(const int64_t t) {shortcut_cpu_time_ms_ += t;} int64_t shortcut_cpu_time_ms() const {return shortcut_cpu_time_ms_;} int64_t aggr_cpu_time_u() const {return cpu_time_u() + children_cpu_time_u_;} const FileName* initial_wd() const {return initial_wd_;} const tsl::hopscotch_set& wds() const {return wds_;} const tsl::hopscotch_set& wds() {return wds_;} const tsl::hopscotch_set& failed_wds() const {return wds_;} tsl::hopscotch_set& failed_wds() {return failed_wds_;} const std::vector& args() const {return args_;} std::vector& args() {return args_;} const std::vector& env_vars() const {return env_vars_;} std::vector& env_vars() {return env_vars_;} const FileName* executable() const {return executable_;} const FileName* executed_path() const {return executed_path_;} const char* original_executed_path() const {return original_executed_path_;} std::vector& libs() {return libs_;} const std::vector& libs() const {return libs_;} tsl::hopscotch_map& file_usages() {return file_usages_;} const tsl::hopscotch_map& file_usages() const { return file_usages_; } void do_finalize(); void set_on_finalized_ack(int id, int fd); Process* exec_proc() const {return const_cast(this);} void resource_usage(const int64_t utime_u, const int64_t stime_u); /** * Initialization stuff that can only be done after placing the * ExecedProcess in the ProcessTree. */ void initialize(); /** * Registers a file operation described in "update" into the filename "name", and bubbles it up to * the root. * * "update" might contain some lazy bits that will be computed on demand. * * In some rare cases the filename within "update" might differ from "name", in that case the * filename mentioned in "update" is used to lazily figure out the required values (such as * checksum), but it is registered as if it belonged to the file mentioned in this method's "name" * parameter. Currently this trick is only used for a rename()'s source path. * * This method also registers the implicit parent directory and bubbles it up, as per the * information contained in "update". */ bool register_file_usage_update(const FileName *name, const FileUsageUpdate& update) __attribute__((nonnull(2))); /** * Register that the parent (a.k.a. dirname) of the given path does (or does not) exist and is of * the given "type" (e.g. ISDIR, NOTEXIST), and bubbles it up to the root. */ bool register_parent_directory(const FileName *name, FileType type = ISDIR); void add_pipe(std::shared_ptr pipe) {created_pipes_.insert(pipe);} std::vector& inherited_files() {return inherited_files_;} const std::vector& inherited_files() const {return inherited_files_;} void set_inherited_files(std::vector inherited_files) {inherited_files_ = inherited_files;} /** * Fail to change to a working directory */ void handle_fail_wd(const char * const d) { failed_wds_.insert(FileName::Get(d)); } /** * Record visited working directory */ void add_wd(const FileName *d) { wds_.insert(d); } /** Returns if the process can be short-cut */ bool can_shortcut() const {return can_shortcut_;} /** Reason for this process can't be short-cut */ const char* cant_shortcut_reason() const {return cant_shortcut_reason_;} /** Process the event preventing short-cutting happened in */ const Process* cant_shortcut_proc() const {return cant_shortcut_proc_;} /** Result of the shortcut attempt if the process can be shotcut. */ void set_shortcut_result(const char* result) {shortcut_result_ = result;} /** Result of the shortcut attempt if the process can be shotcut. */ const char* shortcut_result() const {return shortcut_result_;} /** Find and apply shortcut */ bool shortcut(std::vector *fds_appended_to); /** * This particular process can't be short-cut because it performed calls preventing that. * @param reason reason for can't being short-cut * @param p process the event preventing shortcutting happened in, or * omitted for the current process */ virtual void disable_shortcutting_only_this(const char* reason, const ExecedProcess *p = NULL); /** * Process and parents (transitively) up to (excluding) "stop" can't be short-cut because * it performed calls preventing that. * @param stop Stop before this process * @param reason reason for can't being short-cut * @param p process the event preventing shortcutting happened in, or * omitted for the current process * @param shortcutable_ancestor this ancestor will be the nearest shortcutable ancestor * for all visited execed processes after this call * (when shortcutable_ancestor_is_set is true) * @param shortcutable_ancestor_is_set the shortcutable_ancestor is computed */ void disable_shortcutting_bubble_up_to_excl(ExecedProcess *stop, const char* reason, const ExecedProcess *p = NULL, ExecedProcess *shortcutable_ancestor = nullptr, bool shortcutable_ancestor_is_set = false); void disable_shortcutting_bubble_up_to_excl(ExecedProcess *stop, const char* reason, int fd, const ExecedProcess *p = NULL, ExecedProcess *shortcutable_ancestor = nullptr, bool shortcutable_ancestor_is_set = false); /** * Process and parents (transitively) can't be short-cut because it performed * calls preventing that. * @param reason reason for can't being short-cut * @param p process the event preventing shortcutting happened in, or * omitted for the current process */ void disable_shortcutting_bubble_up(const char* reason, const ExecedProcess *p = NULL); void disable_shortcutting_bubble_up(const char* reason, const int fd, const ExecedProcess *p = NULL); void disable_shortcutting_bubble_up(const char* reason, const FileName& file, const ExecedProcess *p = NULL); void disable_shortcutting_bubble_up(const char* reason, const std::string& str, const ExecedProcess *p = NULL); bool was_shortcut() const {return was_shortcut_;} void set_was_shortcut(bool value) {was_shortcut_ = value;} /** For debugging, a short imprecise reminder of the command line. Omits the path to the * executable, and strips off the middle. Does not escape or quote. */ std::string args_to_short_string() const; /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ virtual std::string d_internal(const int level = 0) const; private: bool can_shortcut_:1; bool was_shortcut_:1; int16_t jobserver_fd_r_ = -1; int16_t jobserver_fd_w_ = -1; /** If points to this (self), the process can be shortcut. Otherwise the process itself is not shortcutable, but the ancestor is, if the ancestor's maybe_shortcutable_ancestor points at itself, etc. */ ForkedProcess *fork_point_ {}; ExecedProcess * maybe_shortcutable_ancestor_; /** Sum of user time in microseconds for all forked but not exec()-ed children */ int64_t utime_u_ = 0; /** Sum of system time in microseconds for all forked but not exec()-ed children */ int64_t stime_u_ = 0; /** * Sum of user and system time in microseconds for all finalized exec()-ed children. * Shortcut processes are treated as if their CPU time was 0. */ int64_t children_cpu_time_u_ = 0; /** * Aggregate CPU time saved by shortcutting in all transitive children and this process. * It becomes final when all exec()-ed children are finalized. */ int64_t shortcut_cpu_time_ms_ {0}; /** Directory the process exec()-started in */ const FileName* initial_wd_; /** Working directories visited by the process and all fork()-children */ tsl::hopscotch_set wds_; /** Working directories the process and all fork()-children failed to chdir() to */ tsl::hopscotch_set failed_wds_; std::vector args_; /** Environment variables in deterministic (sorted) order. */ std::vector env_vars_; /** * The executable running. In case of scripts this is the interpreter or in case of invoking * an executable via a symlink this is the executable the symlink points to. */ const FileName* executable_; /** * The path executed converted to absolute and canonical form. * In case of scripts this is the script's name or in case of invoking executable via a symlink * this is the name of the symlink. */ const FileName* executed_path_; /** * The path executed. In case of scripts this is the script's name or in case of invoking * executable via a symlink this is the name of the symlink. * May be not absolute nor canonical, like ./foo. * It may point to executed_path_.c_str() and in that case it should not be freed. */ char* original_executed_path_; /** * DSO-s loaded by the linker at process startup, in the same order. * (DSO-s later loaded via dlopen(), and DSO-s of descendant processes are registered as regular * file open operations.) */ std::vector libs_; /** File usage per path for p and f. c. (t.) */ tsl::hopscotch_map file_usages_; /** * Pipes created by this process. */ tsl::hopscotch_set> created_pipes_ = {}; /** * The files this process had at startup, grouped by "open file description". * Each such "open file description" might have multiple client-side file descriptors (see dup() * and friends), they are in sorted order. Also, this inherited_files_ array is sorted according * to the first (lowest) fd for each inherited file. */ std::vector inherited_files_ = {}; void store_in_cache(); ExecedProcess* next_shortcutable_ancestor() { if (maybe_shortcutable_ancestor_ == nullptr || maybe_shortcutable_ancestor_->can_shortcut_) { return maybe_shortcutable_ancestor_; } else { ExecedProcess* next = maybe_shortcutable_ancestor_->maybe_shortcutable_ancestor_; while (next != nullptr && !next->can_shortcut_) { next = next->maybe_shortcutable_ancestor_; } maybe_shortcutable_ancestor_ = next; return next; } } ExecedProcess* closest_shortcut_point() { return can_shortcut() ? this : next_shortcutable_ancestor(); } const char* reason_with_fd(const char* reason, const int fd) const; /** Reason for this process can't be short-cut */ const char* cant_shortcut_reason_ = nullptr; /** Reason for this process can't be short-cut */ const char* shortcut_result_ = nullptr; /** Process the event preventing short-cutting happened in */ const Process *cant_shortcut_proc_ = NULL; DISALLOW_COPY_AND_ASSIGN(ExecedProcess); }; } /* namespace firebuild */ #endif // FIREBUILD_EXECED_PROCESS_H_ firebuild-0.8.2/src/firebuild/execed_process_cacher.cc000066400000000000000000002216331447164520700230520ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/execed_process_cacher.h" #include #include #include #include #include #include #include #include #include #include #include "firebuild/config.h" #include "firebuild/debug.h" #include "firebuild/execed_process.h" #include "firebuild/forked_process.h" #include "firebuild/file_name.h" #include "firebuild/hash_cache.h" #include "firebuild/fbbfp.h" #include "firebuild/fbbstore.h" #include "firebuild/process_tree.h" extern bool generate_report; namespace firebuild { static const XXH64_hash_t kFingerprintVersion = 0; static const unsigned int kCacheFormatVersion = 1; static const char kCacheStatsFile[] = "stats"; static const char kCacheSizeFile[] = "size"; unsigned int ExecedProcessCacher::cache_format_ = 0; /* singleton*/ ExecedProcessCacher* execed_process_cacher; void ExecedProcessCacher::init(const libconfig::Config* cfg) { std::string cache_dir; char* cache_dir_env; if ((cache_dir_env = getenv("FIREBUILD_CACHE_DIR")) && cache_dir_env[0] != '\0') { cache_dir = std::string(cache_dir_env); } else if ((cache_dir_env = getenv("XDG_CACHE_HOME")) && cache_dir_env[0] != '\0') { cache_dir = std::string(cache_dir_env) + "/firebuild"; } else if ((cache_dir_env = getenv("HOME")) && cache_dir_env[0] != '\0') { cache_dir = std::string(cache_dir_env) + "/.cache/firebuild"; } else { fb_error("Please set HOME or XDG_CACHE_HOME or FIREBUILD_CACHE_DIR to let " "firebuild place the cache somewhere."); exit(EXIT_FAILURE); } /* Like CCACHE_RECACHE: Don't fetch entries from the cache, but still * potentially store new ones. Note however that it might decrease the * objcache hit ratio: new entries might be stored that eventually * result in the same operation, but go through a slightly different * path (e.g. different tmp file name), and thus look different in * Firebuild's eyes. Firebuild refuses to shortcut a process if two or * more matches are found in the objcache. */ bool no_fetch {getenv("FIREBUILD_RECACHE") != NULL}; /* Like CCACHE_READONLY: Don't store new results in the cache. */ bool no_store = getenv("FIREBUILD_READONLY") != NULL; struct stat st; if (stat(cache_dir.c_str(), &st) == 0) { if (!S_ISDIR(st.st_mode)) { fb_error("cache dir exists but is not a directory"); exit(EXIT_FAILURE); } } else { if (mkdirhier(cache_dir.c_str(), 0700) != 0) { fb_perror("mkdir"); exit(EXIT_FAILURE); } } char* cache_format_file = strdup((cache_dir + "/cache-format").c_str()); if (stat(cache_format_file, &st) == 0) { if (!S_ISREG(st.st_mode)) { fb_error("$FIREBUILD_CACHE_DIR/cache-format exists but is not a regular file"); exit(EXIT_FAILURE); } FILE* f; if (!(f = fopen(cache_format_file, "r"))) { fb_perror("opening cache-format file failed"); exit(EXIT_FAILURE); } else { if (fscanf(f, "%u\n", &cache_format_) != 1 || cache_format() > kCacheFormatVersion) { fb_error("Cache format version is not supported, not reading or writing the cache"); no_fetch = true; no_store = true; } else if (cache_format() == kCacheFormatVersion) { /* Current format, we can use the cache. */ } else { /* Cache is in a prior format. Either use it considering the differences where needed * or upgrade it. */ } fclose(f); } } else { FILE* f; if (!(f = fopen(cache_format_file, "wx"))) { fb_perror("creating cache-format file failed"); exit(EXIT_FAILURE); } if (fprintf(f, "%d\n", kCacheFormatVersion) <= 0) { fb_perror("writing cache-format file failed"); exit(EXIT_FAILURE); } fclose(f); } free(cache_format_file); blob_cache = new BlobCache(cache_dir + "/blobs"); obj_cache = new ObjCache(cache_dir + "/objs"); PipeRecorder::set_base_dir((cache_dir + "/tmp").c_str()); hash_cache = new HashCache(); execed_process_cacher = new ExecedProcessCacher(no_store, no_fetch, cache_dir, cfg); } /** * Add file_name to fingerprint, including the ending '\0'. * * Adding the ending '\0' prevents hash collisions by concatenating filenames. */ static void add_to_hash_state(XXH3_state_t* state, const FileName* file_name) { if (XXH3_128bits_update(state, file_name->c_str(), file_name->length() + 1) == XXH_ERROR) { abort(); } } /** * Add string to fingerprint, including the ending '\0'. * * Adding the ending '\0' prevents hash collisions by concatenating strings. */ static void add_to_hash_state(XXH3_state_t* state, const std::string& str) { if (XXH3_128bits_update(state, str.c_str(), str.length() + 1) == XXH_ERROR) { abort(); } } /** * Add string to fingerprint, including the ending '\0'. * * Adding the ending '\0' prevents hash collisions by concatenating strings. */ static void add_to_hash_state(XXH3_state_t* state, const char* str, size_t length) { if (XXH3_128bits_update(state, str, length + 1) == XXH_ERROR) { abort(); } } /** * Add hash to fingerprint */ static void add_to_hash_state(XXH3_state_t* state, const Hash& hash) { if (XXH3_128bits_update(state, hash.get_ptr(), Hash::hash_size()) == XXH_ERROR) { abort(); } } /** * Add int to fingerprint */ static void add_to_hash_state(XXH3_state_t* state, const int i) { if (XXH3_128bits_update(state, &i, sizeof(int)) == XXH_ERROR) { abort(); } } static Hash state_to_hash(XXH3_state_t* state) { const XXH128_hash_t digest = XXH3_128bits_digest(state); return Hash(digest); } /* Free XXH3_state_t when it is malloc()-ed. */ static inline void maybe_XXH3_freeState(XXH3_state_t* state) { #ifndef XXH_INLINE_ALL XXH3_freeState(state); #else (void)state; #endif } ExecedProcessCacher::ExecedProcessCacher(bool no_store, bool no_fetch, const std::string& cache_dir, const libconfig::Config* cfg) : no_store_(no_store), no_fetch_(no_fetch), envs_skip_(), ignore_locations_hash_(), fingerprints_(), fingerprint_msgs_(), cache_dir_(cache_dir) { try { const libconfig::Setting& envs_skip = cfg->getRoot()["env_vars"]["fingerprint_skip"]; for (int i = 0; i < envs_skip.getLength(); i++) { envs_skip_.insert(envs_skip[i].c_str()); } } catch(libconfig::SettingNotFoundException&) { /* Configuration setting may be missing. This is OK. */ } #ifdef XXH_INLINE_ALL XXH3_state_t state_struct; XXH3_state_t* state = &state_struct; #else XXH3_state_t* state = XXH3_createState(); #endif if (XXH3_128bits_reset(state) == XXH_ERROR) { abort(); } /* Hash the already sorted ignore locations.*/ for (int i = 0; i < ignore_locations.len; i++) { add_to_hash_state(state, ignore_locations.p[i].c_str, ignore_locations.p[i].length); } ignore_locations_hash_ = state_to_hash(state); maybe_XXH3_freeState(state); } bool ExecedProcessCacher::env_fingerprintable(const std::string& name_and_value) const { /* Strip off the "=value" part. */ const std::string name = name_and_value.substr(0, name_and_value.find('=')); /* Env vars to skip, taken from the config files. * Note: FB_SOCKET is already filtered out in the interceptor. */ return envs_skip_.find(name) == envs_skip_.end(); } /* Adaptor from C++ std::vector to FBB's FBB array */ static const FBBFP_Builder *fbbfp_builder_file_vector_item_fn(int i, const void *user_data) { auto fbbs = reinterpret_cast *>(user_data); const FBBFP_Builder_file *builder = &(*fbbs)[i]; return reinterpret_cast(builder); } /* Adaptor from C++ std::vector to FBB's FBB array */ static const FBBFP_Builder *fbbfp_builder_ofd_vector_item_fn(int i, const void *user_data) { auto fbbs = reinterpret_cast *>(user_data); const FBBFP_Builder_ofd *builder = &(*fbbs)[i]; return reinterpret_cast(builder); } /* Adaptor from C++ std::vector to FBB's FBB array */ static const FBBSTORE_Builder *fbbstore_builder_append_to_fd_vector_item_fn(int i, const void *user_data) { auto fbbs = reinterpret_cast *>(user_data); const FBBSTORE_Builder_append_to_fd *builder = &(*fbbs)[i]; return reinterpret_cast(builder); } static void hash_param_file(XXH3_state_t* state, const ExecedProcess *proc, const std::string& file, Hash* hash) { bool is_dir; if (hash_cache->get_hash(proc->get_absolute(AT_FDCWD, file.c_str(), file.length()), 0, hash, &is_dir)) { if (is_dir) { /* Directory params are not hashed. */ *hash = Hash(); } } else { /* File may be an output file or not a file at all */ *hash = Hash(); } add_to_hash_state(state, *hash); } /* * Note: Don't forget updating the debugging part below, too, when changing the * fingerprint generation! */ bool ExecedProcessCacher::fingerprint(const ExecedProcess *proc) { TRACK(FB_DEBUG_PROC, "proc=%s", D(proc)); #ifdef XXH_INLINE_ALL XXH3_state_t state_struct; XXH3_state_t* state = &state_struct; #else XXH3_state_t* state = XXH3_createState(); #endif if (XXH3_128bits_reset_withSeed(state, kFingerprintVersion) == XXH_ERROR) { abort(); } add_to_hash_state(state, ignore_locations_hash_); add_to_hash_state(state, proc->initial_wd()); /* Size is added to not allow collisions between elements of different containers. * Otherwise "cmd foo BAR=1" would collide with "env BAR=1 cmd foo". */ add_to_hash_state(state, proc->args().size()); const std::vector& args = proc->args(); const bool guess_file_params = quirks & FB_QUIRK_GUESS_FILE_PARAMS; std::string found_param_file; Hash found_param_file_hash; for (const auto& arg : args) { add_to_hash_state(state, arg); /* Since we are already iterating over the args let's find a hint for hash_param_files(). */ if (guess_file_params && (arg == "conftest.c" || arg == "objs/autotest.c")) { found_param_file = arg; } } /* Heuristics for including some parameter files in the fingerprint. * For now only a single, already found parameter is covered. * Note: Update kFingerprintVersion whenever this heuristic changes. */ if (guess_file_params && found_param_file.size() > 0) { /* Number of files to be added. */ add_to_hash_state(state, 1); hash_param_file(state, proc, found_param_file, &found_param_file_hash); } else { add_to_hash_state(state, 0); } /* Already sorted by the interceptor */ add_to_hash_state(state, proc->env_vars().size()); for (const auto& env : proc->env_vars()) { if (env_fingerprintable(env)) { add_to_hash_state(state, env); } } /* The executable and its hash */ add_to_hash_state(state, proc->executable()); Hash hash; if (!hash_cache->get_hash(proc->executable(), 0, &hash)) { FB_DEBUG(FB_DEBUG_PROC, "Could not get hash of executable: " + d(proc->executable())); maybe_XXH3_freeState(state); return false; } add_to_hash_state(state, hash); if (proc->executable() == proc->executed_path()) { /* Those often match. Don't calculate the same hash twice then. */ add_to_hash_state(state, proc->executable()); add_to_hash_state(state, hash); } else { add_to_hash_state(state, proc->executed_path()); if (!hash_cache->get_hash(proc->executed_path(), 0, &hash)) { FB_DEBUG(FB_DEBUG_PROC, "Could not get hash of executed path: " + d(proc->executed_path())); maybe_XXH3_freeState(state); return false; } add_to_hash_state(state, hash); } add_to_hash_state(state, proc->original_executed_path()); add_to_hash_state(state, proc->libs().size()); for (const auto lib : proc->libs()) { #ifdef __APPLE__ /* SDK libraries are not present as files, see: * https://developer.apple.com/forums/thread/655588 */ if (strncmp(lib->c_str(), "/usr/lib/", strlen("/usr/lib/")) == 0) { continue; } #endif if (!hash_cache->get_hash(lib, 0, &hash)) { FB_DEBUG(FB_DEBUG_PROC, "Could not get hash of library: " + d(lib)); maybe_XXH3_freeState(state); return false; } add_to_hash_state(state, lib); add_to_hash_state(state, hash); } /* umask */ add_to_hash_state(state, proc->umask()); /* The inherited files */ for (const inherited_file_t& inherited_file : proc->inherited_files()) { /* Workaround for #938. */ fd_type pretended_type = inherited_file.type == FD_PIPE_IN ? FD_IGNORED : inherited_file.type; add_to_hash_state(state, pretended_type); for (int fd : inherited_file.fds) { add_to_hash_state(state, fd); } /* Append an invalid value to each inherited file to avoid collisions. */ add_to_hash_state(state, -1); } fingerprints_[proc] = state_to_hash(state); if (FB_DEBUGGING(FB_DEBUG_CACHE)) { /* Only when debugging: add an entry to fingerprint_msgs_. * The entry is the serialized message so that we don't have to fiddle with * memory allocation/freeing for all the substrings. */ FBBFP_Builder_process_fingerprint fp; fp.set_kFingerprintVersion(kFingerprintVersion); std::vector ignore_locations_vec; for (int i = 0; i < ignore_locations.len; i++) { ignore_locations_vec.push_back(ignore_locations.p[i].c_str); } fp.set_ignore_locations(ignore_locations_vec); fp.set_wd(proc->initial_wd()->c_str()); fp.set_args(proc->args()); if (guess_file_params && found_param_file.size() > 0) { fp.set_param_file_hash(found_param_file_hash.get()); } /* Env vars are already sorted by the interceptor, but we need to do some filtering */ std::vector c_env; c_env.reserve(proc->env_vars().size()); /* likely minor optimization */ for (const auto& env : proc->env_vars()) { if (env_fingerprintable(env)) { c_env.push_back(env.c_str()); } } fp.set_env_with_count(c_env.data(), c_env.size()); /* The executable and its hash */ FBBFP_Builder_file executable; if (!hash_cache->get_hash(proc->executable(), 0, &hash)) { FB_DEBUG(FB_DEBUG_PROC, "Could not get hash of executable: " + d(proc->executable())); maybe_XXH3_freeState(state); return false; } executable.set_path(proc->executable()->c_str()); executable.set_hash(hash.get()); fp.set_executable(reinterpret_cast(&executable)); FBBFP_Builder_file executed_path; if (proc->executable() == proc->executed_path()) { /* Those often match, don't create the same string twice. */ fp.set_executed_path(reinterpret_cast(&executable)); } else { if (!hash_cache->get_hash(proc->executed_path(), 0, &hash)) { FB_DEBUG(FB_DEBUG_PROC, "Could not get hash of executed path: " + d(proc->executed_path())); maybe_XXH3_freeState(state); return false; } executed_path.set_path(proc->executed_path()->c_str()); executed_path.set_hash(hash.get()); fp.set_executed_path(reinterpret_cast(&executed_path)); } fp.set_original_executed_path(proc->original_executed_path()); /* The linked libraries */ std::vector lib_builders; lib_builders.reserve(proc->libs().size()); for (const auto& lib : proc->libs()) { #ifdef __APPLE__ /* SDK libraries are not present as files, see: * https://developer.apple.com/forums/thread/655588 */ if (strncmp(lib->c_str(), "/usr/lib/", strlen("/usr/lib/")) == 0) { continue; } #endif if (!hash_cache->get_hash(lib, 0, &hash)) { FB_DEBUG(FB_DEBUG_PROC, "Could not get hash of library: " + d(lib)); maybe_XXH3_freeState(state); return false; } FBBFP_Builder_file& lib_builder = lib_builders.emplace_back(); lib_builder.set_path(lib->c_str()); lib_builder.set_hash(hash.get()); } fp.set_libs_item_fn(lib_builders.size(), fbbfp_builder_file_vector_item_fn, &lib_builders); /* umask */ fp.set_umask(proc->umask()); /* The inherited files */ std::vector ofd_builders; for (const inherited_file_t& inherited_file : proc->inherited_files()) { FBBFP_Builder_ofd& ofd_builder = ofd_builders.emplace_back(); /* Workaround for #938. */ fd_type pretended_type = inherited_file.type == FD_PIPE_IN ? FD_IGNORED : inherited_file.type; ofd_builder.set_type(pretended_type); ofd_builder.set_fds(inherited_file.fds); } fp.set_ofds_item_fn(ofd_builders.size(), fbbfp_builder_ofd_vector_item_fn, &ofd_builders); FBBFP_Builder *fp_generic = reinterpret_cast(&fp); size_t len = fp_generic->measure(); std::vector buf(len); fp_generic->serialize(buf.data()); fingerprint_msgs_[proc] = buf; } maybe_XXH3_freeState(state); return true; } void ExecedProcessCacher::erase_fingerprint(const ExecedProcess *proc) { fingerprints_.erase(proc); if (FB_DEBUGGING(FB_DEBUG_CACHE) && fingerprint_msgs_.count(proc) > 0) { fingerprint_msgs_.erase(proc); } } static void add_file(std::vector* files, const FileName* file_name, const FileInfo& fi) { FBBSTORE_Builder_file& new_file = files->emplace_back(); new_file.set_path_with_length(file_name->c_str(), file_name->length()); new_file.set_type(fi.type()); if (fi.size_known()) { new_file.set_size(fi.size()); } if (fi.hash_known()) { new_file.set_hash(fi.hash().get()); } if (fi.mode_mask() != 0) { new_file.set_mode(fi.mode()); new_file.set_mode_mask(fi.mode_mask()); } } static const FBBSTORE_Builder* file_item_fn(int idx, const void *user_data) { auto fbb_file_vector = reinterpret_cast *>(user_data); return reinterpret_cast(&(*fbb_file_vector)[idx]); } static bool dir_created_or_could_exist( const char* filename, const size_t length, const tsl::hopscotch_set& out_path_isdir_filename_ptrs, const tsl::hopscotch_map& file_usages) { const FileName* parent_dir = FileName::GetParentDir(filename, length); while (parent_dir != nullptr) { const auto it = file_usages.find(parent_dir); const FileUsage* fu = it->second; if (fu->initial_type() == NOTEXIST || fu->initial_type() == NOTEXIST_OR_ISREG) { if (!fu->written()) { /* The process expects the directory to be missing but it does not create it. * This can't work. */ #ifdef FB_EXTRA_DEBUG assert(0 && "This should have been caught by FileUsage::merge()"); #endif return false; } else { if (out_path_isdir_filename_ptrs.find(parent_dir) != out_path_isdir_filename_ptrs.end()) { /* Directory is expected to be missing and the process creates it. Everything is OK. */ return true; } else { FB_DEBUG(FB_DEBUG_CACHING, "Regular file " + parent_dir->to_string() + " is created instead of a directory"); return false; } } } else if (fu->initial_type() == ISDIR) { /* Directory is expected to exist. */ return true; } parent_dir = FileName::GetParentDir(parent_dir->c_str(), parent_dir->length()); } return true; } static bool consistent_implicit_parent_dirs( const std::vector& out_path_isreg, const tsl::hopscotch_set& out_path_isdir_filename_ptrs, const tsl::hopscotch_map& file_usages) { /* If the parent dir must not exist when shortcutting and the shortcut does not create it * either, then creating the new regular file would fail. */ for (const FBBSTORE_Builder_file& file : out_path_isreg) { if (!dir_created_or_could_exist(file.get_path(), file.get_path_len(), out_path_isdir_filename_ptrs, file_usages)) { return false; } } /* Same for newly created dirs. */ for (const FileName* dir : out_path_isdir_filename_ptrs) { if (!dir_created_or_could_exist(dir->c_str(), dir->length(), out_path_isdir_filename_ptrs, file_usages)) { return false; } } return true; } static bool tmp_file_or_on_tmp_path(const FileUsage* fu, const FileName* filename, const FileName* tmpdir) { if (fu->tmp_file()) { return true; } else { if (strncmp(filename->c_str(), tmpdir->c_str(), tmpdir->length()) == 0 && filename->c_str()[tmpdir->length()] == '/') { const FileName* top_dir = proc_tree->top_dir(); assert(top_dir); return !(strncmp(filename->c_str(), top_dir->c_str(), top_dir->length()) == 0 && filename->c_str()[top_dir->length()] == '/'); } else { return false; } } } static bool rustc_deps_dir(const ExecedProcess* const proc, const FileName* const filename) { const std::vector &args = proc->args(); if (args[0] == "rustc") { for (const std::string& arg : args) { if (arg.starts_with("dependency=")) { const std::string dependency_dir(arg.substr(strlen("dependency="))); /* Assumes that the dependency dir is already absolute. */ if (dependency_dir == filename->to_string()) { return true; } } } } return false; } void ExecedProcessCacher::store(ExecedProcess *proc) { TRACK(FB_DEBUG_PROC, "proc=%s", D(proc)); if (no_store_) { /* This is when FIREBUILD_READONLY is set. We could have decided not to create PipeRecorders * at all. But maybe go with the default code path, i.e. record the data to temporary files, * but at the last step purge them instead of moving them to their final location in the cache. * This way the code path is more similar to the regular case. */ for (const inherited_file_t& inherited_file : proc->inherited_files()) { if (inherited_file.recorder) { assert(inherited_file.type == FD_PIPE_OUT); inherited_file.recorder->abandon(); } } return; } bool parent_may_be_just_sh_c_this = false; const ExecedProcess* const parent_exec_point = proc->parent_exec_point(); if (parent_exec_point) { if (ccache_disabled && parent_exec_point->executable()->without_dirs() == "ccache" && parent_exec_point->can_shortcut()) { proc->disable_shortcutting_only_this("Shortcut parent ccache ... instead"); return; } // Parent may just be sh -c , detect that parent_may_be_just_sh_c_this = (parent_exec_point->can_shortcut() && ((parent_exec_point->args().size() == 4 && parent_exec_point->args()[1] == "-c" && parent_exec_point->args()[2] == "--") || (parent_exec_point->args().size() == 3 && parent_exec_point->args()[1] == "-c") ) && shells->contains(parent_exec_point->args()[0])); } // TODO(rbalint) narrow down the cases when all args are checked const std::vector& args = proc->args(); std::string joined_cmdline = ""; for (const auto& arg : args) { if (parent_may_be_just_sh_c_this) { joined_cmdline += (joined_cmdline == "") ? arg : (" " + arg); } if (arg == "-emit-pch") { bool fno_pch_timestamp_found = false; for (const auto& arg_inner_loop : args) { if (arg_inner_loop == "-fno-pch-timestamp") { fno_pch_timestamp_found = true; break; } } if (!fno_pch_timestamp_found) { proc->disable_shortcutting_bubble_up( "Clang's -emit-pch without -Xclang -fno-pch-timestamp prevents shortcutting"); return; } break; } } if (parent_may_be_just_sh_c_this && joined_cmdline == parent_exec_point->args()[2]) { proc->disable_shortcutting_only_this("Shortcut parent sh -c ... instead"); return; } /* Go through the files the process opened for reading and/or writing. * Construct the cache entry parts describing the initial and the final state * of them. */ /* File inputs */ FBBSTORE_Builder_process_inputs pi; std::vector in_path; std::vector in_path_notexist; /* File outputs */ FBBSTORE_Builder_process_outputs po; std::vector out_path_isreg, out_path_isdir; std::vector out_path_notexist; /* Outputs for verification. */ tsl::hopscotch_set out_path_isdir_filename_ptrs; const FileName* const tmpdir = FileName::default_tmpdir; size_t in_path_non_system_count {0}, in_path_notexist_non_system_count {0}; /* Construct in_path_* in 2 passes. First collect the non-system paths and then the system paths, * for better performance. */ for (int i = 0; i < 2; i++) { for (const auto& pair : proc->file_usages()) { const auto filename = pair.first; const FileUsage* fu = pair.second; if (filename->is_in_read_only_location() == (i == 0)) { continue; } if (fu->generation() != filename->generation()) { // TODO(rbalint) extend hash cache and blob cache to reuse previously saved generations FB_DEBUG(FB_DEBUG_CACHING, "A file (" + d(filename)+ ") changed since the process used it."); proc->disable_shortcutting_only_this( generate_report ? deduplicated_string("A file (" + d(filename) + ") changed since the process used it.").c_str() : "A file could not be stored because it changed since the process used it."); return; } /* If the file's initial contents matter, record it in pb's "inputs". * This is purely data conversion from one format to another. */ switch (fu->initial_type()) { case DONTKNOW: /* Nothing to do. */ break; case NOTEXIST: /* NOTEXIST is handled specially to save space in the FBB. */ in_path_notexist.push_back({filename->c_str(), filename->length()}); break; case ISDIR: if (fu->initial_state().hash_known() && ((quirks & FB_QUIRK_IGNORE_TMP_LISTING && filename == tmpdir) || rustc_deps_dir(proc, filename))) { FileInfo no_hash_initial_state(fu->initial_state()); no_hash_initial_state.set_hash(nullptr); add_file(&in_path, filename, no_hash_initial_state); break; } [[fallthrough]]; default: if (fu->initial_state().type() == ISREG && tmp_file_or_on_tmp_path(fu, filename, tmpdir)) { FB_DEBUG(FB_DEBUG_CACHING, "Not storing cache entry because it read " + d(filename) + ", which is a temporary file"); return; } add_file(&in_path, filename, fu->initial_state()); break; } } in_path_non_system_count = in_path.size(); in_path_notexist_non_system_count = in_path_notexist.size(); } uint64_t stored_blob_bytes = 0; for (const auto& pair : proc->file_usages()) { const auto filename = pair.first; const FileUsage* fu = pair.second; /* fu contains information about the file's original contents or metadata, plus whether it's * been modified. We need to query the current state of the file if anything has been modified. */ if (!fu->written() && !fu->mode_changed()) { /* Completely unchanged, nothing to do. */ continue; } FileInfo new_file_info(DONTKNOW); struct stat64 st; if (stat64(filename->c_str(), &st) == 0) { /* We have something, let's see what it is. */ new_file_info.set_type(EXIST); /* If the file's final contents matter, place it in the file cache, * and also record it in pb's "outputs". This actually needs to * compute the checksums now. */ if (fu->written()) { if (S_ISREG(st.st_mode)) { new_file_info.set_type(ISREG); Hash new_hash; /* TODO don't store and don't record if it was read with the same hash. */ int fd = open(filename->c_str(), O_RDONLY); if (fd >= 0) { off_t stored_bytes = 0; if (!hash_cache->store_and_get_hash(filename, 0, &new_hash, &stored_bytes, fd, &st)) { /* unexpected error, now what? */ FB_DEBUG(FB_DEBUG_CACHING, "Could not store blob in cache, not writing shortcut info"); close(fd); proc->disable_shortcutting_only_this( "Could not store blob in cache, not writing shortcut info"); return; } close(fd); new_file_info.set_size(st.st_size); new_file_info.set_hash(new_hash); if ((stored_blob_bytes += stored_bytes) > max_entry_size) { FB_DEBUG(FB_DEBUG_CACHING, "Could not store blob in cache because it would exceed max_entry_size"); return; } } else { fb_perror("open"); new_file_info.set_type(NOTEXIST); } } else if (S_ISDIR(st.st_mode)) { new_file_info.set_type(ISDIR); } else { // TODO(egmont) handle other types of entries new_file_info.set_type(NOTEXIST); } } if (fu->mode_changed()) { // TODO(egmont) fail if setuid/setgid/sticky is set new_file_info.set_mode_bits(st.st_mode & 07777, 07777); } } else { /* Stat failed, nothing at the new location. */ new_file_info.set_type(NOTEXIST); } switch (new_file_info.type()) { case DONTKNOW: /* This can happen if we figured out that the file didn't actually change. */ break; case EXIST: case ISREG: // FIXME skip adding if the new state is the same as the old one if (tmp_file_or_on_tmp_path(fu, filename, tmpdir)) { FB_DEBUG(FB_DEBUG_CACHING, "Temporary file (" + d(filename) + ") can't be process output."); proc->disable_shortcutting_only_this("Process created a temporary file"); return; } add_file(&out_path_isreg, filename, new_file_info); break; case ISDIR: // FIXME skip adding if the new state is the same as the old one if (tmp_file_or_on_tmp_path(fu, filename, tmpdir)) { FB_DEBUG(FB_DEBUG_CACHING, "Temporary dir (" + d(filename) + ") can't be process output."); proc->disable_shortcutting_only_this("Process created a temporary dir"); return; } add_file(&out_path_isdir, filename, new_file_info); out_path_isdir_filename_ptrs.insert(filename); break; case NOTEXIST: if (fu->initial_type() != NOTEXIST) { out_path_notexist.push_back(filename->c_str()); } break; default: assert(0); } } /* Data appended to inherited files (pipes, regular files) */ std::vector out_append_to_fd; /* Store what was written to the inherited pipes. Use the fd as of when the process started up, * because this is what matters if we want to replay; how the process later dup()ed it to other * fds is irrelevant. Similarly, no need to store the data written to pipes opened by this * process, that data won't ever be replayed. */ for (const inherited_file_t& inherited_file : proc->inherited_files()) { if (is_write(inherited_file.flags)) { /* Record the output as belonging to the lowest fd. */ int fd = inherited_file.fds[0]; if (inherited_file.type == FD_PIPE_OUT) { std::shared_ptr recorder = inherited_file.recorder; if (recorder) { bool is_empty; Hash hash; off_t stored_bytes = 0; if (!recorder->store(&is_empty, &hash, &stored_bytes)) { // FIXME handle error FB_DEBUG(FB_DEBUG_CACHING, "Could not store pipe traffic in cache, not writing shortcut info"); proc->disable_shortcutting_only_this( "Could not store pipe traffic in cache, not writing shortcut info"); return; } if ((stored_blob_bytes += stored_bytes) > max_entry_size) { FB_DEBUG(FB_DEBUG_CACHING, "Could not store blob in cache because it would exceed max_entry_size"); return; } if (!is_empty) { /* Note: pipes with no traffic are just simply not mentioned here in the "outputs" section. * They were taken into account when computing the process's fingerprint. */ FBBSTORE_Builder_append_to_fd& new_append = out_append_to_fd.emplace_back(); new_append.set_fd(fd); new_append.set_hash(hash.get()); } } } else if (inherited_file.type == FD_FILE) { Hash hash; struct stat64 st; if (stat64(inherited_file.filename->c_str(), &st) < 0) { // FIXME handle error FB_DEBUG(FB_DEBUG_CACHING, "Could not stat file, not writing shortcut info"); proc->disable_shortcutting_only_this( "Could not stat file, not writing shortcut info"); return; } else if (!S_ISREG(st.st_mode)) { // FIXME handle error FB_DEBUG(FB_DEBUG_CACHING, "Not a regular file, not writing shortcut info"); proc->disable_shortcutting_only_this( "Not a regular file, not writing shortcut info"); return; } else if (st.st_size < inherited_file.start_offset) { // FIXME handle error FB_DEBUG(FB_DEBUG_CACHING, "File shrank during appending, not writing shortcut info"); proc->disable_shortcutting_only_this( "File shrank during appending, not writing shortcut info"); return; } else if (st.st_size > inherited_file.start_offset) { /* Note: files that weren't appended to are just simply not mentioned here in the * "outputs" section. They were taken into account when computing the fingerprint. */ if (!blob_cache->store_file(inherited_file.filename, 1, -1, inherited_file.start_offset, st.st_size, &hash)) { // FIXME handle error FB_DEBUG(FB_DEBUG_CACHING, "Could not store file fragment in cache, not writing shortcut info"); proc->disable_shortcutting_only_this( "Could not store file fragment in cache, not writing shortcut info"); return; } else { if ((stored_blob_bytes += st.st_size - inherited_file.start_offset) > max_entry_size) { FB_DEBUG(FB_DEBUG_CACHING, "Could not store blob in cache because it would exceed max_entry_size"); return; } FBBSTORE_Builder_append_to_fd& new_append = out_append_to_fd.emplace_back(); new_append.set_fd(fd); new_append.set_hash(hash.get()); } } } } } /* Validate cache entry to be stored. */ if (!consistent_implicit_parent_dirs(out_path_isreg, out_path_isdir_filename_ptrs, proc->file_usages())) { proc->disable_shortcutting_only_this( "Inconsistency: A parent dir of an output file must not exit for shortcutting."); return; } /* Sort the entries for better cache compression ratios and easier debugging. * * Note that previously we carefully collected the inputs and outputs in system and non-system * locations separately for performance reasons. Here the outputs are mixed because they are not * likely to be in system locations, but inputs are still separated to system and non-system * locations. */ struct { bool operator()(const FBBSTORE_Builder_file& a, const FBBSTORE_Builder_file& b) const { return strcmp(a.get_path(), b.get_path()) < 0; } } file_less; /* Sort non-system and system paths separately to not regress in shortcutting performance. */ std::sort(in_path.begin(), in_path.begin() + in_path_non_system_count, file_less); std::sort(in_path.begin() + in_path_non_system_count, in_path.end(), file_less); std::sort(out_path_isreg.begin(), out_path_isreg.end(), file_less); std::sort(out_path_isdir.begin(), out_path_isdir.end(), file_less); struct { bool operator()(const cstring_view& a, const cstring_view& b) const { return strcmp(a.c_str, b.c_str) < 0; } } cstring_view_less; /* Sort non-system and system paths separately to not regress in shortcutting performance. */ std::sort(in_path_notexist.begin(), in_path_notexist.begin() + in_path_notexist_non_system_count, cstring_view_less); std::sort(in_path_notexist.begin() + in_path_notexist_non_system_count, in_path_notexist.end(), cstring_view_less); std::sort(out_path_notexist.begin(), out_path_notexist.end()); pi.set_path_item_fn(in_path.size(), file_item_fn, &in_path); pi.set_path_notexist(in_path_notexist); po.set_path_isreg_item_fn(out_path_isreg.size(), file_item_fn, &out_path_isreg); po.set_path_isdir_item_fn(out_path_isdir.size(), file_item_fn, &out_path_isdir); po.set_path_notexist(out_path_notexist); po.set_append_to_fd_item_fn(out_append_to_fd.size(), fbbstore_builder_append_to_fd_vector_item_fn, &out_append_to_fd); po.set_exit_status(proc->fork_point()->exit_status()); // TODO(egmont) Add all sorts of other stuff FBBSTORE_Builder_process_inputs_outputs pio; pio.set_inputs(reinterpret_cast(&pi)); pio.set_outputs(reinterpret_cast(&po)); if (!FB_DEBUGGING(FB_DEBUG_DETERMINISTIC_CACHE)) { pio.set_cpu_time_ms((proc->aggr_cpu_time_u() / 1000) + proc->shortcut_cpu_time_ms()); } const FBBFP_Serialized *debug_msg = NULL; if (FB_DEBUGGING(FB_DEBUG_CACHE)) { debug_msg = reinterpret_cast(fingerprint_msgs_[proc].data()); } /* Store in the cache everything about this process. */ const Hash fingerprint = fingerprints_[proc]; obj_cache->store(fingerprint, reinterpret_cast(&pio), stored_blob_bytes, debug_msg); } void ExecedProcessCacher::update_cached_bytes(off_t bytes) { this_runs_cached_bytes_ += bytes; #ifdef FB_EXTRA_DEBUG off_t total = obj_cache->gc_collect_total_objects_size() + blob_cache->gc_collect_total_blobs_size(); off_t stored = get_stored_bytes_from_cache(); FB_DEBUG(FB_DEBUG_CACHING, " Cache-size real: " + d(total) + " calculated: " + d(stored + this_runs_cached_bytes_) + " stored: " + d(stored)); assert_cmp(total, ==, stored + this_runs_cached_bytes_); #endif } /** * Create a FileInfo object based on an FBB's File entry. */ static FileInfo file_to_file_info(const FBBSTORE_Serialized_file *file) { FileInfo info(file->get_type()); if (file->has_size()) { info.set_size(file->get_size()); } if (file->has_hash()) { Hash hash(file->get_hash()); info.set_hash(hash); } info.set_mode_bits(file->get_mode_with_fallback(0), file->get_mode_mask_with_fallback(0)); return info; } /** * Create a FileUsageUpdate object based on an FBB's File entry. */ static FileUsageUpdate file_to_file_usage_update(const FileName *filename, const FBBSTORE_Serialized_file *file) { bool written = (file->get_type() == ISREG && file->has_size()) || (file->get_type() == ISDIR); bool mode_changed = file->has_mode(); /* "file" represents the file's _new_ state, so it's not suitable for the _initial_ state of the * update. The initial state can be anything, i.e. DONTKNOW. */ FileUsageUpdate update(filename, DONTKNOW, written, mode_changed); return update; } static const FBBSTORE_Serialized_file* find_input_file(const FBBSTORE_Serialized_process_inputs *pi, const FileName* path) { for (size_t i = 0; i < pi->get_path_count(); i++) { auto file = reinterpret_cast(pi->get_path_at(i)); if (FileName::Get(file->get_path(), file->get_path_len()) == path) { return file; } } return nullptr; } /** * Check whether the given process inputs match the file system's current contents * and the outputs are likely applicable. */ static bool pio_matches_fs(const FBBSTORE_Serialized_process_inputs_outputs *candidate_inouts, const char* const subkey, ExecedProcess* proc) { TRACK(FB_DEBUG_PROC, "subkey=%s", D(subkey)); const FBBSTORE_Serialized *inputs_fbb = candidate_inouts->get_inputs(); assert_cmp(inputs_fbb->get_tag(), ==, FBBSTORE_TAG_process_inputs); auto inputs = reinterpret_cast(inputs_fbb); size_t i; for (i = 0; i < inputs->get_path_count(); i++) { auto file = reinterpret_cast(inputs->get_path_at(i)); const auto path = FileName::Get(file->get_path(), file->get_path_len()); const FileInfo query = file_to_file_info(file); if (!hash_cache->file_info_matches(path, query)) { FB_DEBUG(FB_DEBUG_SHORTCUT, "│ " + d(subkey) + " mismatches e.g. at " + d(path)); /* Store only the first mismatch. */ if (generate_report && !proc->shortcut_result()) { proc->set_shortcut_result(deduplicated_string( d(subkey) + " mismatches e.g. at " + d(path)).c_str()); } return false; } } for (i = 0; i < inputs->get_path_notexist_count(); i++) { const auto path = FileName::Get(inputs->get_path_notexist_at(i), inputs->get_path_notexist_len_at(i)); const FileInfo query(NOTEXIST); if (!hash_cache->file_info_matches(path, query)) { /* Store only the first mismatch. */ if (generate_report && !proc->shortcut_result()) { proc->set_shortcut_result(deduplicated_string( d(subkey) + + " mismatches e.g. at " + d(path) + ": path expected to be missing, existing object is found").c_str()); } FB_DEBUG(FB_DEBUG_SHORTCUT, "│ " + d(subkey) + " mismatches e.g. at " + d(path) + ": path expected to be missing, existing object is found"); return false; } } const FBBSTORE_Serialized_process_outputs *outputs = reinterpret_cast (candidate_inouts->get_outputs()); /* Check if shortcut is applicable, i.e. outputs can be created/can be written, etc. */ // TODO(rbalint) extend these checks for (i = 0; i < outputs->get_path_isreg_count(); i++) { auto file = reinterpret_cast(outputs->get_path_isreg_at(i)); if (file->get_type() == ISREG && access(file->get_path(), W_OK) == -1) { if (errno == EACCES) { /* The regular file can't be written, let's see if that was expected. */ const auto path = FileName::Get(file->get_path(), file->get_path_len()); const FBBSTORE_Serialized_file* input_file = find_input_file(inputs, path); if (input_file && (file_to_file_info(file).mode_mask() & 0200)) { /* The file has already been checked to be not writable and will be replaced while * applying the shortcut. */ } else { if (generate_report && !proc->shortcut_result()) { proc->set_shortcut_result(deduplicated_string( std::string("file to be written is not writable: ") + file->get_path()).c_str()); } return false; } } } } return true; } const FBBSTORE_Serialized_process_inputs_outputs * ExecedProcessCacher::find_shortcut( ExecedProcess *proc, uint8_t **inouts_buf, size_t *inouts_buf_len, Subkey* subkey_out) { TRACK(FB_DEBUG_PROC, "proc=%s", D(proc)); const FBBSTORE_Serialized_process_inputs_outputs *inouts = nullptr; int shortcut_attempts {0}; #ifdef FB_EXTRA_DEBUG int count = 0; #endif Hash fingerprint = fingerprints_[proc]; // FIXME error handling FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Candidates:"); const std::vector subkeys = obj_cache->list_subkeys(fingerprint); if (subkeys.empty()) { if (generate_report) { proc->set_shortcut_result(deduplicated_string("no candidate found").c_str()); } FB_DEBUG(FB_DEBUG_SHORTCUT, "│ None found"); } for (const Subkey& subkey : subkeys) { uint8_t *candidate_inouts_buf; size_t candidate_inouts_buf_len; if (shortcut_attempts++ > shortcut_tries) { FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Maximum shortcutting attempts (" + d(shortcut_tries) + ") exceeded, giving up"); break; } if (!obj_cache->retrieve(fingerprint, subkey.c_str(), &candidate_inouts_buf, &candidate_inouts_buf_len)) { if (generate_report) { proc->set_shortcut_result(deduplicated_string( "could not retrieve " + d(subkey) + " from objcache").c_str()); } FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Cannot retrieve " + d(subkey) + " from objcache, ignoring"); continue; } auto candidate_inouts_fbb = reinterpret_cast(candidate_inouts_buf); assert_cmp(candidate_inouts_fbb->get_tag(), ==, FBBSTORE_TAG_process_inputs_outputs); auto candidate_inouts = reinterpret_cast(candidate_inouts_fbb); if (pio_matches_fs(candidate_inouts, subkey.c_str(), proc)) { FB_DEBUG(FB_DEBUG_SHORTCUT, "│ " + d(subkey) + " matches the file system"); #ifdef FB_EXTRA_DEBUG count++; if (count == 1) { #endif *inouts_buf = candidate_inouts_buf; *inouts_buf_len = candidate_inouts_buf_len; *subkey_out = subkey; inouts = candidate_inouts; #ifdef FB_EXTRA_DEBUG /* Let's play safe for now and not break out of this loop, let's * make sure that there are no other matches. */ } if (count == 2) { FB_DEBUG(FB_DEBUG_SHORTCUT, "│ More than 1 matching candidates found, still using the first one"); munmap(candidate_inouts_buf, candidate_inouts_buf_len); break; } #else /* In rare cases there could be multiple matches because the same content can be stored under * different file names. This is not likely to happen because if a process can be shortcut it * is shortcut from the existing cache object and the result is not cached again. OTOH if two * processes with identical inputs and outputs start and end around the same time and none of * them could be shortcut from the cache, then both can be cached generating differently named * cache files with identical content. */ break; #endif } else { munmap(candidate_inouts_buf, candidate_inouts_buf_len); } } /* The retval is currently the same as the memory address to unmap (i.e. *inouts_buf). * They used to be different, and could easily become different again in the future, * so leave the two for now. */ return inouts; } /** * Restore output directories with the right mode. * * Create them in ascending order of the pathname lengths, so that parent directories are guaranteed * to be processed before its children. */ static bool restore_dirs( ExecedProcess* proc, const FBBSTORE_Serialized_process_outputs *outputs) { /* Construct indices 0 .. path_isdir_count()-1 and initialize them with these values */ std::vector indices(outputs->get_path_isdir_count()); size_t i; for (i = 0; i < outputs->get_path_isdir_count(); i++) { indices[i] = i; } /* Sort the indices according to the pathname length at the given index */ struct { const FBBSTORE_Serialized_process_outputs *outputs; bool operator()(const int& i1, const int& i2) const { const FBBSTORE_Serialized *file1_generic = outputs->get_path_isdir_at(i1); auto file1 = reinterpret_cast(file1_generic); const FBBSTORE_Serialized *file2_generic = outputs->get_path_isdir_at(i2); auto file2 = reinterpret_cast(file2_generic); return file1->get_path_len() < file2->get_path_len(); } } pathname_length_less; pathname_length_less.outputs = outputs; std::sort(indices.begin(), indices.end(), pathname_length_less); /* Process the directory names in ascending order of their lengths */ for (i = 0; i < outputs->get_path_isdir_count(); i++) { const FBBSTORE_Serialized *dir_generic = outputs->get_path_isdir_at(indices[i]); assert_cmp(dir_generic->get_tag(), ==, FBBSTORE_TAG_file); auto dir = reinterpret_cast(dir_generic); const auto path = FileName::Get(dir->get_path(), dir->get_path_len()); assert(dir->has_mode()); mode_t mode = dir->get_mode(); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Creating directory: " + d(path)); int ret = mkdir(path->c_str(), mode); if (ret != 0) { if (errno == EEXIST) { struct stat64 st; if (stat64(path->c_str(), &st) != 0) { fb_perror("Failed to stat() existing pathname"); assert_cmp(ret, !=, -1); return false; } if (!S_ISDIR(st.st_mode)) { fb_perror("Failed to restore directory, the target already exists and is not a dir"); assert_cmp(ret, !=, -1); return false; } if (chmod(path->c_str(), mode) != 0) { fb_perror("Failed to restore directory's permissions"); assert_cmp(ret, !=, -1); return false; } } else { fb_perror("Failed to restore directory"); assert_cmp(ret, !=, -1); return false; } } if (proc->parent_exec_point()) { FileUsageUpdate update = file_to_file_usage_update(path, dir); proc->parent_exec_point()->register_file_usage_update(path, update); } } return true; } /** * Remove files and directories. * * Remove them in descending order of the pathname lengths, so that parent directories are * guaranteed to be processed after its children. * * Note: There's deliberately no error checking. Maybe a program creates a temporary file in a way * that we cannot tell whether that file existed previously, and then deletes it. So by design the * unlink attempt might fail. Maybe one day we can refine this logic. */ static void remove_files_and_dirs( ExecedProcess* proc, const FBBSTORE_Serialized_process_outputs *outputs) { /* Construct indices 0 .. path_notexist_count()-1 and initialize them with these values */ std::vector indices(outputs->get_path_notexist_count()); size_t i; for (i = 0; i < outputs->get_path_notexist_count(); i++) { indices[i] = i; } /* Reverse sort the indices according to the pathname length at the given index */ struct { const FBBSTORE_Serialized_process_outputs *outputs; bool operator()(const int& i1, const int& i2) const { int len1 = outputs->get_path_notexist_len_at(i1); int len2 = outputs->get_path_notexist_len_at(i2); return len1 > len2; } } pathname_length_greater; pathname_length_greater.outputs = outputs; std::sort(indices.begin(), indices.end(), pathname_length_greater); /* Process the directory names in descending order of their lengths */ for (i = 0; i < outputs->get_path_notexist_count(); i++) { const auto path = FileName::Get(outputs->get_path_notexist_at(indices[i]), outputs->get_path_notexist_len_at(indices[i])); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Deleting file or directory: " + d(path)); if (unlink(path->c_str()) < 0 && errno == EISDIR) { rmdir(path->c_str()); } if (proc->parent_exec_point()) { // FIXME Register that it was an _empty_ directory FileUsageUpdate update = FileUsageUpdate(path, ISDIR, true); proc->parent_exec_point()->register_file_usage_update(path, update); } } } static void close_all(std::vector* fds) { for (int fd : *fds) { close(fd); } } static bool add_blob_fd_from_hash(const XXH128_hash_t& fbb_hash, std::vector* blob_fds) { Hash hash(fbb_hash); int fd; if ((fd = blob_cache->get_fd_for_file(hash)) != -1) { blob_fds->push_back(fd); } else { close_all(blob_fds); return false; } return true; } /** * Applies the given shortcut. * * Modifies the file system to match the given instructions. Propagates * upwards all the shortcutted file read and write events. */ bool ExecedProcessCacher::apply_shortcut(ExecedProcess *proc, const FBBSTORE_Serialized_process_inputs_outputs *inouts, std::vector *fds_appended_to) { TRACK(FB_DEBUG_PROC, "proc=%s", D(proc)); size_t i; std::vector blob_fds; const FBBSTORE_Serialized_process_outputs *outputs = reinterpret_cast (inouts->get_outputs()); /* Pre-open blobs to be used later to prevent a parallel GC run from removing them while * shortcutting. */ for (i = 0; i < outputs->get_path_isreg_count(); i++) { auto file = reinterpret_cast(outputs->get_path_isreg_at(i)); if (file->get_type() == ISREG) { assert(file->has_hash()); if (!add_blob_fd_from_hash(file->get_hash(), &blob_fds)) { return false; } } } for (i = 0; i < outputs->get_append_to_fd_count(); i++) { auto append_to_fd = reinterpret_cast (outputs->get_append_to_fd_at(i)); if (!add_blob_fd_from_hash(append_to_fd->get_hash(), &blob_fds)) { return false; } } /* Bubble up all the file operations we're about to perform. */ ExecedProcess* registration_point = generate_report ? proc : proc->parent_exec_point(); if (registration_point) { const FBBSTORE_Serialized_process_inputs *inputs = reinterpret_cast (inouts->get_inputs()); for (i = 0; i < inputs->get_path_count(); i++) { auto file = reinterpret_cast(inputs->get_path_at(i)); const auto path = FileName::Get(file->get_path(), file->get_path_len()); FileInfo info = file_to_file_info(file); registration_point->register_file_usage_update(path, FileUsageUpdate(path, info)); } for (i = 0; i < inputs->get_path_notexist_count(); i++) { const auto path = FileName::Get(inputs->get_path_notexist_at(i), inputs->get_path_notexist_len_at(i)); registration_point->register_file_usage_update(path, FileUsageUpdate(path, NOTEXIST)); } } if (!restore_dirs(proc, outputs)) { close_all(&blob_fds); return false; } size_t next_blob_fd_idx = 0; for (i = 0; i < outputs->get_path_isreg_count(); i++) { auto file = reinterpret_cast(outputs->get_path_isreg_at(i)); const auto path = FileName::Get(file->get_path(), file->get_path_len()); if (file->get_type() == ISREG) { FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Fetching file from blobs cache: " + d(path)); assert(file->has_hash()); Hash hash(file->get_hash()); if (!blob_cache->retrieve_file(blob_fds[next_blob_fd_idx++], path, false)) { /* The file may not be writable but it may be expected and already checked. */ const FBBSTORE_Serialized_file* input_file = find_input_file( reinterpret_cast(inouts->get_inputs()), path); if (errno == EACCES && input_file && (file_to_file_info(file).mode_mask() & 0200)) { /* The file has already been checked to be not writable and should be completely replaced * from the cache. Let's remove it and try again. */ if (unlink(path->c_str()) == -1) { fb_perror("Failed removing file to be replaced from cache"); assert(0); } /* Try retrieving the same file again. */ if (!blob_cache->retrieve_file(blob_fds[next_blob_fd_idx - 1], path, false)) { fb_perror("Failed creating file from cache"); assert(0); } } else { fb_perror("Failed opening file to be recreated from cache"); assert(0); } } } if (file->has_mode()) { /* Refuse to apply setuid, setgid, sticky bit. */ // FIXME warn on them, even when we store them. chmod(path->c_str(), file->get_mode() & 0777); } if (registration_point) { FileUsageUpdate update = file_to_file_usage_update(path, file); registration_point->register_file_usage_update(path, update); } } remove_files_and_dirs(proc, outputs); /* See what the process originally wrote to its inherited files (pipes or regular files). * Replay these. */ for (i = 0; i < outputs->get_append_to_fd_count(); i++) { auto append_to_fd = reinterpret_cast (outputs->get_append_to_fd_at(i)); FileFD *ffd = proc->get_fd(append_to_fd->get_fd()); assert(ffd); if (ffd->type() == FD_PIPE_OUT) { Pipe *pipe = ffd->pipe().get(); assert(pipe); Hash hash(append_to_fd->get_hash()); int fd = blob_fds[next_blob_fd_idx++]; struct stat64 st; if (fstat64(fd, &st) < 0) { assert(0 && "fstat"); } pipe->add_data_from_fd(fd, st.st_size); if (proc->parent()) { /* Bubble up the replayed pipe data. */ std::vector>& recorders = pipe->proc2recorders[proc->parent_exec_point()]; PipeRecorder::record_data_from_regular_fd(&recorders, fd, st.st_size); } close(fd); } else if (ffd->type() == FD_FILE) { assert(ffd->filename()); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Fetching file fragment from blobs cache: " + d(ffd->filename())); Hash hash(append_to_fd->get_hash()); blob_cache->retrieve_file(blob_fds[next_blob_fd_idx++], ffd->filename(), true); /* Tell the interceptor to seek forward in this fd. */ fds_appended_to->push_back(ffd->fd()); } else { assert(0 && "wrong file_fd type"); } /* Bubble up the event that we wrote to this inherited fd. Currently this doesn't do anything, * but it might change in the future. */ proc->handle_write_to_inherited(ffd->fd(), false); } /* Set the exit code, propagate upwards. */ // TODO(egmont) what to do with resource usage? proc->fork_point()->set_exit_status(outputs->get_exit_status()); close_all(&blob_fds); return true; } /** * Tries to shortcut the process. * * Returns if it succeeded. */ bool ExecedProcessCacher::shortcut(ExecedProcess *proc, std::vector *fds_appended_to) { TRACK(FB_DEBUG_PROC, "proc=%s", D(proc)); if (no_fetch_) { return false; } shortcut_attempts_++; bool ret = false; uint8_t * inouts_buf = NULL; size_t inouts_buf_len = 0; const FBBSTORE_Serialized_process_inputs_outputs *inouts = NULL; if (FB_DEBUGGING(FB_DEBUG_SHORTCUT)) { FB_DEBUG(FB_DEBUG_SHORTCUT, "┌─"); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Trying to shortcut process:"); if (proc->can_shortcut()) { FB_DEBUG(FB_DEBUG_SHORTCUT, "│ fingerprint = " + d(fingerprints_[proc])); } FB_DEBUG(FB_DEBUG_SHORTCUT, "│ executed path = " + d(proc->executed_path())); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ exe = " + d(proc->executable())); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ arg = " + d(proc->args())); /* FB_DEBUG(FB_DEBUG_SHORTCUT, "│ env = " + d(proc->env_vars())); */ } Subkey subkey; if (proc->can_shortcut()) { inouts = find_shortcut(proc, &inouts_buf, &inouts_buf_len, &subkey); } FB_DEBUG(FB_DEBUG_SHORTCUT, inouts ? "│ Shortcutting:" : "│ Not shortcutting."); if (inouts) { ret = apply_shortcut(proc, inouts, fds_appended_to); FB_DEBUG(FB_DEBUG_SHORTCUT, "│ Exiting with " + d(proc->fork_point()->exit_status())); if (ret) { Hash fp = fingerprints_[proc]; obj_cache->mark_as_used(fp, subkey.c_str()); shortcut_hits_++; if (inouts->has_cpu_time_ms()) { proc->add_shortcut_cpu_time_ms(inouts->get_cpu_time_ms()); } } else if (generate_report) { proc->set_shortcut_result("applying shortcut failed"); } /* Trigger cleanup of ProcessInputsOutputs. */ inouts = nullptr; munmap(inouts_buf, inouts_buf_len); } FB_DEBUG(FB_DEBUG_SHORTCUT, "└─"); proc->set_was_shortcut(ret); return ret; } /** * Checks if the blob is present in the blob cache and saves existing blobs' hash to * referenced_blobs. */ static bool blob_present(const Hash& hash, tsl::hopscotch_set* referenced_blobs) { char ascii_hash_buf[Hash::kAsciiLength + 1]; hash.to_ascii(ascii_hash_buf); AsciiHash ascii_hash {ascii_hash_buf}; if (referenced_blobs->find(ascii_hash) == referenced_blobs->end()) { int fd = blob_cache->get_fd_for_file(hash); if (fd == -1) { FB_DEBUG(FB_DEBUG_CACHING, "Cache entry contains reference to an output blob missing from the cache: " + d(ascii_hash)); FB_DEBUG(FB_DEBUG_CACHING, "The cache may have been corrupted."); return false; } else { // TODO(rbalint) validate content's hash close(fd); referenced_blobs->insert(ascii_hash); } } return true; } bool ExecedProcessCacher::is_entry_usable(uint8_t* entry_buf, tsl::hopscotch_set* referenced_blobs) { auto inouts_fbb = reinterpret_cast(entry_buf); if (inouts_fbb->get_tag() != FBBSTORE_TAG_process_inputs_outputs) { return false; } auto inouts = reinterpret_cast(inouts_fbb); const FBBSTORE_Serialized *inputs_fbb = inouts->get_inputs(); if (inputs_fbb->get_tag() != FBBSTORE_TAG_process_inputs) { return false; } auto inputs = reinterpret_cast(inputs_fbb); /* Check existing regular system files files. * Only existing ones because --gc may be run when some build dependencies are missing which * would be installed before CI runs where firebuild is in use. */ for (size_t i = 0; i < inputs->get_path_count(); i++) { auto file = reinterpret_cast(inputs->get_path_at(i)); const auto path {FileName::Get(file->get_path(), file->get_path_len())}; const FileInfo query {file_to_file_info(file)}; if (query.type() == ISREG && path->is_in_read_only_location() && !hash_cache->file_info_matches(path, query) && hash_cache->file_info_matches(path, FileInfo(EXIST))) { FB_DEBUG(FB_DEBUG_CACHING, "Cache entry expects a system file that has changed: " + d(path)); return false; } } /* The entry seems to be valid, collect the referenced blobs. */ auto outputs = reinterpret_cast(inouts->get_outputs()); for (size_t i = 0; i < outputs->get_path_isreg_count(); i++) { auto file = reinterpret_cast(outputs->get_path_isreg_at(i)); if (file->get_type() == ISREG && file->has_hash()) { if (!blob_present(Hash(file->get_hash()), referenced_blobs)) { return false; } } } for (size_t i = 0; i < outputs->get_append_to_fd_count(); i++) { auto append_to_fd = reinterpret_cast (outputs->get_append_to_fd_at(i)); if (!blob_present(Hash(append_to_fd->get_hash()), referenced_blobs)) { return false; } } return true; } static void print_time(FILE* f, const int time_ms) { double time = time_ms; if (time_ms < 0) { fprintf(f, "-"); time = -time_ms; } if (time < 1000) { fprintf(f, "%.0f ms", time); return; } time /= 1000; if (time < 60) { fprintf(f, "%.2f seconds", time); return; } time /= 60; if (time < 60) { fprintf(f, "%.2f minutes", time); return; } time /= 60; if (time < 24) { fprintf(f, "%.2f hours", time); return; } time /= 24; if (time < 7) { fprintf(f, "%.2f days", time); return; } time /= 7; fprintf(f, "%.2f weeks", time); } static void print_bytes(FILE* f, const off_t bytes) { double size = bytes; if (size < 0) { fprintf(f, "-"); size = -size; } size /= 1000; if (size < 1000) { fprintf(f, "%.2f kB", size); return; } size /= 1000; if (size < 1000) { fprintf(f, "%.2f MB", size); return; } size /= 1000; fprintf(f, "%.2f GB", size); } void ExecedProcessCacher::print_stats(stats_type what) { printf("Statistics of %s:\n", what == FB_SHOW_STATS_CURRENT ? "current run" : "stored cache"); printf(" Hits: %6u / %u (%.2f %%)\n", shortcut_hits_, shortcut_attempts_, shortcut_attempts_ > 0 ? (static_cast(100 * shortcut_hits_) / shortcut_attempts_) : 0); printf(" Misses: %6u\n", shortcut_attempts_ - shortcut_hits_); printf(" Uncacheable: %6u\n", not_shortcutting_); printf(" GC runs: %6u\n", gc_runs_); if (what == FB_SHOW_STATS_CURRENT) { printf("Newly cached: "); print_bytes(stdout, this_runs_cached_bytes_); } else { printf("Cache size: "); print_bytes(stdout, get_stored_bytes_from_cache()); } printf("\n"); printf("Saved CPU time: "); print_time(stdout, cache_saved_cpu_time_ms_ - self_cpu_time_ms_ + (proc_tree ? proc_tree->shortcut_cpu_time_ms() : 0)); printf("\n"); } void ExecedProcessCacher::add_stored_stats() { /* Read cache statistics. */ FILE* f; unsigned int shortcut_attempts, shortcut_hits, not_shortcutting, gc_runs; const std::string stats_file = cache_dir_ + "/" + kCacheStatsFile; if ((f = fopen(stats_file.c_str(), "r"))) { if (fscanf(f, "attempts: %u\nhits: %u\nskips: %u\ngc_runs: %u\nsaved_cpu_ms: %" SCNd64 "\n", &shortcut_attempts, &shortcut_hits, ¬_shortcutting, &gc_runs, &cache_saved_cpu_time_ms_) != 5) { fb_error("Invalid stats file format at " + stats_file + ", using only current run's stats."); } else { shortcut_attempts_ += shortcut_attempts; shortcut_hits_ += shortcut_hits; not_shortcutting_ += not_shortcutting; gc_runs_ += gc_runs; } fclose(f); } } void ExecedProcessCacher::reset_stored_stats() { const std::string stats_file = cache_dir_ + "/" + kCacheStatsFile; if (unlink(stats_file.c_str()) == -1 && errno != ENOENT) { fb_perror("removing stats file failed"); exit(EXIT_FAILURE); } } void ExecedProcessCacher::update_stored_stats() { // FIXME(rbalint) There is a slight chance for two parallel builds updating the stats at the // same time making them inaccurate. add_stored_stats(); const std::string stats_file = cache_dir_ + "/" + kCacheStatsFile; if (file_overwrite_printf( stats_file, "attempts: %u\nhits: %u\nskips: %u\ngc_runs: %u\nsaved_cpu_ms: %" PRId64 "\n", shortcut_attempts_, shortcut_hits_, not_shortcutting_, gc_runs_, cache_saved_cpu_time_ms_ - self_cpu_time_ms_ + (proc_tree ? proc_tree->shortcut_cpu_time_ms() : 0)) < 0) { fb_error("writing cache stats file failed"); exit(EXIT_FAILURE); } } off_t ExecedProcessCacher::get_stored_bytes_from_cache() const { FILE* f; const std::string size_file = cache_dir_ + "/" + kCacheSizeFile; off_t cached_bytes = 0; if ((f = fopen(size_file.c_str(), "r"))) { if (fscanf(f, "%" SCNoff "\n", &cached_bytes) != 1) { fb_error("Invalid size file format in " + size_file + ", fixing it."); fclose(f); return fix_stored_bytes(); } fclose(f); } if (cached_bytes < 0) { fb_error("Invalid size in " + size_file + ", fixing it."); cached_bytes = fix_stored_bytes(); } return cached_bytes; } void ExecedProcessCacher::read_stored_cached_bytes() { stored_cached_bytes_ = get_stored_bytes_from_cache(); } void ExecedProcessCacher::update_stored_bytes() { // FIXME(rbalint) There is a slight chance for two parallel builds updating the size at the // same time making the file content inaccurate. const std::string size_file = cache_dir_ + "/" + kCacheSizeFile; const off_t new_size = this_runs_cached_bytes_ + stored_cached_bytes_; if (file_overwrite_printf(size_file, "%ld\n", new_size) < 0) { fb_error("writing cache size file failed"); exit(EXIT_FAILURE); } } off_t ExecedProcessCacher::fix_stored_bytes() const { // FIXME(rbalint) There is a slight chance for two parallel builds updating the size at the // same time making the file content inaccurate. const std::string size_file = cache_dir_ + "/" + kCacheSizeFile; off_t starting_cached_bytes = obj_cache->gc_collect_total_objects_size() + blob_cache->gc_collect_total_blobs_size() - this_runs_cached_bytes_; if (file_overwrite_printf(size_file, "%ld\n", starting_cached_bytes) < 0) { fb_error("writing cache size file failed"); exit(EXIT_FAILURE); } return starting_cached_bytes; } bool ExecedProcessCacher::is_gc_needed() const { return (get_stored_bytes_from_cache() + this_runs_cached_bytes_) > max_cache_size; } void ExecedProcessCacher::gc() { gc_runs_++; /* Remove unusable entries first. */ tsl::hopscotch_set referenced_blobs {}; off_t cache_bytes = 0, debug_bytes = 0, unexpected_file_bytes = 0; obj_cache->gc(&referenced_blobs, &cache_bytes, &debug_bytes, &unexpected_file_bytes); blob_cache->gc(referenced_blobs, &cache_bytes, &debug_bytes, &unexpected_file_bytes); if (unexpected_file_bytes > 0) { fb_error("There are " + d(unexpected_file_bytes) + " bytes in the cache stored in files " "with unexpected name."); } stored_cached_bytes_ = cache_bytes + debug_bytes - this_runs_cached_bytes_; if (FB_DEBUGGING(FB_DEBUG_CACHING)) { if (cache_bytes + debug_bytes != this_runs_cached_bytes_ + get_stored_bytes_from_cache()) { FB_DEBUG(FB_DEBUG_CACHING, "A parallel firebuild process modified the cache or the stored " "cache size was wrong. Adjusting the stored cache size."); } } /* Check if the cache size is within limits. */ if (stored_cached_bytes_ + this_runs_cached_bytes_ > max_cache_size) { FB_DEBUG(FB_DEBUG_CACHING, "Cache size (" + d(stored_cached_bytes_ + this_runs_cached_bytes_) + ") " + "is above " + d(max_cache_size) + " bytes limit, removing older entries"); /** Target for this_runs_cached_bytes_ to end up with a cache 20% below its size limit. */ const off_t target_this_runs_cached_bytes = (max_cache_size * 0.8) - stored_cached_bytes_; std::vector obj_ts_sizes = obj_cache->gc_collect_sorted_obj_timestamp_sizes(); int round = 0; while (this_runs_cached_bytes_ > target_this_runs_cached_bytes) { /* Set kept_ratio to to keep ~80% of the objs to keep ~80% of targeted cache size * and target lower kept ratio in each round if the target 80% is not reached. */ const double kept_ratio = (max_cache_size * (0.8 - round * 0.05)) / (stored_cached_bytes_ + this_runs_cached_bytes_); if (kept_ratio <= 0.0) { break; } off_t keep_objects_count = obj_ts_sizes.size() * kept_ratio; FB_DEBUG(FB_DEBUG_CACHING, "Removing " + d(obj_ts_sizes.size() - keep_objects_count) + " " + "cache objects out of " + d(obj_ts_sizes.size())); for (size_t i = keep_objects_count; i < obj_ts_sizes.size(); i++) { const char* name = obj_ts_sizes[i].obj.c_str(); if (unlink(name) != 0) { fb_error(name); fb_perror("unlink"); } else { update_cached_bytes(-obj_ts_sizes[i].size); } } obj_ts_sizes.resize(keep_objects_count); /* Remove unreferenced blobs, too. */ referenced_blobs.clear(); /* Not adjusting the stored cache size this time. */ cache_bytes = debug_bytes = unexpected_file_bytes = 0; obj_cache->gc(&referenced_blobs, &cache_bytes, &debug_bytes, &unexpected_file_bytes); blob_cache->gc(referenced_blobs, &cache_bytes, &debug_bytes, &unexpected_file_bytes); round++; } } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/execed_process_cacher.h000066400000000000000000000140651447164520700227130ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_EXECED_PROCESS_CACHER_H_ #define FIREBUILD_EXECED_PROCESS_CACHER_H_ #include #include #include #include #include #include "firebuild/blob_cache.h" #include "firebuild/obj_cache.h" #include "firebuild/execed_process.h" #include "firebuild/file_name.h" #include "firebuild/hash.h" #include "firebuild/fbbfp.h" #include "firebuild/fbbstore.h" namespace firebuild { enum stats_type { FB_SHOW_STATS_CURRENT, FB_SHOW_STATS_STORED, }; class ExecedProcessCacher { public: /** * One object is responsible for handling the fingerprinting and caching * of multiple ExecedProcesses which potentially come from / go to the * same cache. */ static void init(const libconfig::Config* cfg); static unsigned int cache_format() {return cache_format_;} /** * Compute the fingerprint, store it keyed by the process in fingerprints_. * Also store fingerprint_msgs_ if debugging is enabled. */ bool fingerprint(const ExecedProcess *proc); void erase_fingerprint(const ExecedProcess *proc); void store(ExecedProcess *proc); /** * Look up the cache for an entry describing what this process did the * last time. * * This means fetching all the entries corresponding to the process's * fingerprint, and finding the one matching the file system. * * Returns a new object, to be deleted by the caller, if exactly one * match was found. */ const FBBSTORE_Serialized_process_inputs_outputs *find_shortcut(ExecedProcess *proc, uint8_t **inouts_buf, size_t *inouts_buf_len, Subkey* subkey_out); bool apply_shortcut(ExecedProcess *proc, const FBBSTORE_Serialized_process_inputs_outputs *inouts, std::vector *fds_appended_to); bool shortcut(ExecedProcess *proc, std::vector *fds_appended_to); void not_shortcutting() {if (!no_fetch_) not_shortcutting_++;} /** Add stored hit statistics and cache size to current run's counters. */ void add_stored_stats(); void reset_stored_stats(); void set_self_cpu_time_ms(unsigned int time_ms) { self_cpu_time_ms_ = time_ms; } void print_stats(stats_type what); void update_stored_stats(); /** Get bytes stored in the cache reading cachedir/size file. */ off_t get_stored_bytes_from_cache() const; void read_stored_cached_bytes(); /** Store number of bytes cached to cachedir/size file. */ void update_stored_bytes(); /** * Fix number of bytes cached in cachedir/size file and return fixed value. */ off_t fix_stored_bytes() const; /** Register cache size change occurred in the current run. */ void update_cached_bytes(off_t bytes); /* A garbage collection run is needed, e.g. because the cache is too big. */ bool is_gc_needed() const; void gc(); /** * Checks if the object cache entry can be used for shortcutting, i.e. all the referenced * blobs are present in the blob cache and all the referenced system files on the system * match the process inputs. * @param[in] entry_buf object cache entry as stored * @param[out] referenced_blobs if the entry is usable all the referenced blobs are added to this * set */ bool is_entry_usable(uint8_t* entry_buf, tsl::hopscotch_set* referenced_blobs); private: ExecedProcessCacher(bool no_store, bool no_fetch, const std::string& cache_dir, const libconfig::Config* cfg); /** * Helper for fingerprint() to decide which env vars matter */ bool env_fingerprintable(const std::string& name_and_value) const; bool no_store_; bool no_fetch_; tsl::hopscotch_set envs_skip_; unsigned int shortcut_attempts_ {0}; unsigned int shortcut_hits_ {0}; unsigned int not_shortcutting_ {0}; int64_t self_cpu_time_ms_ {0}; int64_t cache_saved_cpu_time_ms_ {0}; /** * Number of bytes added to (or freed from) the cache in the current run. * It can be negative in case of a garbage collection run. * "This run" means the lifetime for the firebuild process including potentially * running a build command and running gc(), including potentially processing the cache * multiple times in ExecedProcessCacher::gc(). */ off_t this_runs_cached_bytes_ {0}; /** Number of bytes in the cache as stored in the cachedir/size file. */ off_t stored_cached_bytes_ {0}; unsigned int gc_runs_ {0}; /** The hashed fingerprint of configured ignore locations. */ Hash ignore_locations_hash_; /* The hashed fingerprint of the processes handled by this cacher. */ tsl::hopscotch_map fingerprints_; /* The entire fingerprint of the processes handled by this cacher, for debugging * purposes, only if debugging is enabled. In serialized FBBFP format. */ tsl::hopscotch_map> fingerprint_msgs_; static unsigned int cache_format_; std::string cache_dir_; DISALLOW_COPY_AND_ASSIGN(ExecedProcessCacher); }; /* singleton */ extern ExecedProcessCacher* execed_process_cacher; } /* namespace firebuild */ #endif // FIREBUILD_EXECED_PROCESS_CACHER_H_ firebuild-0.8.2/src/firebuild/execed_process_env.cc000066400000000000000000000034051447164520700224100ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include "firebuild/execed_process_env.h" #include "firebuild/debug.h" namespace firebuild { ExecedProcessEnv::ExecedProcessEnv(std::vector>* fds, LaunchType launch_type) : argv_(), launch_type_(launch_type), type_flags_(), fds_(fds) { } void ExecedProcessEnv::set_sh_c_command(const std::string &cmd) { argv_.push_back("sh"); argv_.push_back("-c"); argv_.push_back("--"); argv_.push_back(cmd); } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const ExecedProcessEnv& env, const int level) { (void)level; /* unused */ return d(env.argv()); } std::string d(const ExecedProcessEnv *env, const int level) { if (env) { return d(*env, level); } else { return "{ExecedProcessEnv NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/execed_process_env.h000066400000000000000000000054551447164520700222610ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_EXECED_PROCESS_ENV_H_ #define FIREBUILD_EXECED_PROCESS_ENV_H_ #include #include #include #include "firebuild/file_fd.h" namespace firebuild { typedef enum { LAUNCH_TYPE_SYSTEM, LAUNCH_TYPE_POPEN, LAUNCH_TYPE_POSIX_SPAWN, LAUNCH_TYPE_OTHER } LaunchType; /** * A process' inherited environment, command line parameters and file descriptors, * file actions to be executed on startup (for posix_spawn'ed children), * (and later perhaps the environment variables too). */ class ExecedProcessEnv { public: ExecedProcessEnv(std::vector>* fds, LaunchType launch_type); ~ExecedProcessEnv() {delete fds_;} std::vector& argv() {return argv_;} const std::vector& argv() const {return argv_;} void set_argv(const std::vector& argv) {argv_ = argv;} std::vector>* pop_fds() { std::vector>* ret = fds_; fds_ = nullptr; return ret; } LaunchType launch_type() const {return launch_type_;} void set_type_flags(int type_flags) {type_flags_ = type_flags;} int type_flags() const {return type_flags_;} void set_sh_c_command(const std::string&); private: std::vector argv_; /** Whether it's launched via system() or popen() or other */ LaunchType launch_type_; /** popen(command, type)'s type encoded as O_WRONLY | O_RDONLY | O_CLOEXEC flags */ int type_flags_; /** File descriptor states intherited from parent */ std::vector>* fds_; // TODO(egmont) add envp ? DISALLOW_COPY_AND_ASSIGN(ExecedProcessEnv); }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const ExecedProcessEnv& env, const int level = 0); std::string d(const ExecedProcessEnv *env, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_EXECED_PROCESS_ENV_H_ firebuild-0.8.2/src/firebuild/fbbfp.def000066400000000000000000000111551447164520700177760ustar00rootroot00000000000000# Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # This is the definition of the FBB format used for fingerprinting a # process. # This is a Python dictionary, to be read and processed by "generate_fbb". { "types_with_custom_debugger": ["mode_t", "firebuild::fd_type", "XXH128_hash_t"], "extra_c": """ /* Debugger method for mode_t fields. */ static void fbbfp_debug_mode_t(FILE *f, mode_t mode) { fprintf(f, "\\""); debug_mode_t(f, mode); fprintf(f, "\\""); } /* Debugger method for firebuild::fd_type fields. */ static void fbbfp_debug_firebuild__fd_type(FILE *f, firebuild::fd_type type) { fprintf(f, "\\"%s\\"", fd_type_to_string(type)); } /* Debugger method for XXH128_hash_t fields. */ static void fbbfp_debug_XXH128_hash_t(FILE *f, XXH128_hash_t value) { firebuild::Hash hash(value); char ascii[firebuild::Hash::kAsciiLength + 1]; hash.to_ascii(ascii); fprintf(f, "\\"%s\\"", ascii); } """, "extra_h": """ #include #include "firebuild/file_fd.h" #include "firebuild/hash.h" #include "common/debug_sysflags.h" """, "tags": [ ("file", [ # file path, absolute or relative (REQUIRED, STRING, "path"), # checksum (binary) of the file content, empty if file is not found (REQUIRED, "XXH128_hash_t", "hash"), # TODO add alternate hash values generated after preprocessing the file # with programs keeping the semantic content (e.g. removing white spaces) #(REQUIRED, "XXH128_hash_t", "alt_hash"), # last modification time - FIXME in what unit? #(OPTIONAL, "long", "mtime"), # file size, length in case of stdio #(OPTIONAL, "size_t", "size"), # TODO refine mode (OPTIONAL, "mode_t", "mode"), # The reason why the file could not be opened. #(OPTIONAL, "int", "error_no"), ]), # Represents an inherited open file description, along with the fds ("ofd", [ # Type (REQUIRED, "firebuild::fd_type", "type"), # Client-side fds pointing to the same open file description, in # ascending order (ARRAY, "int", "fds"), # FIXME fcntl flags? # FIXME seek positions? ]), # Properties of a process that are known when it starts up, and are # expected to be the same across identical launches. These take a key # part in deciding whether it can be shortcutted. This includes command # line flags, selected environment variables and such. Things that are # expected to change across runs, e.g. pid, ppid, $FB_SOCKET etc. are # not included here. Things that only become known while the process is # running, such as the files it reads, aren't included either. ("process_fingerprint", [ (REQUIRED, "int", "kFingerprintVersion"), (ARRAY, STRING, "ignore_locations"), # The initial umask (REQUIRED, "mode_t", "umask"), # The initial working directory (REQUIRED, STRING, "wd"), # Program file (REQUIRED, FBB, "executable"), # tag "file" # Pathname used to execute the program in absolute and canonical form (REQUIRED, FBB, "executed_path"), # tag "file" # Original pathname used to execute the program (REQUIRED, STRING, "original_executed_path"), # tag "file" # Command parameters, starting with name of the command (ARRAY, STRING, "args"), (OPTIONAL, "XXH128_hash_t", "param_file_hash"), # Environment variables, filtered (to exclude e.g. $FB_SOCKET) # and sorted to a deterministic order (ARRAY, STRING, "env"), # Libraries loaded upon startup (ARRAY, FBB, "libs"), # tag "file" # An entry for each inherited open file description, in ascending # order of their lowest fd (ARRAY, FBB, "ofds"), # tag "ofd" ]), ] } firebuild-0.8.2/src/firebuild/fbbstore.def000066400000000000000000000120631447164520700205240ustar00rootroot00000000000000# Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # This is the definition of the FBB format used for storing the cache # entries. # This is a Python dictionary, to be read and processed by "generate_fbb". { "types_with_custom_debugger": ["mode_t", "firebuild::FileType", "XXH128_hash_t"], "extra_c": """ /* Debugger method for mode_t fields. */ static void fbbstore_debug_mode_t(FILE *f, mode_t mode) { fprintf(f, "\\""); debug_mode_t(f, mode); fprintf(f, "\\""); } /* Debugger method for firebuild::FileType fields. */ static void fbbstore_debug_firebuild__FileType(FILE *f, firebuild::FileType type) { fprintf(f, "\\"%s\\"", file_type_to_string(type)); } /* Debugger method for XXH128_hash_t fields. */ static void fbbstore_debug_XXH128_hash_t(FILE *f, XXH128_hash_t value) { firebuild::Hash hash(value); char ascii[firebuild::Hash::kAsciiLength + 1]; hash.to_ascii(ascii); fprintf(f, "\\"%s\\"", ascii); } """, "extra_h": """ #include #include "firebuild/hash.h" #include "firebuild/file_info.h" #include "common/debug_sysflags.h" """, "tags": [ ("file", [ # file path, absolute or relative (REQUIRED, STRING, "path"), # file type, e.g. ISREG, NOTEXIST_OR_ISREG, ISDIR etc. (REQUIRED, "firebuild::FileType", "type"), # file size (OPTIONAL, "size_t", "size"), # checksum (binary) of the file content, if relevant and known (OPTIONAL, "XXH128_hash_t", "hash"), # TODO add alternate hash values generated after preprocessing the file # with programs keeping the semantic content (e.g. removing white spaces) #(OPTIONAL, "XXH128_hash_t", "alt_hash"), # last modification time - FIXME in what unit? #(OPTIONAL, "long", "mtime"), # the known bits of the mode, if any (OPTIONAL, "mode_t", "mode"), # which bits of the mode are known, if any (OPTIONAL, "mode_t", "mode_mask"), # The reason why the file could not be opened. #(OPTIONAL, "int", "error_no"), ]), ("dir", [ (REQUIRED, STRING, "path"), (ARRAY, STRING, "entry"), ]), ("append_to_fd", [ # fd at the time the process started, the lowest one if dup()'ed to # multiple fds (REQUIRED, "int", "fd"), # Checksum (binary) of the written data (REQUIRED, "XXH128_hash_t", "hash"), ]), # Things that are read from the external world by the process while # it's running, but aren't known in advance that the process will need # them. In order to shortcut a process, there has to be a cached entry # that matches the current world. ("process_inputs", [ # Files that are opened for reading, with various results. (ARRAY, FBB, "path"), # tag "file" (ARRAY, STRING, "path_notexist"), # TODO: Directories that are opendir'ed, even if opendir failed. # FIXME: need to fingerprint the entire directory listing?? #(ARRAY, FBB, "path_isdir_listed"), # tag "dir" # TODO: readlink and friends... ]), # Things that are modified in the external world by the process while # it's running. ("process_outputs", [ # Files that are written to (or removed), only if opening them for # writing succeeded. (ARRAY, FBB, "path_isreg"), # tag "file" # Directories created (ARRAY, FBB, "path_isdir"), # tag "file" (ARRAY, STRING, "path_notexist"), # Maybe special handling of files that are appended to? # TODO: # unlink, rmdir # link, symlink # chown, chmod # etc. # Data appended to inherited files (currently pipes only) (ARRAY, FBB, "append_to_fd"), # tag "append_to_fd" (REQUIRED, "int", "exit_status"), ]), ("process_inputs_outputs", [ (REQUIRED, FBB, "inputs"), # tag "process_inputs" (REQUIRED, FBB, "outputs"), # tag "process_outputs" # Aggregate CPU time used by this process and all children in milliseconds. # Not set when deterministic cache debugging is enabled. (OPTIONAL, "int", "cpu_time_ms"), ]), ] } firebuild-0.8.2/src/firebuild/file_fd.cc000066400000000000000000000062261447164520700201410ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/file_fd.h" #include #include "firebuild/execed_process.h" #include "firebuild/pipe.h" namespace firebuild { int FileOFD::id_counter_ = 0; FileOFD::FileOFD(fd_type type, const FileName *filename, int flags, Process *opened_by) : id_(id_counter_++), type_(type), filename_(filename), flags_(flags & ~FILE_CREATION_FLAGS), opened_by_(opened_by) { if (filename_ && is_write(flags_)) { filename_->open_for_writing(opened_by_->exec_point()); } } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileOFD& fofd, const int level) { (void)level; /* unused */ std::string ret = "{FileOFD #" + d(fofd.id()); ret += " type=" + std::string(fd_type_to_string(fofd.type())) + " "; // FIXME replace this with printing all the flags switch (fofd.flags() & O_ACCMODE) { case O_RDONLY: ret += "r"; break; case O_WRONLY: ret += "w"; break; case O_RDWR: ret += "rw"; break; default: ret += "unknown_mode"; } if (fofd.filename()) { ret += " " + d(fofd.filename(), level + 1); } ret += "}"; return ret; } std::string d(const FileOFD *fofd, const int level) { if (fofd) { return d(*fofd, level); } else { return "{FileOFD NULL}"; } } std::string d(const FileFD& ffd, const int level) { std::string ret = "{FileFD fd=" + d(ffd.fd()) + " " + d(ffd.ofd(), level); if (ffd.pipe()) { ret += " " + d(ffd.pipe().get(), level + 1); ret += " close_on_popen=" + d(ffd.close_on_popen()); } ret += " cloexec=" + d(ffd.cloexec()); ret += "}"; return ret; } std::string d(const FileFD *ffd, const int level) { if (ffd) { return d(*ffd, level); } else { return "{FileFD NULL}"; } } const char *fd_type_to_string(fd_type type) { switch (type) { case FD_UNINITIALIZED: return "FD_UNINITIALIZED"; case FD_IGNORED: return "FD_IGNORED"; case FD_FILE: return "FD_FILE"; case FD_PIPE_IN: return "FD_PIPE_IN"; case FD_PIPE_OUT: return "FD_PIPE_OUT"; case FD_SPECIAL: return "FD_SPECIAL"; case FD_SCM_RIGHTS: return "FD_SCM_RIGHTS"; default: assert(0 && "unknown type"); return "UNKNOWN"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/file_fd.h000066400000000000000000000202401447164520700177730ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FILE_FD_H_ #define FIREBUILD_FILE_FD_H_ #include #include #include #include #include "firebuild/file_name.h" #include "firebuild/cxx_lang_utils.h" #include "firebuild/pipe.h" namespace firebuild { enum fd_type : char { FD_UNINITIALIZED, /* only used intermittently during object construction */ FD_IGNORED, /* a path that's on ignore_list, e.g. /dev/null */ FD_FILE, /* regular file */ FD_PIPE_IN, /* the incoming endpoint of a pipe(), or the toplevel stdin */ FD_PIPE_OUT, /* the outgoing endpoint of a pipe(), or the toplevel stdout, stderr */ FD_SPECIAL, /* backed by memory, e.g. memfd, eventfd etc. */ FD_SCM_RIGHTS, /* received by a recv[m]msg() with SCM_RIGHTS, we don't know its type */ }; class Process; class Pipe; /* We don't track these "file creation flags" because fcntl(F_SETFL) ignores them and fcntl(F_GETFL) * doesn't report them back. The list is taken from the open(2) manpage. * Also O_CLOEXEC is tracked in FileFD where it belongs to, rather than in FileOFD. * O_TMPFILE is not listed here, because it is multiple bits and also does not create a named file. * */ #define FILE_CREATION_FLAGS (O_CLOEXEC | O_CREAT | O_DIRECTORY | O_EXCL | O_NOCTTY | O_NOFOLLOW | \ O_TRUNC) /** * Represents an "open file description" ("ofd") of the intercepted processes, as per the term's * definition in POSIX, in the open(2) manual, and in #919. That is, these are the bits that are * shared across a dup() or fork(). * * For outgoing pipes, as per #689, Firebuild's intercepting mechanism changes the behavior: across * an exec() it undups what are supposed to be dups of each other. Here we model this altered * behavior, that is, new OFDs are created upon reopening a pipe. * * Note: As with Unix pipe()s, the read and the write endpoints are different OFDs. */ class FileOFD { public: FileOFD(fd_type type, const FileName *filename, int flags, Process *opened_by); ~FileOFD() { if (filename_ && is_write(flags_)) { filename_->close_for_writing(); } } int id() const {return id_;} fd_type type() const {return type_;} const FileName *filename() const {return filename_;} void set_flags(int flags) { flags_ = flags & ~(O_ACCMODE | FILE_CREATION_FLAGS); } int flags() const {return flags_;} Process *opened_by() const {return opened_by_;} private: /* Unique FileOFD id, for debugging. */ int id_; /* Type. */ fd_type type_; /** If the file was opened by name during firebuild's supervision. */ const FileName *filename_; /** The open() flags except for O_CLOEXEC, a.k.a. the fcntl(F_GETFL/F_SETFL) flags. */ int flags_; /** Process that opened this file by name. * Remains the same (doesn't get updated to the current process) at dup2() or alike, * also including the case when an outgoing pipe is reopened on an exec(). * NULL if the topmost intercepted process already inherited it from the supervisor. */ Process *opened_by_; static int id_counter_; DISALLOW_COPY_AND_ASSIGN(FileOFD); }; /** * Represents a "file descriptor" ("fd") of the intercepted process, as per the term's definition in * POSIX, in the open(2) manual, and in #919. That is, these are the bits that are _not_ shared * across a dup() or fork(), plus a pointer to the shared ("ofd") bits. */ class FileFD { public: /** Constructor for fds of a certain type. */ FileFD(int fd, int flags, fd_type type, Process *opened_by) : fd_(fd), ofd_(std::make_shared(type, nullptr, flags, opened_by)), pipe_(), cloexec_(flags & O_CLOEXEC) { assert(fd >= 0); } /** Constructor for fds backed by a pipe including ones created by popen(). */ FileFD(int fd, int flags, std::shared_ptr pipe, Process *opened_by, bool close_on_popen = false) : fd_(fd), ofd_(std::make_shared(is_write(flags) ? FD_PIPE_OUT : FD_PIPE_IN, nullptr, flags, opened_by)), pipe_(pipe), cloexec_(flags & O_CLOEXEC), close_on_popen_(close_on_popen) { assert(fd >= 0); } /** Constructor for fds created from other fds through dup() or exec() */ FileFD(int fd, std::shared_ptr ffd_src, bool cloexec) : fd_(fd), ofd_(ffd_src->ofd_), pipe_(ffd_src->pipe_), cloexec_(cloexec) { assert(fd >= 0); if (pipe_) { pipe_->handle_dup(ffd_src.get(), this); } } /** Constructor for fds obtained through opening files. */ FileFD(const FileName* filename, int fd, int flags, Process *opened_by) : fd_(fd), ofd_(std::make_shared(filename->is_in_ignore_location() ? FD_IGNORED : FD_FILE, filename, flags, opened_by)), pipe_(), cloexec_(flags & O_CLOEXEC) { assert(fd >= 0); } FileFD(const FileFD& other) : fd_(other.fd_), ofd_(other.ofd_), pipe_(other.pipe_), cloexec_(other.cloexec_), close_on_popen_(other.close_on_popen_) { } FileFD& operator= (const FileFD& other) { fd_ = other.fd_; ofd_ = other.ofd_; pipe_ = other.pipe_; cloexec_ = other.cloexec_; close_on_popen_ = other.close_on_popen_; return *this; } /* Getters/setters, some are just convenience proxies to ofd_'s corresponding method. */ int fd() const {return fd_;} std::shared_ptr ofd() const {return ofd_;} fd_type type() const {return ofd_->type();} const FileName* filename() const {return ofd_->filename();} /* Note: This method does NOT change the O_CLOEXEC flag, use set_cloexec() for that. */ void set_flags(int flags) {ofd_->set_flags(flags);} /* Note: This method does NOT report the O_CLOEXEC flag, use cloexec() for that. */ int flags() const {return ofd_->flags();} Process *opened_by() {return ofd_->opened_by();} void set_cloexec(bool cloexec) {cloexec_ = cloexec;} bool cloexec() const {return cloexec_;} bool close_on_popen() const {return close_on_popen_;} void set_close_on_popen(bool c) {close_on_popen_ = c;} void set_pipe(std::shared_ptr pipe) { if (pipe_) { pipe_->handle_close(this); } pipe_ = pipe; } std::shared_ptr pipe() {return pipe_;} const std::shared_ptr pipe() const {return pipe_;} /* Like kcmp(KCMP_FILE), checks if the two objects point to the same open file description. * Returns -1/0/1 as the usual cmp functions or <=>. */ int fdcmp(const FileFD& other) const { // FIXME Switch to c++20 spaceship: return ofd_ <=> other.ofd_; return ofd_ < other.ofd_ ? -1 : ofd_ == other.ofd_ ? 0 : 1; } private: int fd_; std::shared_ptr ofd_; /** If it's a pipe, i.e. type is FD_PIPE_[IN|OUT]. Except for the toplevel stdin where type is ** FD_PIPE_IN but pipe_ is NULL. */ // FIXME Should be moved to FileOFD. Requires nontrivial work around handle_close(), see #939. std::shared_ptr pipe_; bool cloexec_; bool close_on_popen_ {false}; }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileOFD& fofd, const int level = 0); std::string d(const FileOFD *fofd, const int level = 0); std::string d(const FileFD& ffd, const int level = 0); std::string d(const FileFD *ffd, const int level = 0); const char *fd_type_to_string(fd_type type); } /* namespace firebuild */ #endif // FIREBUILD_FILE_FD_H_ firebuild-0.8.2/src/firebuild/file_info.cc000066400000000000000000000045121447164520700204770ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/file_info.h" #include "common/firebuild_common.h" #include "firebuild/debug.h" #include "firebuild/hash.h" #include "firebuild/hash_cache.h" namespace firebuild { /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileInfo& fi, const int level) { (void)level; /* unused */ char mode_str[8], mode_mask_str[8]; snprintf(mode_str, sizeof(mode_str), "0%03o", fi.mode()); snprintf(mode_mask_str, sizeof(mode_mask_str), "0%03o", fi.mode_mask()); return std::string("{FileInfo type=") + file_type_to_string(fi.type()) + (fi.size_known() ? ", size=" + d(fi.size()) : "") + (fi.hash_known() ? ", hash=" + d(fi.hash()) : "") + (fi.mode_mask() != 0 ? ", mode=" + std::string(mode_str) + ", mode_mask=" + std::string(mode_mask_str) : "") + "}"; } std::string d(const FileInfo *fi, const int level) { if (fi) { return d(*fi, level); } else { return "{FileInfo NULL}"; } } const char *file_type_to_string(FileType type) { switch (type) { case DONTKNOW: return "dontknow"; case EXIST: return "exist"; case NOTEXIST: return "notexist"; case NOTEXIST_OR_ISREG: return "notexist_or_isreg"; case ISREG: return "isreg"; case ISDIR: return "isdir"; default: assert(0 && "unknown type"); return "UNKNOWN"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/file_info.h000066400000000000000000000160431447164520700203430ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FILE_INFO_H_ #define FIREBUILD_FILE_INFO_H_ #include #include "firebuild/hash.h" namespace firebuild { typedef enum { /** No information about the filesystem entry (its presence/absence, type, etc.). */ DONTKNOW, /** We know that the filesystem entry exists, but don't know if it's a regular file or a * directory. This happens at a successful access(F_OK). */ EXIST, /** We know that the filesystem entry doesn't exist. We might know it by a failed access(F_OK) * or stat(). We also might know it about the initial state of the filesystem entry, if later an * open(O_CREAT|O_WRONLY|O_EXCL) or mkdir() succeeds. */ NOTEXIST, /** We know that the filesystem entry either does not exist, or is a regular file, but we don't * know which. We might know it about the initial state of a file, if later a creat() a.k.a. * open(O_CREAT|O_WRONLY|O_TRUNC) succeeds, or an open(O_CREAT|O_WRONLY) succeeds and results in * a zero length file. In the latter case, size is set to 0 in the corresponding FileInfo. */ NOTEXIST_OR_ISREG, /** We know that the filesystem entry is a regular fie. */ ISREG, /** We know that the filesystem entry is a directory. */ ISDIR, FILE_TYPE_MAX = ISDIR } FileType; /** * FileInfo describes the (potentially partial) information that we know about a certain file, as it * looked like / looks like / will look like in a certain point in time. It's up to the user of this * structure to decide which point in time they refer to. */ class FileInfo { public: explicit FileInfo(FileType type = DONTKNOW, off_t size = -1, const Hash *hash = nullptr) : type_(type), size_(size), hash_known_(hash != nullptr), hash_(hash != nullptr ? *hash : Hash()) { assert(type == ISREG || type == ISDIR || hash == nullptr); } FileType type() const {return type_;} void set_type(FileType type) {type_ = type;} bool size_known() const {return size_ >= 0;} off_t size() const {return size_;} void set_size(off_t size) { size_ = size; } bool hash_known() const {return hash_known_;} const Hash& hash() const {return hash_;} void set_hash(const Hash& hash) { hash_ = hash; hash_known_ = true; } void set_hash(const Hash *hash) { if (hash) { hash_ = *hash; hash_known_ = true; } else { hash_ = Hash(); hash_known_ = false; } } mode_t mode() const {return mode_;} mode_t mode_mask() const {return mode_mask_;} /* Set or clear the file mode bits where enabled by the mask, leave the other bits unchanged. */ void set_mode_bits(mode_t mode, mode_t mask) { mode_ &= ~mask; mode_ |= (mode & mask); mode_mask_ |= mask; } /* Misc */ static int file_type_to_int(const FileType t) { switch (t) { case DONTKNOW: return DONTKNOW; case EXIST: return EXIST; case NOTEXIST: return NOTEXIST; case NOTEXIST_OR_ISREG: return NOTEXIST_OR_ISREG; case ISREG: return ISREG; case ISDIR: return ISDIR; default: abort(); } } static FileType int_to_file_type(const int t) { switch (t) { case DONTKNOW: return DONTKNOW; case EXIST: return EXIST; case NOTEXIST: return NOTEXIST; case NOTEXIST_OR_ISREG: return NOTEXIST_OR_ISREG; case ISREG: return ISREG; case ISDIR: return ISDIR; default: abort(); } } private: /** File type. * * If DONTKNOW or NOTEXIST then the remaining fields are meaningless and unset. * * If NOTEXIST_OR_ISREG then the remaining fields refer to the state of the file in case it is * actually a regular file (ISREG) rather than missing (NOTEXIST). */ FileType type_; /** The size, if known. Only if type_ is ISREG or NOTEXIST_OR_ISREG. In these cases, if the * checksum is known then the size is also known. If the size is not known or is irrelevant * (type_ isn't one of these) then -1. * * (If the type is NOTEXIST_OR_ISREG and the size is known, the size is necessarily 0. This is * our knowledge about the initial / prior state of a file if open(O_CREAT|O_WRONLY) results in * an empty file.) */ off_t size_; /** Whether the checksum is known. Only if type_ is ISREG, NOTEXIST_OR_ISREG or ISDIR. * For regular files, knowing the checksum implies we know the size, too. * * (Note: currently the type cannot actually be NOTEXIST_OR_ISREG if this field is set. That's * because if the type is NOTEXIST_OR_ISREG then the size, if known, is necessarily 0, and we * don't fill in the checksum. As per the FIXME below, this might change in the future.) */ // // FIXME(egmont) Do we want to have special treatment for zero-length files, // either always set the hash (copy from a global variable), or never set it? bool hash_known_ : 1; /** The checksum, if known. Only if type_ is ISREG, NOTEXIST_OR_ISREG, or ISDIR. * For directories, it's the checksum of its listing. * * (Note: currently the type cannot actually be NOTEXIST_OR_ISREG if this field is set. That's * because if the type is NOTEXIST_OR_ISREG then the size, if known, is necessarily 0, and we * don't fill in the checksum. As per the FIXME below, this might change in the future.) */ // // FIXME(egmont) Do we want to have special treatment for zero-length files, // either always set the hash (copy from a global variable), or never set it? Hash hash_; /** The mode of the file, i.e. the 12 bits: setuid, setgid, sticky, owner-readable, etc. * If the corresponding bit in mode_mask_ is set then the mode is known to * have the given property (set or unset) as contained in this mode_ here. * If the corresponding bit in mode_mask_ is unset then that bit here is zero (unused). */ mode_t mode_ {0}; /** Which of the bits in mode_ are known. */ mode_t mode_mask_ {0}; friend bool operator==(const FileInfo& lhs, const FileInfo& rhs) = default; }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileInfo& fi, const int level = 0); std::string d(const FileInfo *fi, const int level = 0); const char *file_type_to_string(FileType type); } /* namespace firebuild */ #endif // FIREBUILD_FILE_INFO_H_ firebuild-0.8.2/src/firebuild/file_name.cc000066400000000000000000000126031447164520700204640ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include "firebuild/file_name.h" #include "firebuild/execed_process.h" #include "common/firebuild_common.h" namespace firebuild { std::unordered_set* FileName::db_; tsl::hopscotch_map* FileName::hash_db_; tsl::hopscotch_map>* FileName::write_ofds_db_; tsl::hopscotch_map* FileName::generation_db_; const FileName* FileName::default_tmpdir; FileName::DbInitializer::DbInitializer() { db_ = new std::unordered_set(); hash_db_ = new tsl::hopscotch_map(); write_ofds_db_ = new tsl::hopscotch_map>(); generation_db_ = new tsl::hopscotch_map(); } bool FileName::isDbEmpty() { return !db_ || db_->empty(); } FileName::DbInitializer FileName::db_initializer_; void FileName::open_for_writing(ExecedProcess* proc) const { TRACKX(FB_DEBUG_FS, 1, 0, FileName, this, "proc=%s", D(proc)); if (is_in_ignore_location()) { /* Ignored locations can be ignored here, too. */ return; } assert(proc); auto it = write_ofds_db_->find(this); if (it != write_ofds_db_->end()) { auto& pair = it.value(); assert(pair.first > 0); pair.first++; if (proc != pair.second) { /* A different process opened the file for writing. */ ExecedProcess* common_ancestor = proc->common_exec_ancestor(pair.second); const ExecedProcess* other_proc = pair.second; if (common_ancestor != proc) { proc->disable_shortcutting_bubble_up_to_excl( common_ancestor, deduplicated_string( "Opened " + this->to_string() + " for writing which file is already opened for writing by [" + d(other_proc->pid()) + "] \"" + other_proc->args_to_short_string() + "\"").c_str()); } if (common_ancestor != pair.second) { pair.second->disable_shortcutting_bubble_up_to_excl( common_ancestor, deduplicated_string( "An other process opened " + this->to_string() + " for writing which file is already opened for writing by [" + d(other_proc->pid()) + "] \"" + other_proc->args_to_short_string() + "\"").c_str()); pair.second = common_ancestor; } } } else { write_ofds_db_->insert({this, {1, proc}}); auto it2 = generation_db_->find(this); if (it2 != generation_db_->end()) { assert(it2->second < UINT32_MAX); it2.value()++; /* Bubble up the generation change */ proc->register_file_usage_update(this, FileUsageUpdate(this)); } else { generation_db_->insert({this, 1}); } } } void FileName::close_for_writing() const { TRACKX(FB_DEBUG_FS, 1, 0, FileName, this, ""); if (is_in_ignore_location()) { /* Ignored locations can be ignored here, too. */ return; } auto it = write_ofds_db_->find(this); assert(it != write_ofds_db_->end()); assert(it->second.first > 0); if (it->second.first > 1) { it.value().first--; } else { write_ofds_db_->erase(it); } } const FileName* FileName::GetParentDir(const char * const name, ssize_t length) { /* name is canonicalized, so just simply strip the last component */ ssize_t slash_pos = length - 1; for (; slash_pos >= 0; slash_pos--) { if (name[slash_pos] == '/') { break; } } /* A path that does not have a '/' in it or "/" itself does not have a parent */ if (slash_pos == -1 || length == 1) { return nullptr; } if (slash_pos == 0) { /* Path is in the "/" dir. */ return Get("/", 1); } else { char* parent_name = reinterpret_cast(alloca(slash_pos + 1)); memcpy(parent_name, name, slash_pos); parent_name[slash_pos] = '\0'; return Get(parent_name, slash_pos); } } bool FileName::is_at_locations(const cstring_view_array* locations) const { return is_path_at_locations(this->name_, this->length_, locations); } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileName& fn, const int level) { (void)level; /* unused */ return d(fn.to_string()); } std::string d(const FileName *fn, const int level) { if (fn) { return d(*fn, level); } else { return "{FileName NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/file_name.h000066400000000000000000000154751447164520700203400ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FILE_NAME_H_ #define FIREBUILD_FILE_NAME_H_ #include #include #include #include #include #include #include #include "common/firebuild_common.h" #include "common/platform.h" #include "firebuild/debug.h" namespace firebuild { class ExecedProcess; typedef uint32_t file_generation_t; struct FileNameHasher; class FileName { public: FileName(const FileName& other) : name_(reinterpret_cast(malloc(other.length_ + 1))), length_(other.length_), in_ignore_location_(other.in_ignore_location_), in_read_only_location_(other.in_read_only_location_) { memcpy(const_cast(name_), other.name_, other.length_ + 1); } const char * c_str() const {return name_;} std::string to_string() const {return std::string(name_);} uint32_t length() const {return length_;} size_t hash() const {return XXH3_64bits(name_, length_);} const XXH128_hash_t& hash_XXH128() const { auto it = hash_db_->find(this); if (it != hash_db_->end()) { return it->second; } else { /* Not found, add a copy to the set. */ return (hash_db_->insert({this, XXH3_128bits(name_, length_)}).first)->second; } } int writers_count() const { /* Files in ignored locations should not even be queried. */ assert(!is_in_ignore_location()); auto it = write_ofds_db_->find(this); if (it != write_ofds_db_->end()) { assert(it->second.first > 0); return it->second.first; } else { return 0; } } void open_for_writing(ExecedProcess* proc) const; void close_for_writing() const; file_generation_t generation() const { auto it = generation_db_->find(this); if (it != generation_db_->end()) { assert(it->second > 0); return it->second; } else { return 0; } } static bool isDbEmpty(); static const FileName* Get(const char * const name, ssize_t length); static const FileName* Get(const std::string& name) { return Get(name.c_str(), name.size()); } /** * Return parent dir or nullptr for "/" */ static const FileName* GetParentDir(const char * const name, ssize_t length); bool is_in_ignore_location() const {return in_ignore_location_;} bool is_in_read_only_location() const {return in_read_only_location_;} std::string without_dirs() const { // TODO(rbalint) use std::string::ends_with when we switch to c++20 const char* last_slash = strrchr(name_, '/'); if (last_slash) { return std::string(last_slash + 1); } else { return to_string(); } } static const FileName* default_tmpdir; private: FileName(const char * const name, size_t length, bool copy_name) : name_(copy_name ? reinterpret_cast(malloc(length + 1)) : name), length_(length), in_ignore_location_(false), in_read_only_location_(false) { if (copy_name) { memcpy(const_cast(name_), name, length); const_cast(name_)[length] = '\0'; } } /** * Checks if a path semantically begins with one of the given sorted subpaths. * * Does string operations only, does not look at the file system. */ bool is_at_locations(const cstring_view_array *locations) const; const char * const name_; const uint32_t length_; const bool in_ignore_location_; const bool in_read_only_location_; static std::unordered_set* db_; static tsl::hopscotch_map* hash_db_; /** Number of FileOFDs open for writing referencing this file. */ static tsl::hopscotch_map>* write_ofds_db_; /** * A generation of the file is when it is kept open by a set of writers. * Whenever all writers close the file and thus the refcount in write_ofds_db_ decreases to zero * the generation is closed, but the generation number stays the same. When the new writer opens * the file a new generation is opened. * A file's generation number is 0 until it is opened for writing for the first time. */ static tsl::hopscotch_map* generation_db_; /* Disable assignment. */ void operator=(const FileName&); /* This, along with the FileName::db_initializer_ definition in file_namedb.cc, * initializes the filename database once at startup. */ class DbInitializer { public: DbInitializer(); }; friend class DbInitializer; static DbInitializer db_initializer_; }; inline bool operator==(const FileName& lhs, const FileName& rhs) { return lhs.length() == rhs.length() && memcmp(lhs.c_str(), rhs.c_str(), lhs.length()) == 0; } struct FileNameHasher { std::size_t operator()(const FileName& s) const noexcept { return s.hash(); } }; /** Helper struct for std::sort */ struct FileNameLess { bool operator()(const FileName* f1, const FileName* f2) const { return strcmp(f1->c_str(), f2->c_str()) < 0; } }; extern cstring_view_array ignore_locations; extern cstring_view_array read_only_locations; inline const FileName* FileName::Get(const char * const name, ssize_t length) { FileName tmp_file_name(name, (length == -1) ? strlen(name) : length, false); #ifdef FB_EXTRA_DEBUG assert(is_canonical(tmp_file_name.name_, tmp_file_name.length_)); #endif auto it = db_->find(tmp_file_name); if (it != db_->end()) { return &*it; } else { *const_cast(&tmp_file_name.in_ignore_location_) = tmp_file_name.is_at_locations(&ignore_locations); *const_cast(&tmp_file_name.in_read_only_location_) = tmp_file_name.is_at_locations(&read_only_locations); /* Not found, add a copy to the set. */ return &*db_->insert(tmp_file_name).first; } } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileName& fn, const int level = 0); std::string d(const FileName *fn, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_FILE_NAME_H_ firebuild-0.8.2/src/firebuild/file_usage.cc000066400000000000000000000177431447164520700206620ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/file_usage.h" #include #include #include #include "common/firebuild_common.h" #include "firebuild/debug.h" #include "firebuild/hash.h" namespace firebuild { std::unordered_set* FileUsage::db_; const FileUsage* FileUsage::no_hash_not_written_states_[FILE_TYPE_MAX + 1]; FileUsage::DbInitializer::DbInitializer() { db_ = new std::unordered_set(); for (int i = 0; i <= FILE_TYPE_MAX; i++) { const FileUsage fu(FileInfo::int_to_file_type(i)); no_hash_not_written_states_[i] = &*db_->insert(fu).first; } } FileUsage::DbInitializer FileUsage::db_initializer_; const FileUsage* FileUsage::Get(const FileUsage& candidate) { auto it = db_->find(candidate); if (it != db_->end()) { return &*it; } else { /* Not found, add a copy to the set. */ return &*db_->insert(candidate).first; } } const FileUsage* FileUsage::merge(const FileUsageUpdate& update, const bool propagated) const { FileUsage tmp = *this; /* Make sure the merged FileUsage is debug-printed upon leaving this method. */ #ifdef FB_EXTRA_DEBUG const FileUsage *fu = &tmp; #endif TRACKX(FB_DEBUG_PROC, 1, 1, FileUsage, fu, "other=%s", D(update)); bool changed = false; if (generation() != update.generation()) { /* Ensured by the caller. */ assert((generation() == 0 && initial_type() == DONTKNOW) || generation() + 1 == update.generation()); tmp.generation_ = update.generation(); changed = true; } if (!written_) { /* Note: this might lazily query the type now. Avoid calling it multiple times. */ FileType update_initial_type; if (!update.get_initial_type(&update_initial_type)) { return nullptr; } switch (initial_type()) { case DONTKNOW: { if (initial_type() != update_initial_type) { tmp.set_initial_type(update_initial_type); changed = true; } if (!initial_size_known() && update.initial_size_known()) { tmp.set_initial_size(update.initial_size()); changed = true; } if (!initial_hash_known() && update.initial_hash_known()) { Hash hash; /* Note: this might lazily compute the hash now. */ if (!update.get_initial_hash(&hash)) { return nullptr; } tmp.set_initial_hash(hash); changed = true; } break; } case EXIST: { if (update_initial_type == NOTEXIST) { return nullptr; } else if (update_initial_type == NOTEXIST_OR_ISREG) { /* We knew from an access() that it existed, now we got to know from an open() that it * either didn't exist or was a regular file. That is: it was a regular file. */ tmp.set_initial_type(ISREG); if (update.initial_size_known()) { assert(update.initial_size() == 0); tmp.set_initial_size(update.initial_size()); } changed = true; } else { /* Copy over the new values */ // FIXME This is copied from the DONTKNOW case, maybe factor out to a helper method. if (initial_type() != update_initial_type) { tmp.set_initial_type(update_initial_type); changed = true; } if (!initial_size_known() && update.initial_size_known()) { tmp.set_initial_size(update.initial_size()); changed = true; } if (!initial_hash_known() && update.initial_hash_known()) { Hash hash; /* Note: this might lazily compute the hash now. */ if (!update.get_initial_hash(&hash)) { return nullptr; } tmp.set_initial_hash(hash); changed = true; } } break; } case NOTEXIST: { if (update_initial_type != DONTKNOW && update_initial_type != NOTEXIST && update_initial_type != NOTEXIST_OR_ISREG) { return nullptr; } break; } case NOTEXIST_OR_ISREG: { /* This initial state, without the written_ bit, is possible intermittently while * shortcutting a process. See #791. */ break; } case ISREG: { if (update_initial_type != DONTKNOW && update_initial_type != EXIST && update_initial_type != NOTEXIST_OR_ISREG && update_initial_type != ISREG) { return nullptr; } if (!initial_size_known() && update.initial_size_known()) { /* Note this might lazily figure out the size now. */ tmp.set_initial_size(update.initial_size()); changed = true; } if (!initial_hash_known() && update.initial_hash_known()) { Hash hash; /* Note: this might lazily compute the hash now. */ if (!update.get_initial_hash(&hash)) { return nullptr; } tmp.set_initial_hash(hash); changed = true; } break; } case ISDIR: { if (update_initial_type != DONTKNOW && update_initial_type != EXIST && update_initial_type != ISDIR) { return nullptr; } if (!initial_hash_known() && update.initial_hash_known()) { Hash hash; /* Note: this might lazily compute the hash now. */ if (!update.get_initial_hash(&hash)) { return nullptr; } tmp.set_initial_hash(hash); changed = true; } break; } } if (update.written()) { tmp.written_ = true; changed = true; } } if (!mode_changed_) { // FIXME this condition could be even more fine-grained to detect if things won't change if (initial_mode() != update.initial_mode() || initial_mode_mask() != update.initial_mode_mask()) { tmp.set_initial_mode_bits(update.initial_mode(), update.initial_mode_mask()); changed = true; } if (update.mode_changed()) { tmp.mode_changed_ = true; changed = true; } } if (!tmp_file_) { if (update.tmp_file()) { tmp.tmp_file_ = true; changed = true; } } if (propagated_ != propagated) { tmp.propagated_ = propagated; changed = true; } if (!changed) { return this; } else { return FileUsage::Get(tmp); } } bool file_file_usage_cmp(const file_file_usage& lhs, const file_file_usage& rhs) { return strcmp(lhs.file->c_str(), rhs.file->c_str()) < 0; } std::string FileUsage::d_internal(const int level) const { (void)level; /* unused */ return std::string("{FileUsage initial_state=") + d(initial_state_, level) + ", written=" + d(written_) + ", mode_changed=" + d(mode_changed_) + ", generation=" + d(generation_) + "}"; } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileUsage& fu, const int level) { return fu.d_internal(level); } std::string d(const FileUsage *fu, const int level) { if (fu) { return d(*fu, level); } else { return "{FileUsage NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/file_usage.h000066400000000000000000000200721447164520700205110ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FILE_USAGE_H_ #define FIREBUILD_FILE_USAGE_H_ #include #include #include #include #include "firebuild/file_info.h" #include "firebuild/file_usage_update.h" #include "firebuild/hash.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { struct FileUsageHasher; /** * FileUsage describes, for one particular Process and one particular * filename, the inital and final contents found at the given location * with as much accuracy as it matters to us. * * E.g. if the Process potentially reads from the file then its original * hash is computed and stored here, but if the Process does not read * the contents then it is not stored. Similarly, it's recorded whether * the process potentially modified the file. * * All these objects are kept in a global pool. If two such objects have * identical contents then they are the same object (same pointer). */ class FileUsage { public: bool written() const {return written_;} bool mode_changed() const {return mode_changed_;} bool tmp_file() const {return tmp_file_;} bool propagated() const {return propagated_;} file_generation_t generation() const {return generation_;} int unknown_err() {return unknown_err_;} void set_unknown_err(int e) {unknown_err_ = e;} FileType initial_type() const {return initial_state_.type();} void set_initial_type(FileType type) {initial_state_.set_type(type);} bool initial_size_known() const {return initial_state_.size_known();} size_t initial_size() const {return initial_state_.size();} void set_initial_size(size_t size) {initial_state_.set_size(size);} bool initial_hash_known() const {return initial_state_.hash_known();} const Hash& initial_hash() const {return initial_state_.hash();} void set_initial_hash(const Hash& hash) {initial_state_.set_hash(hash);} void set_initial_mode_bits(mode_t mode, mode_t mode_mask) {initial_state_.set_mode_bits(mode, mode_mask);} mode_t initial_mode() const {return initial_state_.mode();} mode_t initial_mode_mask() const {return initial_state_.mode_mask();} const FileInfo& initial_state() const {return initial_state_;} static const FileUsage* Get(FileType type = DONTKNOW) { return no_hash_not_written_states_[FileInfo::file_type_to_int(type)]; } /** * Merge a FileUsageUpdate object into this one. * * "this" describes the older events which happened to a file, and "update" describes the new ones. * * "this" is not updated, a possibly different pointer is returned which refers to the merged value. * * "update" might on demand compute certain values (currently the hash). It's formally "const", but * with some "mutable" members. The value behind the "update" reference is updated, so when this * change is bubbled up, at the next levels it'll have this field already filled in. * * Sometimes the file usages to merge are conflicting, like a directory was expected to not exist, * then it is expected to exist without creating it in the meantime. In those cases the return is * nullptr and it should disable shortcutting of the process and its ancestors. * * @return pointer to the merge result, or nullptr in case of an error */ const FileUsage* merge(const FileUsageUpdate& update, const bool propagated) const; /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d_internal(const int level = 0) const; private: explicit FileUsage(FileType type = DONTKNOW) : initial_state_(type) {} FileUsage(const FileName* filename, const FileInfo *initial_state, bool written, bool mode_changed, bool tmp_file, bool propagated, int unknown_err): initial_state_(*initial_state), written_(written), mode_changed_(mode_changed), tmp_file_(tmp_file), propagated_(propagated), generation_(filename->generation()), unknown_err_(unknown_err) {} /* Things that describe the filesystem when the process started up */ FileInfo initial_state_; /* Things that describe what the process potentially did */ /** The file's contents were altered by the process, e.g. written to, * or modified in any other way, including removal of the file, or * another file getting renamed to this one. */ bool written_ {false}; /** The file's mode was altered by the process. * (Luckily for us there's no way to set individual bits, chmod() always sets all of them. * So a single boolean can refer to all the 12 mode bits.) */ bool mode_changed_ {false}; /** Created as a temporary file with mktemp() and friends or inferred to be a temporary file * by the supervisor. */ bool tmp_file_ {false}; bool propagated_ {true}; /** Generation of the file the process last seen (either by reading or writing to the file). */ file_generation_t generation_ {0}; /* Note: stuff like the final hash are not stored here. They are * computed right before being placed in the cache, don't need to be * remembered in memory. */ /** Global FileUsage db*/ static std::unordered_set* db_; /** Frequently used singletons */ static const FileUsage* no_hash_not_written_states_[FILE_TYPE_MAX + 1]; /* This, along with the FileUsage::db_initializer_ definition in file_usage.cc, * initializes the file usage database once at startup. */ class DbInitializer { public: DbInitializer(); }; friend class DbInitializer; static DbInitializer db_initializer_; friend struct FileUsageHasher; friend bool operator==(const FileUsage& lhs, const FileUsage& rhs) = default; /** An unhandled error occurred during operation on the file. The process * can't be short-cut, but the first such error code is stored here. */ int unknown_err_ {0}; static const FileUsage* Get(const FileUsage& candidate); }; struct FileUsageHasher { std::size_t operator()(const FileUsage& f) const noexcept { XXH64_hash_t hash = XXH3_64bits_withSeed(f.initial_hash().get_ptr(), Hash::hash_size(), f.unknown_err_); ssize_t size = f.initial_size(); hash = XXH3_64bits_withSeed(&size, sizeof(size), hash); struct { uint64_t initial_type : 3; uint64_t initial_mode : 12; uint64_t initial_mode_mask : 12; uint64_t written : 1; uint64_t mode_changed : 1; uint64_t tmp_file : 1; uint64_t unused : 2; /* 32 bits so far */ uint64_t generation : 32; } merged_state = {f.initial_type(), f.initial_mode(), f.initial_mode_mask(), f.written_, f.mode_changed_, f.tmp_file(), 0, f.generation_}; hash = XXH3_64bits_withSeed(&merged_state, sizeof(merged_state), hash); return hash; } }; struct file_file_usage { const FileName* file; const FileUsage* usage; }; bool file_file_usage_cmp(const file_file_usage& lhs, const file_file_usage& rhs); /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileUsage& fu, const int level = 0); std::string d(const FileUsage *fu, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_FILE_USAGE_H_ firebuild-0.8.2/src/firebuild/file_usage_update.cc000066400000000000000000000437561447164520700222270ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /** * FileUsageUpdate describes, for one particular Process and one particular filename, some pieces of * information that we get to know right now. * * Such structures are not stored in our long-term memory, these are ephemeral objects describing a * change that we quickly register. * * The differences from FileUsage are: * * - A FileUsageUpdate object exists on its own, rather than in a pool of unique objects. * * - A FileUsageUpdate object can describe that some information (e.g. type or hash) matters to us, * but we haven't queried or computed it yet. This allows for lazy on-demand computation, and * therefore save precious CPU time if the information isn't needed. * * - A FileUsageUpdate knows which file it belongs to, so it can perform the on-demand work on its own. * * - A FileUsageUpdate carries information about what to do with its parent directory, e.g. whether it * needs to be registered that it must or must not exist. */ #include "firebuild/file_usage_update.h" #include #include #include #include #include "common/firebuild_common.h" #include "firebuild/debug.h" #include "firebuild/hash_cache.h" namespace firebuild { /** * If we saw a successful open(..., O_RDONLY) then this method initializes the file type (regular * vs. directory) and the hash lazily on demand. */ void FileUsageUpdate::type_computer_open_rdonly() const { Hash hash; bool is_dir; ssize_t size = -1; if (!hash_cache->get_hash(filename_, max_writers_, &hash, &is_dir, &size)) { unknown_err_ = errno; return; } if (is_dir) { initial_state_.set_type(ISDIR); initial_state_.set_hash(hash); } else { initial_state_.set_type(ISREG); initial_state_.set_size(size); initial_state_.set_hash(hash); } type_computer_ = nullptr; hash_computer_ = nullptr; } /** * Get the file type, looking it up on demand if necessary. * * Due to the nature of lazy lookup, an unexpected error can occur, in which case false is returned. */ bool FileUsageUpdate::get_initial_type(FileType *type_ptr) const { TRACKX(FB_DEBUG_PROC, 1, 1, FileUsageUpdate, this, ""); if (type_computer_) { (this->*type_computer_)(); } if (unknown_err_) { return false; } else { *type_ptr = initial_state_.type(); return true; } } /** * This method executes the lazy on-demand retrieval or computation of the hash. */ void FileUsageUpdate::hash_computer() const { Hash hash; if (hash_cache->get_hash(filename_, max_writers_, &hash)) { initial_state_.set_hash(hash); } else { unknown_err_ = errno; } hash_computer_ = nullptr; } /** * Get the file hash, figuring it out on demand if necessary. * * Due to the nature of lazy lookup, an unexpected error can occur, in which case false is returned. */ bool FileUsageUpdate::get_initial_hash(Hash *hash_ptr) const { TRACKX(FB_DEBUG_PROC, 1, 1, FileUsageUpdate, this, ""); assert(type_computer_ == nullptr && (initial_state_.type() == ISREG || initial_state_.type() == ISDIR)); if (hash_computer_) { (this->*hash_computer_)(); } if (unknown_err_) { return false; } else { *hash_ptr = initial_state_.hash(); return true; } } /** * Based on the parameters and return value of an open() or similar call, returns a FileUsageUpdate * object that reflects how our usage of this file changed. * * If the file's hash is important then we don't compute it yet but set hash_computer_ so that we * can compute it on demand. */ FileUsageUpdate FileUsageUpdate::get_from_open_params( const FileName *filename, int flags, mode_t mode_with_umask, int err, bool tmp_file) { TRACK(FB_DEBUG_PROC, "flags=%d, mode_with_umask=0%03o, err=%d, tmp_file=%s", flags, mode_with_umask, err, tmp_file ? "true" : "false"); FileUsageUpdate update(filename); if (!err) { if (is_write(flags)) { /* If successfully opened for writing: * * trunc creat excl * A + - => prev file must exist, contents don't matter * B + + - => prev file doesn't matter * C + + + => prev file mustn't exist * D - - => prev file must exist, contents preserved and matter * E - + - => contents preserved (or new empty) and matter * F - + + => prev file mustn't exist */ if ((flags & O_CREAT) && (flags & O_EXCL)) { /* C+F: If an exclusive new file was created, take a note that the file didn't exist * previously, that the permissions will have to be set on it, and that its parent dir has * to exist. */ update.set_initial_type(NOTEXIST); update.mode_changed_ = true; update.parent_type_ = ISDIR; update.tmp_file_ = tmp_file; } else if (flags & O_TRUNC) { assert(!tmp_file); if (!(flags & O_CREAT)) { /* A: What a nasty combo! We must take a note that the file existed, but don't care about * its previous contents (also it's too late now to figure that out). This implies that * the parent directory exists, no need to note that separately. */ update.set_initial_type(ISREG); } else { /* B: The old contents could have been any regular file, or even no such file (but not * e.g. a directory). Also, the parent directory has to exist. */ struct stat64 st; if (stat64(filename->c_str(), &st) < 0) { update.unknown_err_ = errno; return update; } if (st.st_size > 0) { /* We had O_TRUNC, so this is unexpected. */ update.unknown_err_ = EEXIST; return update; } // FIXME handle if we see a directory. This cannot normally happen due to O_CREAT, but // can if the file has just been replaced by a directory. update.set_initial_type(NOTEXIST_OR_ISREG); if ((st.st_mode & 07777) != mode_with_umask) { /* A mode mismatch implies that the file necessarily existed before. See #861. */ update.set_initial_type(ISREG); } else { /* The mode matches. The file may or may not have existed before, but in either case, * it'll have the given permissions now. Pretend that they were set explicitly. #861. */ update.mode_changed_ = true; } update.parent_type_ = ISDIR; } } else { assert(!tmp_file); if (!(flags & O_CREAT)) { /* D: Contents unchanged. Need to checksum the file, we'll do that lazily. Implies that * the parent directory exists, no need to note that separately. */ update.set_initial_type(ISREG); update.hash_computer_ = &FileUsageUpdate::hash_computer; } else { /* E: Another nasty combo. We can't distinguish a newly created empty file from a * previously empty one. If the file is non-empty, we need to store its hash. Also, the * parent directory has to exist. */ struct stat64 st; if (stat64(filename->c_str(), &st) < 0) { update.unknown_err_ = errno; return update; } if (st.st_size > 0) { // FIXME handle if we see a directory. This cannot normally happen due to O_CREAT, but // can if the file has just been replaced by a directory. update.set_initial_type(ISREG); /* We got to know that this was a regular non-empty file. Delay hash computation until * necessary. */ update.hash_computer_ = &FileUsageUpdate::hash_computer; } else { update.set_initial_type(NOTEXIST_OR_ISREG); } update.set_initial_size(st.st_size); if ((st.st_mode & 07777) != mode_with_umask) { /* A mode mismatch implies that the file necessarily existed before. See #861. */ update.set_initial_type(ISREG); } else { /* The mode matches. The file may or may not have existed before, but in either case, * it'll have the given permissions now. Pretend that they were set explicitly. #861. */ update.mode_changed_ = true; } update.parent_type_ = ISDIR; } } update.parent_type_ = ISDIR; update.written_ = true; update.max_writers_ = 1; } else { /* The file or directory was successfully opened for reading only. * Note that a plain open() can open a directory for reading, even without O_DIRECTORY. */ update.type_computer_ = &FileUsageUpdate::type_computer_open_rdonly; update.hash_computer_ = &FileUsageUpdate::hash_computer; } } else /* if (err) */ { /* The attempt to open failed. */ if (is_write(flags)) { if (err == ENOENT) { if (!(flags & O_CREAT)) { /* If opening without O_CREAT failed then the file didn't exist. */ update.set_initial_type(NOTEXIST); } else { /* When opening a file for writing the absence of the parent dir * results in NOTEXIST error. The grandparent dir could be missing as well, * but the missing parent dir would cause the same error thus it will not be a mistake * to shortcut the process if the parent dir is indeed missing. */ update.parent_type_ = NOTEXIST; } } else if (err == EEXIST) { if (!tmp_file) { assert(flags & O_CREAT && flags & O_EXCL); update.set_initial_type(EXIST); } else { /* Could not create a unique temporary filename. Now the contents of template are * undefined.*/ update.set_initial_type(DONTKNOW); update.tmp_file_ = tmp_file; /* This error is actually known and handled, but it is safer to just prevent merging * this update by setting unknown_err_ because the path is undefined. */ update.unknown_err_ = err; } } else if (err == ENOTDIR) { /* Occurs when opening the "foo/baz/bar" path when "foo/baz" is not a directory, * but for example a regular file. Or when "foo" is a regular file. We can't distinguish * between those cases, but if "/foo/baz" is a regular file we can safely shortcut the * process, because the process could not tell the difference either. */ update.parent_type_ = ISREG; } else if (err == EINVAL) { update.set_initial_type(DONTKNOW); if (tmp_file) { /* Template was invalid, and is unmodified. We know nothing about that path. */ update.set_initial_type(DONTKNOW); update.tmp_file_ = tmp_file; } /* This error is actually known and handled, but it is safer to just prevent merging * this update because the path is not used */ update.unknown_err_ = err; return update; } else { /* We don't support other errors such as permission denied. */ update.unknown_err_ = err; return update; } } else { assert(!tmp_file); /* Opening for reading failed. */ if (err == ENOENT) { update.set_initial_type(NOTEXIST); } else if (err == ENOTDIR) { /* See the comment in the is_write() branch. */ update.parent_type_ = ISREG; } else { /* We don't support other errors such as permission denied. */ update.unknown_err_ = err; return update; } } } return update; } /** * Based on the parameters and return value of an mkdir() call, returns a FileUsageUpdate object that * reflects how our usage of this file changed. */ FileUsageUpdate FileUsageUpdate::get_from_mkdir_params(const FileName *filename, int err, bool tmp_dir) { TRACK(FB_DEBUG_PROC, "err=%d", err); FileUsageUpdate update(filename); if (!err) { update.set_initial_type(NOTEXIST); update.parent_type_ = ISDIR; update.written_ = true; update.mode_changed_ = true; update.tmp_file_ = tmp_dir; } else { if (err == EEXIST) { /* The directory already exists. It may not be a directory, but in that case process inputs * will not match either. */ update.set_initial_type(ISDIR); } else if (err == ENOENT) { /* A directory component in pathname does not exist or is a dangling symbolic link */ // FIXME(rbalint) handle the dangling symlink case, too update.set_initial_type(NOTEXIST); update.parent_type_ = NOTEXIST; } else if (err == EINVAL) { update.set_initial_type(DONTKNOW); if (tmp_dir) { /* Template was invalid, and is unmodified. We know nothing about that path. */ update.set_initial_type(DONTKNOW); update.tmp_file_ = tmp_dir; /* This error is actually known and handled, but it is safer to just prevent merging * this update by still setting unknown_err_ because the path is not used */ } update.unknown_err_ = err; } else { /* We don't support other errors such as permission denied. */ update.unknown_err_ = err; } } return update; } /** * Based on the parameters and return value of a stat() or similar call, returns a FileUsageUpdate * object that reflects how our usage of this file changed. */ FileUsageUpdate FileUsageUpdate::get_from_stat_params(const FileName *filename, mode_t mode, off_t size, int err) { TRACK(FB_DEBUG_PROC, "mode=%d, size=%" PRIoff ", err=%d", mode, size, err); FileUsageUpdate update(filename); if (!err) { if (S_ISREG(mode)) { update.set_initial_type(ISREG); update.set_initial_mode_bits(mode, 07777 /* we know all the mode bits */); update.set_initial_size(size); } else if (S_ISDIR(mode)) { update.set_initial_type(ISDIR); update.set_initial_mode_bits(mode, 07777 /* we know all the mode bits */); } else if (S_ISLNK(mode)) { /* It's a symlink. We got to know absolutely nothing about the underlying file, directory, or * lack thereof. FIXME: Refine this logic as per #784. */ update.set_initial_type(DONTKNOW); } else { /* Neither regular file nor directory. Pretend for now that there's nothing there. */ update.set_initial_type(NOTEXIST); } } else { update.set_initial_type(NOTEXIST); } return update; } /** * Based on the parameters and return value of a rename() or similar call, returns a FileUsageUpdate * object that reflects how our usage of old file changed. */ FileUsageUpdate FileUsageUpdate::get_oldfile_usage_from_rename_params(const FileName *old_name, const FileName *new_name, int error) { TRACK(FB_DEBUG_PROC, "err=%d", error); /* Read the file's hash from the new location, but update generation from the old one's name * to keep the generation number increasing. Otherwise it would be reset to 1, which is valid * for the newly created file (if the file did not exist before). */ // TODO(rbalint) Error handling is way more complicated for rename than for open, fix that here. FileUsageUpdate update = get_from_open_params(new_name, O_RDONLY, 0, error, false); update.written_ = true; update.mode_changed_ = true; update.generation_ = old_name->generation(); return update; } /** * Based on the parameters and return value of a rename() or similar call, returns a FileUsageUpdate * object that reflects how our usage of new file changed. */ FileUsageUpdate FileUsageUpdate::get_newfile_usage_from_rename_params(const FileName *new_name, int error) { TRACK(FB_DEBUG_PROC, "err=%d", error); (void)error; /* maybe unused */ /* The file at the new name now necessarily exists. It may or may not empty, it doesn't matter. We * have to set mode_changed so that the mode will be restored when replaying from the cache. * This does not match any of the A..F cases of get_from_open_params(). */ FileUsageUpdate update(new_name, DONTKNOW, true, true); return update; } /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string FileUsageUpdate::d_internal(const int level) const { (void)level; /* unused */ return std::string("{FileUsageUpdate initial_state=") + d(initial_state_, level) + (type_computer_ ? ", type_computer=" : "") + (hash_computer_ ? ", hash_computer=" : "") + ", written=" + d(written_) + ", mode_changed=" + d(mode_changed_) + ", generation=" + d(generation_) + ", unknown_err=" + d(unknown_err_) + "}"; } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileUsageUpdate& fuu, const int level) { return fuu.d_internal(level); } std::string d(const FileUsageUpdate *fuu, const int level) { if (fuu) { return d(*fuu, level); } else { return "{FileUsageUpdate NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/file_usage_update.h000066400000000000000000000154621447164520700220620ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FILE_USAGE_UPDATE_H_ #define FIREBUILD_FILE_USAGE_UPDATE_H_ #include #include "firebuild/file_info.h" #include "firebuild/file_name.h" #include "firebuild/hash.h" namespace firebuild { class FileUsageUpdate { public: FileUsageUpdate(const FileName *filename, FileInfo info, bool written = false, bool mode_changed = false) : initial_state_(info), filename_(filename), written_(written), mode_changed_(mode_changed), generation_(filename->generation()) {} explicit FileUsageUpdate(const FileName *filename, FileType type = DONTKNOW, bool written = false, bool mode_changed = false) : initial_state_(type), filename_(filename), written_(written), mode_changed_(mode_changed), generation_(filename->generation()) {} static FileUsageUpdate get_from_open_params(const FileName *filename, int flags, mode_t mode_with_umask, int err, bool tmp_file); static FileUsageUpdate get_from_mkdir_params(const FileName *filename, int err, bool tmp_dir); static FileUsageUpdate get_from_stat_params(const FileName *filename, mode_t mode, off_t size, int err); static FileUsageUpdate get_oldfile_usage_from_rename_params(const FileName* old_name, const FileName* new_name, int err); static FileUsageUpdate get_newfile_usage_from_rename_params(const FileName* new_name, int err); FileType parent_type() const {return parent_type_;} bool written() const {return written_;} bool mode_changed() const {return mode_changed_;} bool tmp_file() const {return tmp_file_;} file_generation_t generation() const {return generation_;} bool unknown_err() const {return unknown_err_;} bool get_initial_type(FileType *type_ptr) const; void set_initial_type(FileType type) {initial_state_.set_type(type);} bool initial_size_known() const {return initial_state_.size_known();} size_t initial_size() const {return initial_state_.size();} void set_initial_size(size_t size) {initial_state_.set_size(size);} bool initial_hash_known() const {return initial_state_.hash_known() || hash_computer_ != nullptr;} bool get_initial_hash(Hash *hash_ptr) const; void set_initial_hash(const Hash& hash) {initial_state_.set_hash(hash);} void set_initial_mode_bits(mode_t mode, mode_t mode_mask) {initial_state_.set_mode_bits(mode, mode_mask);} mode_t initial_mode() const {return initial_state_.mode();} mode_t initial_mode_mask() const {return initial_state_.mode_mask();} const FileInfo& initial_state() const {return initial_state_;} /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d_internal(const int level = 0) const; private: /* The information we got to know about the file, prior to the changes that potentially occurred * to it. * * If type_computer_ is set then initial_state_.type_ is not yet set to its correct value, it'll * be figured out on demand. * * If hash_computer_ is set then initial_state_.hash_ is not yet set to its correct value, it'll * be figured out on demand. However, initial_state_.hash_known() reports true. */ mutable FileInfo initial_state_; /* The filename, used when needed to lazily initialize some fields. */ const FileName *filename_; /* Whenever hash computation takes place, this is the maximum number of allowed writers. * E.g. if a file is opened for reading then this number is 0 (meaning no writers allowed at all), * but when a file is opened for writing then this number is 1 (because the intercepted process * has just opened it for writing, but there must not be any other writers.) */ int max_writers_ {0}; void type_computer_open_rdonly() const; void type_computer_open_wronly_creat_notrunc_noexcl() const; void hash_computer() const; /* If initial_state_'s type_ or hash_ aren't known yet (but in case of hash_ we know that we'll * need to know it), they will be initialized on demand by these methods. */ mutable void (FileUsageUpdate::*type_computer_)() const {nullptr}; /* NOLINT(readability/braces) */ mutable void (FileUsageUpdate::*hash_computer_)() const {nullptr}; /* NOLINT(readability/braces) */ /* The file's contents were altered by the process, e.g. written to, or modified in any other way, * including removal of the file, or another file getting renamed to this one. */ bool written_ {false}; /** The file's mode was altered by the process. * (Luckily for us there's no way to set individual bits, chmod() always sets all of them. * So a single boolean can refer to all the 12 mode bits.) */ bool mode_changed_ {false}; /** Created as a temporary file with mktemp() and friends or inferred to be a temporary file * by the supervisor. */ bool tmp_file_ {false}; /* File's current generation. */ file_generation_t generation_; /* What we know and are interested in about the parent path. E.g. * - DONTKNOW = nothing of interest * - NOTEXIST = no such entry on the filesystem * - ISDIR = it is a directory * - ISREG = it is a regular file */ FileType parent_type_ {DONTKNOW}; /* This does not strictly obey the semantics of "mutable" because lazy evaluation of * initial_state_.type_ or initial_state_.hash_ might modify it, but I don't think it'll be a * problem, since we don't query this variable before performing the lazy evaluation. */ mutable int unknown_err_ {0}; }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const FileUsageUpdate& fuu, const int level = 0); std::string d(const FileUsageUpdate *fuu, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_FILE_USAGE_UPDATE_H_ firebuild-0.8.2/src/firebuild/firebuild.cc000066400000000000000000000460201447164520700205120ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/firebuild.h" #ifdef __APPLE__ #include #endif #include #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #include #include #include #include #include "firebuild/debug.h" #include "firebuild/sigchild_callback.h" #include "firebuild/config.h" #include "firebuild/connection_context.h" #include "firebuild/epoll.h" #include "firebuild/file_name.h" #include "firebuild/message_processor.h" #include "firebuild/execed_process_cacher.h" #include "firebuild/process.h" #include "firebuild/process_tree.h" #include "firebuild/report.h" bool generate_report = false; int sigchild_selfpipe[2]; int listener; int child_pid, child_ret = 1; namespace { static char *fb_tmp_dir; static char *fb_conn_string; static bool insert_trace_markers = false; static const char *report_file = "firebuild-build-report.html"; /** only if debugging "time" */ struct timespec start_time; static void usage() { printf("Usage: firebuild [OPTIONS] \n" "Execute BUILD COMMAND with Firebuild instrumentation\n" "\n" "Mandatory arguments to long options are mandatory for short options too.\n" " -c --config-file=FILE Use FILE as configuration file.\n" " If not specified, load .firebuild.conf, ~/.firebuild.conf,\n" " $XDG_CONFIG_HOME/firebuild/firebuild.conf or\n" " /etc/firebuild.conf in that order.\n" " -C --directory=DIR change directory before running the command\n" " -d --debug-flags=list comma separated list of debug flags,\n" " \"-d help\" to get a list.\n" " -D --debug-filter=list comma separated list of commands to debug.\n" " Debug messages related to processes which are not listed\n" " are suppressed.\n" " -g --gc Garbage collect the cache.\n" " Keeps debugging entries related to kept files when used\n" " together with \"--debug cache\".\n" " -r --generate-report[=HTML] generate a report on the build command execution.\n" " the report's filename can be specified \n" " (firebuild-build-report.html by default). \n" " -h --help show this help\n" " -o --option=key=val Add or replace a scalar in the config\n" " -o --option=key=[] Clear an array in the config\n" " -o --option=key+=val Append to an array of scalars in the config\n" " -o --option=key-=val Remove from an array of scalars in the config\n" " -s --show-stats Show cache hit statistics.\n" " -z --zero-stats Zero cache hit statistics.\n" " -i --insert-trace-markers perform open(\"/FIREBUILD \", 0) calls\n" " to let users find unintercepted calls using\n" " strace or ltrace. This works in debug builds only.\n" " --version output version information and exit\n" "Exit status:\n" " exit status of the BUILD COMMAND\n" " 1 in case of failure\n"); } /** * Get the system temporary directory to use. * * TMPDIR is used if it's nonempty. * Note that relative path is accepted and used correctly by the * firebuild process itself, although the build command it launches * might not support it. It's highly recommended to use absolute path. * * If TMPDIR is unset or empty, use the default "/tmp". * * @return the system temporary directory to use */ static const char *get_tmpdir() { const char *tmpdir = getenv("TMPDIR"); if (tmpdir != NULL && tmpdir[0] != '\0') { return tmpdir; } else { return "/tmp"; } } } /* namespace */ /** * Create connection sockets for the interceptor */ static int create_listener() { int listener; if ((listener = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { firebuild::fb_perror("socket"); exit(EXIT_FAILURE); } struct sockaddr_un local; local.sun_family = AF_UNIX; strncpy(local.sun_path, fb_conn_string, sizeof(local.sun_path) - 1); if (bind(listener, (struct sockaddr *)&local, sizeof(local)) == -1) { firebuild::fb_perror("bind"); exit(EXIT_FAILURE); } if (listen(listener, 500) == -1) { firebuild::fb_perror("listen"); exit(EXIT_FAILURE); } return listener; } /* This is the installed handler for SIGCHLD, using the good ol' self-pipe trick to cooperate with * epoll_wait() without race condition. Our measurements show this is faster than epoll_pwait(). */ static void sigchild_handler(int signum) { (void)signum; /* unused */ /* listener being -1 means that we're already exiting, and might have closed sigchild_selfpipe. * In case an orphan descendant dies now and we get a SIGCHLD, just ignore it. */ if (listener >= 0) { char dummy = 0; int write_ret = write(sigchild_selfpipe[1], &dummy, 1); (void)write_ret; /* unused */ } } static void accept_ic_conn(const struct epoll_event* event, void *arg) { TRACK(firebuild::FB_DEBUG_COMM, "listener=%d", listener); struct sockaddr_storage remote; socklen_t slen = sizeof(remote); (void) event; /* unused */ (void) arg; /* unused */ int fd = accept(listener, (struct sockaddr*)&remote, &slen); if (fd < 0) { firebuild::fb_perror("accept"); } else { if (firebuild::epoll->is_added_fd(fd)) { /* This happens very rarely. Just when the file descriptor has been closed by the other end, * the epoll loop did not process this event yet, but the file descriptor got reused for the * new connection. */ fd = firebuild::epoll->remap_to_not_added_fd(fd); } firebuild::bump_fd_age(fd); auto conn_ctx = new firebuild::ConnectionContext(fd); fcntl(fd, F_SETFL, O_NONBLOCK); firebuild::epoll->add_fd(fd, EPOLLIN, firebuild::MessageProcessor::ic_conn_readcb, conn_ctx); } } static bool running_under_valgrind() { const char *v = getenv(LD_PRELOAD); if (v == NULL) { return false; } else { return (strstr(v, "/valgrind/") != NULL || strstr(v, "/vgpreload") != NULL); } } int main(const int argc, char *argv[]) { char *config_file = NULL; char *directory = NULL; std::list config_strings = {}; int c; bool gc = false, print_stats = false, reset_stats = false; /* init global data */ firebuild::cfg = new libconfig::Config(); /* parse options */ setenv("POSIXLY_CORRECT", "1", true); while (1) { int option_index = 0; static struct option long_options[] = { {"config-file", required_argument, 0, 'c' }, {"gc", no_argument, 0, 'g' }, {"directory", required_argument, 0, 'C' }, {"debug-flags", required_argument, 0, 'd' }, {"debug-filter", required_argument, 0, 'D' }, {"generate-report", optional_argument, 0, 'r' }, {"help", no_argument, 0, 'h' }, {"option", required_argument, 0, 'o' }, {"show-stats", no_argument, 0, 's' }, {"zero-stats", no_argument, 0, 'z' }, {"insert-trace-markers", no_argument, 0, 'i' }, {"version", no_argument, 0, 'v' }, {0, 0, 0, 0 } }; c = getopt_long(argc, argv, "c:C:d:D:r::o:ghisz", long_options, &option_index); if (c == -1) break; switch (c) { case 'c': config_file = optarg; break; case 'C': directory = optarg; break; case 'd': /* Merge the values, so that multiple '-d' options are also allowed. */ firebuild::debug_flags |= firebuild::parse_debug_flags(optarg); break; case 'g': gc = true; break; case 'D': firebuild::init_debug_filter(optarg); break; case 'h': usage(); exit(EXIT_SUCCESS); /* break; */ case 'o': if (optarg != NULL) { config_strings.push_back(std::string(optarg)); } else { usage(); exit(EXIT_FAILURE); } break; case 'i': #ifdef FB_EXTRA_DEBUG insert_trace_markers = true; #endif break; case 'r': generate_report = true; if (optarg != NULL) { report_file = optarg; } break; case 's': print_stats = true; break; case 'v': printf("Firebuild " FIREBUILD_VERSION "\n\n" "Copyright (c) 2022 Firebuild Inc.\n" "All rights reserved.\n" "Free for personal use and commercial trial.\n" "Non-trial commercial use requires licenses available from https://firebuild.com.\n" "\n" "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n" "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n" "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n" "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n" "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n" "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n" "SOFTWARE.\n"); exit(EXIT_SUCCESS); break; case 'z': reset_stats = true; break; default: usage(); exit(EXIT_FAILURE); } } if (optind >= argc) { if (!gc && !print_stats && !reset_stats) { usage(); exit(EXIT_FAILURE); } } else { if (gc) { printf("The --gc (or -g) option can be used only without a BUILD COMMAND."); exit(EXIT_FAILURE); } } if (FB_DEBUGGING(firebuild::FB_DEBUG_TIME)) { clock_gettime(CLOCK_MONOTONIC, &start_time); } firebuild::read_config(firebuild::cfg, config_file, config_strings); /* Initialize the cache */ firebuild::ExecedProcessCacher::init(firebuild::cfg); if (reset_stats) { firebuild::execed_process_cacher->reset_stored_stats(); } if (optind >= argc) { if (gc) { firebuild::execed_process_cacher->gc(); firebuild::execed_process_cacher->update_stored_bytes(); /* Store GC runs, too. */ firebuild::execed_process_cacher->update_stored_stats(); } if (print_stats) { if (!gc) { firebuild::execed_process_cacher->add_stored_stats(); } firebuild::execed_process_cacher->print_stats(firebuild::FB_SHOW_STATS_STORED); } exit(0); } { char *pattern; if (asprintf(&pattern, "%s/firebuild.XXXXXX", get_tmpdir()) < 0) { firebuild::fb_perror("asprintf"); } fb_tmp_dir = mkdtemp(pattern); if (fb_tmp_dir == NULL) { firebuild::fb_perror("mkdtemp"); exit(EXIT_FAILURE); } fb_conn_string = strdup((std::string(fb_tmp_dir) + "/socket").c_str()); } firebuild::FileName::default_tmpdir = firebuild::FileName::Get("/tmp", strlen("/tmp")); auto env_exec = firebuild::get_sanitized_env(firebuild::cfg, fb_conn_string, insert_trace_markers); /* Set up sigchild handler */ if (fb_pipe2(sigchild_selfpipe, O_CLOEXEC | O_NONBLOCK) != 0) { firebuild::fb_perror("pipe"); exit(EXIT_FAILURE); } struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_handler = sigchild_handler; sa.sa_flags = SA_RESTART; sigaction(SIGCHLD, &sa, NULL); /* Configure epoll */ firebuild::epoll = new firebuild::Epoll(); /* Open listener socket before forking child to always let the child connect */ listener = create_listener(); firebuild::epoll->add_fd(listener, EPOLLIN, accept_ic_conn, NULL); #ifdef __linux__ /* Collect orphan children */ prctl(PR_SET_CHILD_SUBREAPER, 1); #endif /* run command and handle interceptor messages */ if ((child_pid = fork()) == 0) { int i; /* intercepted process */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wstrict-overflow" char* argv_exec[argc - optind + 1]; #pragma GCC diagnostic pop /* we don't need that */ close(listener); /* create and execute build command */ for (i = 0; i < argc - optind ; i++) { argv_exec[i] = argv[optind + i]; } argv_exec[i] = NULL; if (directory != NULL && chdir(directory) != 0) { firebuild::fb_perror("chdir"); exit(EXIT_FAILURE); } #ifdef __APPLE__ *_NSGetEnviron() = env_exec; execvp(argv[optind], argv_exec); #else execvpe(argv[optind], argv_exec, env_exec); #endif firebuild::fb_perror("Executing build command failed"); exit(EXIT_FAILURE); } else { /* supervisor process */ /* This creates some Pipe objects, so needs ev_base being set up. */ firebuild::proc_tree = new firebuild::ProcessTree(); /* Add a ForkedProcess for the supervisor's forked child we never directly saw. */ firebuild::proc_tree->insert_root(child_pid, STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO); bump_limits(); /* no SIGPIPE if a supervised process we're writing to unexpectedly dies */ signal(SIGPIPE, SIG_IGN); firebuild::epoll->add_fd(sigchild_selfpipe[0], EPOLLIN, firebuild::sigchild_cb, NULL); /* Main loop for processing interceptor messages */ /* Runs until the only remaining epoll-monitored fd is the sigchild_selfpipe fd. */ while (firebuild::epoll->fds() > 1) { /* This is where the process spends its idle time: waiting for an event over a fd, or a * sigchild. * * If our immediate child exited (rather than some orphan descendant thereof, see * prctl(PR_SET_CHILD_SUBREAPER) above) then the handler sigchild_cb() will set listener to * -1, that's how we'll break out of this loop. */ firebuild::epoll->wait(); /* Process the reported events, if any. */ firebuild::epoll->process_all_events(); firebuild::proc_tree->GcProcesses(); } /* Finish all top pipes */ firebuild::proc_tree->FinishInheritedFdPipes(); /* Close the self-pipe */ close(sigchild_selfpipe[0]); close(sigchild_selfpipe[1]); } if (firebuild::debug_filter) { firebuild::debug_suppressed = false; } if (!firebuild::proc_tree->root()) { fprintf(stderr, "ERROR: Could not collect any information about the build " "process\n"); child_ret = EXIT_FAILURE; } else { struct rusage ru_myslf; getrusage(RUSAGE_SELF, &ru_myslf); const unsigned int cpu_time_self_ms = ru_myslf.ru_utime.tv_sec * 1000 + ru_myslf.ru_utime.tv_usec / 1000 + ru_myslf.ru_stime.tv_sec * 1000 + ru_myslf.ru_stime.tv_usec / 1000; firebuild::execed_process_cacher->set_self_cpu_time_ms(cpu_time_self_ms); /* Print times, including user and sys time separately for firebuild itself and its children. * The syntax is similar to bash's "time", although easier to parse (raw seconds in decimal). */ if (FB_DEBUGGING(firebuild::FB_DEBUG_TIME)) { struct timespec end_time, diff_time; struct rusage ru_chldr, ru_total; clock_gettime(CLOCK_MONOTONIC, &end_time); getrusage(RUSAGE_CHILDREN, &ru_chldr); timespecsub(&end_time, &start_time, &diff_time); timeradd(&ru_myslf.ru_utime, &ru_chldr.ru_utime, &ru_total.ru_utime); timeradd(&ru_myslf.ru_stime, &ru_chldr.ru_stime, &ru_total.ru_stime); fprintf(stderr, "\nResource usages, in seconds:\n" "real %5ld.%03ld\n" #ifdef __APPLE__ "user firebuild %5ld.%03d\n" "user children %5ld.%03d\n" "user total %5ld.%03d\n" "sys firebuild %5ld.%03d\n" "sys children %5ld.%03d\n" "sys total %5ld.%03d\n" #else "user firebuild %5ld.%03ld\n" "user children %5ld.%03ld\n" "user total %5ld.%03ld\n" "sys firebuild %5ld.%03ld\n" "sys children %5ld.%03ld\n" "sys total %5ld.%03ld\n" #endif "\n" "firebuild's memory usage in MiB:\n" "max. res. set %9.03f\n", diff_time.tv_sec, diff_time.tv_nsec / (1000 * 1000), ru_myslf.ru_utime.tv_sec, ru_myslf.ru_utime.tv_usec / 1000, ru_chldr.ru_utime.tv_sec, ru_chldr.ru_utime.tv_usec / 1000, ru_total.ru_utime.tv_sec, ru_total.ru_utime.tv_usec / 1000, ru_myslf.ru_stime.tv_sec, ru_myslf.ru_stime.tv_usec / 1000, ru_chldr.ru_stime.tv_sec, ru_chldr.ru_stime.tv_usec / 1000, ru_total.ru_stime.tv_sec, ru_total.ru_stime.tv_usec / 1000, static_cast(ru_myslf.ru_maxrss) / 1024); } if (firebuild::execed_process_cacher->is_gc_needed()) { firebuild::execed_process_cacher->gc(); } if (print_stats) { /* Separate stats from other output. */ fprintf(stdout, "\n"); firebuild::execed_process_cacher->print_stats(firebuild::FB_SHOW_STATS_CURRENT); } firebuild::execed_process_cacher->read_stored_cached_bytes(); firebuild::execed_process_cacher->update_stored_bytes(); firebuild::execed_process_cacher->update_stored_stats(); /* show process tree if needed */ if (generate_report) { const std::string datadir(getenv("FIREBUILD_DATA_DIR") ? getenv("FIREBUILD_DATA_DIR") : FIREBUILD_DATADIR); firebuild::Report::write(report_file, datadir); } } unlink(fb_conn_string); rmdir(fb_tmp_dir); #ifdef FB_EXTRA_DEBUG (void)running_under_valgrind; { #else if (running_under_valgrind()) { #endif /* keep Valgrind happy */ { char* env_str; for (int i = 0; (env_str = env_exec[i]) != NULL; i++) { free(env_str); } free(env_exec); } /* No more epoll needed, this also closes all tracked fds */ delete firebuild::epoll; free(fb_conn_string); free(fb_tmp_dir); delete(firebuild::proc_tree); delete(firebuild::cfg); fclose(stdin); fclose(stdout); fclose(stderr); } exit(child_ret); } firebuild-0.8.2/src/firebuild/firebuild.h000066400000000000000000000017731447164520700203620ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FIREBUILD_H_ #define FIREBUILD_FIREBUILD_H_ extern int child_pid, child_ret; extern int sigchild_selfpipe[2]; extern int listener; #endif // FIREBUILD_FIREBUILD_H_ firebuild-0.8.2/src/firebuild/forked_process.cc000066400000000000000000000072421447164520700215600ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include "firebuild/file_name.h" #include "firebuild/execed_process.h" #include "firebuild/forked_process.h" #include "firebuild/debug.h" namespace firebuild { ForkedProcess::ForkedProcess(const int pid, const int ppid, Process* parent, std::vector>* fds) : Process(pid, ppid, 0, parent ? parent->wd() : FileName::Get(""), parent ? parent->umask() : 0, parent, fds, parent ? parent->debug_suppressed() : false) { TRACKX(FB_DEBUG_PROC, 0, 1, Process, this, "pid=%d, ppid=%d, parent=%s", pid, ppid, D(parent)); /* add as fork child of parent */ if (parent) { exec_point_ = parent->exec_point(); parent->fork_children().push_back(this); } } void ForkedProcess::set_been_waited_for() { assert(!been_waited_for_); been_waited_for_ = true; /* Try finalizing the process at the bottom of the exec chain. If that succeeds it bubbles up. */ last_exec_descendant()->maybe_finalize(); } void ForkedProcess::set_has_orphan_descendant_bubble_up() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); #ifdef FB_EXTRA_DEBUG assert(!exec_point() || !exec_point()->can_shortcut()); #endif /* This may set has_orphan_descendant_ again, but the bubble up needs to go up to the top * for each new orphan to potentially unblock waits. */ has_orphan_descendant_ = true; /* Unblock waits if every descendant is finalized or orphan with terminated ancestors. */ if (has_on_finalized_ack_set() && can_ack_parent_wait()) { maybe_send_on_finalized_ack(); } if (parent()) { parent()->fork_point()->set_has_orphan_descendant_bubble_up(); } } void ForkedProcess::maybe_send_on_finalized_ack() { if (on_finalized_ack_id_ != -1) { assert(on_finalized_ack_fd_ != -1); ack_msg(on_finalized_ack_fd_, on_finalized_ack_id_); on_finalized_ack_id_ = -1; on_finalized_ack_fd_ = -1; } } void ForkedProcess::do_finalize() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); /* Now we can ack the previous system()'s second message, * or a pending pclose() or wait*(). */ maybe_send_on_finalized_ack(); close_fds(); /* Call the base class's method */ Process::do_finalize(); } ForkedProcess::~ForkedProcess() { TRACKX(FB_DEBUG_PROC, 1, 0, Process, this, ""); } std::string ForkedProcess::d_internal(const int level) const { if (level > 0) { /* brief */ return Process::d_internal(level); } else { /* verbose */ return "{ForkedProcess " + pid_and_exec_count() + ", " + state_string() + ", " + (been_waited_for() ? "" : "not ") + "been waited for, parent " + (parent() ? parent()->pid_and_exec_count() : "NULL") + ", " + (exec_point() ? d(exec_point()->args_to_short_string()) : "") + ", fds=" + d(fds(), level + 1) + "}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/forked_process.h000066400000000000000000000075701447164520700214260ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_FORKED_PROCESS_H_ #define FIREBUILD_FORKED_PROCESS_H_ #include #include #include #include #include #include "firebuild/process.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { class ExecedProcess; class ForkedProcess : public Process { public: explicit ForkedProcess(const int pid, const int ppid, Process* parent, std::vector>* fds); virtual ~ForkedProcess(); ExecedProcess* exec_point() {return exec_point_;} ForkedProcess* fork_point() {return this;} const ForkedProcess* fork_point() const {return this;} const ExecedProcess* exec_point() const {return exec_point_;} int exit_status() const {return exit_status_;} void set_exit_status(const int e) {exit_status_ = e;} bool has_orphan_descendant() const {return has_orphan_descendant_;} void set_has_orphan_descendant_bubble_up(); /** * Fail to change to a working directory */ void handle_fail_wd(const char * const d) { assert(parent()); parent()->handle_fail_wd(d); } /** * Record visited working directory */ void add_wd(const FileName *d) { assert(parent()); parent()->add_wd(d); } Process* exec_proc() const {return parent()->exec_proc();} void do_finalize(); void set_on_finalized_ack(int id, int fd) { assert(on_finalized_ack_id_ == -1 && on_finalized_ack_fd_ == -1); on_finalized_ack_id_ = id; on_finalized_ack_fd_ = fd; } bool has_on_finalized_ack_set() const { return on_finalized_ack_id_ != -1; } void maybe_send_on_finalized_ack(); /* Parent's wait for this process can be ACK-ed. */ bool can_ack_parent_wait() const { return (state() == FB_PROC_FINALIZED) || (state() == FB_PROC_TERMINATED && has_orphan_descendant() && !any_child_not_finalized_or_terminated_with_orphan()); } bool been_waited_for() const {return been_waited_for_;} void set_been_waited_for(); bool orphan() const {return orphan_;} void set_orphan() {orphan_ = true;} /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ virtual std::string d_internal(const int level = 0) const; private: /** * Exit status of the process. * 0..255 if the process exited cleanly and the parent successfully waited for it. * -1, if the process did not exit cleanly (died due to a signal) or the parent has not * waited for it yet. */ int exit_status_ = -1; ExecedProcess *exec_point_ {}; int on_finalized_ack_id_ = -1; bool been_waited_for_ = false; bool orphan_ = false; bool has_orphan_descendant_ = false; int on_finalized_ack_fd_ = -1; virtual void disable_shortcutting_only_this(const std::string &reason, const Process *p = NULL) { (void) reason; /* unused */ (void) p; /* unused */ } DISALLOW_COPY_AND_ASSIGN(ForkedProcess); }; } /* namespace firebuild */ #endif // FIREBUILD_FORKED_PROCESS_H_ firebuild-0.8.2/src/firebuild/hash.cc000066400000000000000000000201561447164520700174720ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/hash.h" #include #include #include #include #include #include #include #include #include #include #include "firebuild/base64.h" #include "firebuild/debug.h" #include "firebuild/file_name.h" #include "firebuild/utils.h" namespace firebuild { static const off_t kHashingBufSize = 4096; void Hash::set_from_data(const void *data, ssize_t size) { TRACKX(FB_DEBUG_HASH, 0, 1, Hash, this, ""); /* xxhash's doc says: * "Streaming functions [...] is slower than single-call functions, due to state management." * Let's take the faster path. */ hash_ = XXH3_128bits(data, size); } static ssize_t pread_checked_eof(int fd, char* buf, const off_t count, const off_t offset) { ssize_t read_bytes = TEMP_FAILURE_RETRY(pread(fd, buf, count, offset)); if (read_bytes < 0) { return -1; } else if (read_bytes < count && TEMP_FAILURE_RETRY( pread(fd, buf + read_bytes, count - read_bytes, offset + read_bytes)) != 0) { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of regular file: pread could not read the whole file"); return -1; } else { return read_bytes; } } bool Hash::set_from_fd_pread(int fd, off_t* const size) { TRACKX(FB_DEBUG_HASH, 0, 1, Hash, this, "fd=%d, size=%" PRIoff, fd, *size); char buf[kHashingBufSize]; if (*size <= kHashingBufSize) { ssize_t read_bytes = pread_checked_eof(fd, buf, *size, 0); if (read_bytes == -1) { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of regular file: pread failed"); return false; } else { if (read_bytes != *size) { *size = read_bytes; } set_from_data(buf, *size); return true; } } else { /* File does not fit in the buffer, needs multiple reads. */ #ifdef XXH_INLINE_ALL XXH3_state_t state_struct; XXH3_state_t* state = &state_struct; #else XXH3_state_t* state = XXH3_createState(); #endif if (XXH3_128bits_reset(state) == XXH_ERROR) { abort(); } off_t pos = 0; while (pos < *size) { const off_t to_read = std::min(kHashingBufSize, *size - pos); ssize_t read_bytes = pread_checked_eof(fd, buf, to_read, pos); if (read_bytes == -1 || (XXH3_128bits_update(state, buf, read_bytes) == XXH_ERROR)) { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of regular file: pread failed"); #ifndef XXH_INLINE_ALL XXH3_freeState(state); #endif return false; } pos += read_bytes; if (read_bytes < to_read) { *size = pos; } } hash_ = XXH3_128bits_digest(state); #ifndef XXH_INLINE_ALL XXH3_freeState(state); #endif } return true; } bool Hash::set_from_fd(int fd, const struct stat64 *stat_ptr, bool *is_dir_out, off_t *size_out) { TRACKX(FB_DEBUG_HASH, 0, 1, Hash, this, "fd=%d, stat=%s", fd, D(stat_ptr)); struct stat64 st_local; if (!stat_ptr && fstat64(fd, &st_local) == -1) { fb_perror("fstat"); return false; } const struct stat64 *st = stat_ptr ? stat_ptr : &st_local; if (S_ISREG(st->st_mode)) { off_t size = st->st_size; /* Compute the hash of a regular file. */ if (is_dir_out != NULL) { *is_dir_out = false; } if (size_out != NULL) { *size_out = size; } if (size < 0) { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of file with negative size"); return false; } else if (size == 0) { set_from_data(nullptr, 0); return true; } else { /* st->st_size > 0 */ void *map_addr; map_addr = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); if (map_addr == MAP_FAILED) { if (errno == ENODEV) { return set_from_fd_pread(fd, size_out ? size_out : &size); } else { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of regular file: mmap failed"); return false; } } else { } set_from_data(map_addr, size); munmap(map_addr, size); } return true; } else if (S_ISDIR(st->st_mode)) { /* Compute the hash of a directory. Its listing is sorted, and * concatenated using '\0' as a terminator after each entry. Then * this string is hashed. */ // FIXME place d_type in the string, too? if (is_dir_out != NULL) { *is_dir_out = true; } /* Quoting fdopendir(3): * "After a successful call to fdopendir(), fd is used internally by the * implementation, and should not otherwise be used by the application." * and closedir(3): * "A successful call to closedir() also closes the underlying file descriptor" * * It would be an unconventional and hard to use API for this method to close the passed fd. * Not calling closedir() on the other hand could leave garbage in the memory, and * the caller of this method directly calling close() would also go against the manpage. * If we call closedir() and the caller also calls a failing close() then it's prone to * raceable errors if one day we go multithreaded or so. * * So work on a duplicated fd and eventually close that, while keeping the original fd opened. */ DIR *dir = fdopendir(dup(fd)); if (dir == NULL) { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of directory: fdopendir failed"); return false; } std::vector listing; struct dirent *entry; while ((entry = readdir(dir)) != NULL) { listing.push_back(entry->d_name); } closedir(dir); std::sort(listing.begin(), listing.end()); std::string concat; for (const auto& entry : listing) { concat += entry; concat += '\0'; } set_from_data(concat.c_str(), concat.size()); return true; } else { FB_DEBUG(FB_DEBUG_HASH, "Cannot compute hash of special file"); return false; } } bool Hash::set_from_file(const FileName *filename, const struct stat64 *stat_ptr, bool *is_dir_out, off_t *size_out) { TRACKX(FB_DEBUG_HASH, 0, 1, Hash, this, "filename=%s", D(filename)); int fd; fd = open(filename->c_str(), O_RDONLY); if (fd == -1) { if (FB_DEBUGGING(FB_DEBUG_HASH)) { FB_DEBUG(FB_DEBUG_HASH, "File " + d(filename)); fb_perror("open"); } return false; } if (!set_from_fd(fd, stat_ptr, is_dir_out, size_out)) { close(fd); return false; } if (FB_DEBUGGING(FB_DEBUG_HASH)) { FB_DEBUG(FB_DEBUG_HASH, "xxh64sum: " + d(filename) + " => " + d(this)); } close(fd); return true; } void Hash::set(XXH128_hash_t value) { TRACKX(FB_DEBUG_HASH, 0, 1, Hash, this, ""); hash_ = value; } /** * Get the ASCII representation. * * See the class's documentation for the exact format. */ void Hash::to_ascii(char *out) const { XXH128_canonical_t canonical; XXH128_canonicalFromHash(&canonical, hash_); Base64::encode(canonical.digest, out, sizeof(canonical.digest)); out[kAsciiLength] = '\0'; } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const Hash& hash, const int level) { (void)level; /* unused */ return hash.to_ascii(); } std::string d(const Hash *hash, const int level) { if (hash) { return d(*hash, level); } else { return "{Hash NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/hash.h000066400000000000000000000120471447164520700173340ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_HASH_H_ #define FIREBUILD_HASH_H_ #include #include #include #include "firebuild/base64.h" #include "firebuild/file_name.h" namespace firebuild { /** * A Hash object represents the binary hash of some blob, * and provides methods to compute the hash, and convert to/from * an ASCII representation that can be used in filenames. * * The binary hash is the raw (i.e. architecture dependent XXH128_hash_t) * version of the XXH128 sum. * * The ASCII hash is the base64-like representation of the canonical * representation, in ceil(128/6) = 22 characters. The two * non-alphanumeric characters of our base64 alphabet are '+' and '^' and * none of the characters are at their usual position. * The characters are reordered compared to the original base64 mapping. * They are ordered by increasing ASCII code to have a set of hash values * and their ASCII representation sort the same way. * No trailing '=' signs to denote the partial block. * * Command line equivalent: * xxh128sum | xxd -r -p | base64 | cut -c1-22 | tr A-Za-z0-9+/ +0-9A-Z^a-z */ class Hash { public: Hash() : hash_() {} explicit Hash(XXH128_hash_t value) : hash_(value) {} bool operator==(const Hash& other) const { return hash_.high64 == other.hash_.high64 && hash_.low64 == other.hash_.low64; } bool operator!=(const Hash& other) const { return hash_.high64 != other.hash_.high64 || hash_.low64 != other.hash_.low64; } static size_t hash_size() {return hash_size_;} /** ASCII representation length without the trailing '\0' */ static const size_t kAsciiLength {22}; /** * Set the hash from the given buffer. */ void set_from_data(const void *data, ssize_t size); /** * Set the hash from the given opened file descriptor. * The file seek position (read/write offset) is irrelevant. * * If fd is a directory, its sorted listing is hashed. * * If stat_ptr is not NULL then it must contain fd's stat data. This can save an fstat() call. * * @param fd The file descriptor * @param stat_ptr Optionally the stat data of fd * @param is_dir_out Optionally store here whether fd refers to a directory * @param size_out Optionally store the file's size (only if it's a regular file) * @return Whether succeeded */ bool set_from_fd(int fd, const struct stat64 *stat_ptr, bool *is_dir_out, off_t *size_out = NULL); /** * Set the hash from the given file or directory. * * If a directory is specified, its sorted listing is hashed. * * @param filename The filename * @param stat_ptr Optionally the stat data of fd * @param is_dir_out Optionally store here whether filename refers to a directory * @param size_out Optionally store the file's size (only if it's a regular file) * @return Whether succeeded */ bool set_from_file(const FileName *filename, const struct stat64 *stat_ptr, bool *is_dir_out = NULL, off_t *size_out = NULL); /** * Sets the hash value directly from the given value. * No hash computation takes place. */ void set(XXH128_hash_t); XXH128_hash_t get() const { return hash_; } const XXH128_hash_t *get_ptr() const { return &hash_; } void to_ascii(char *out) const; std::string to_ascii() const { char ascii[Hash::kAsciiLength + 1]; to_ascii(ascii); return std::string(ascii); } static bool valid_ascii(const char* const str) { return Base64::valid_ascii(str, kAsciiLength); } private: /** * Set the hash from the given opened file descriptor with read operations. * The file seek position (read/write offset) is irrelevant. * * @param fd The file descriptor * @param size expected file size, updated if the size is smaller * @return Whether succeeded */ bool set_from_fd_pread(int fd, off_t* const size); static const unsigned int hash_size_ = sizeof(XXH128_hash_t); XXH128_hash_t hash_; }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const Hash& hash, const int level = 0); std::string d(const Hash *hash, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_HASH_H_ firebuild-0.8.2/src/firebuild/hash_cache.cc000066400000000000000000000362771447164520700206300ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/hash_cache.h" #include "firebuild/debug.h" #include "firebuild/blob_cache.h" #include "firebuild/file_info.h" #include "firebuild/file_name.h" namespace firebuild { /* singleton */ HashCache *hash_cache; /* Update the stat information in the cache. Forget the hash if the stat info changed. */ static bool update_statinfo(const FileName* path, int fd, const struct stat64 *stat_ptr, HashCacheEntry *entry) { TRACKX(FB_DEBUG_HASH, 1, 1, HashCacheEntry, entry, "path=%s, fd=%d, stat=%s", D(path), fd, D(stat_ptr)); if (path->is_in_read_only_location() && entry->info.type() != DONTKNOW) { /* Assume that for system locations the statinfo never changes. */ return true; } if (!path->is_in_read_only_location()) { /* For system locations, as per the previous condition, we're updating a brand new record, i.e. * type=DONTKNOW. For non-system locations, we're updating a brand new record or an old ISREG or * ISDIR type, there's no negative caching for non-system locations so the old type cannot be * NOTEXIST. */ assert(entry->info.type() == DONTKNOW || entry->info.type() == ISREG || entry->info.type() == ISDIR); } struct stat64 st_local; if (!stat_ptr && (fd >= 0 ? fstat64(fd, &st_local) : stat64(path->c_str(), &st_local)) == -1) { entry->info.set_type(NOTEXIST); entry->is_stored = false; return true; } const struct stat64 *st = stat_ptr ? stat_ptr : &st_local; if (!S_ISREG(st->st_mode) && !S_ISDIR(st->st_mode)) { entry->info.set_type(NOTEXIST); entry->is_stored = false; return true; } if (((S_ISREG(st->st_mode) && entry->info.type() == ISREG) || (S_ISDIR(st->st_mode) && entry->info.type() == ISDIR)) && (S_ISDIR(st->st_mode) || st->st_size == entry->info.size()) && st->st_mtim.tv_sec == entry->mtime.tv_sec && st->st_mtim.tv_nsec == entry->mtime.tv_nsec && st->st_ino == entry->inode) { /* Metadata is the same. Assume contents didn't change, nothing else to do. */ return true; } /* Metadata changed. Update entry, remove hash. */ entry->mtime = st->st_mtim; entry->inode = st->st_ino; entry->is_stored = false; if (S_ISREG(st->st_mode)) { entry->info.set_type(ISREG); entry->info.set_mode_bits(st->st_mode & 07777, 07777 /* we know all the mode bits */); entry->info.set_size(st->st_size); } else { entry->info.set_type(ISDIR); entry->info.set_mode_bits(st->st_mode & 07777, 07777 /* we know all the mode bits */); entry->info.set_size(-1); } entry->info.set_hash(nullptr); return true; } /* Update the hash, maybe assuming that the statinfo is up-to-date. */ static bool update_hash(const FileName* path, int fd, const struct stat64 *stat_ptr, HashCacheEntry *entry, bool store, off_t* stored_bytes, bool skip_statinfo_update) { TRACKX(FB_DEBUG_HASH, 1, 1, HashCacheEntry, entry, "path=%s, fd=%d, stat=%s, store=%s, skip_statinfo_update=%s", D(path), fd, D(stat_ptr), D(store), D(skip_statinfo_update)); /* This is used by file_info_matches() for a two-phase update, checking in between whether the * stat info matches. We want to delay computing the checksum until it's necessary, but we also * want to avoid stat()ing the file twice. */ if (!skip_statinfo_update) { update_statinfo(path, fd, stat_ptr, entry); } /* If there's no such file or directory then there's nothing to hash. */ if (entry->info.type() == NOTEXIST) { return true; } assert(entry->info.type() == ISREG || entry->info.type() == ISDIR); if (store && !entry->is_stored) { if (entry->info.type() != ISREG) { // FIXME handle if the file type has just changed from regular to something else return false; } /* We need to not only remember this entry in this hash cache, but also store the underlying * file in the blob cache. So use blob_cache's methods which in turn will compute the hash. * The file needs to be a regular file, cannot be a directory. */ Hash hash; bool ret = blob_cache->store_file(path, 0, fd, 0, entry->info.size(), &hash); if (ret) { entry->info.set_type(ISREG); // FIXME if hash_known_() then we could verify that it didn't change entry->info.set_hash(&hash); entry->is_stored = true; *stored_bytes = entry->info.size(); } return ret; } else { if (entry->info.hash_known()) { /* If the hash is known then it's up-to-date because otherwise update_statinfo() would have * cleared it. */ if (store) { /* The entry would be stored if it was not already in the cache. */ *stored_bytes = entry->info.size(); } return true; } /* We don't store the file in the blob cache, so just compute the hash directly. * The file can be a regular file or a directory. */ Hash hash; bool is_dir; bool ret; /* In order to save an fstat64() call in set_from_fd(), create a "fake" stat result here. We * know that it's a regular file, we know its size, and the rest are irrelevant. */ struct stat64 st; st.st_mode = entry->info.type() == ISREG ? S_IFREG : S_IFDIR; st.st_size = entry->info.size(); if (fd == -1) { ret = hash.set_from_file(path, &st, &is_dir); } else { ret = hash.set_from_fd(fd, &st, &is_dir); } // FIXME verify that is_dir matches entry->info.type() if (ret) { entry->info.set_hash(hash); if (store) { /* The entry would be stored if it was not already in the cache. */ *stored_bytes = entry->info.size(); } } return ret; } } const HashCacheEntry* HashCache::get_entry_with_statinfo(const FileName* path, int fd, const struct stat64 *stat_ptr) { TRACK(FB_DEBUG_HASH, "path=%s, fd=%d, stat=%s", D(path), fd, D(stat_ptr)); if (db_.count(path) > 0) { HashCacheEntry& entry = db_[path]; if (!update_statinfo(path, fd, stat_ptr, &entry)) { db_.erase(path); return ¬exist_; } if (!path->is_in_read_only_location() && entry.info.type() == NOTEXIST) { /* For non-system locations don't store negative entries. */ db_.erase(path); return ¬exist_; } return &entry; } else { struct HashCacheEntry new_entry {FileInfo(DONTKNOW)}; if (!update_statinfo(path, fd, stat_ptr, &new_entry)) { return ¬exist_; } if (!path->is_in_read_only_location() && new_entry.info.type() == NOTEXIST) { /* For non-system locations don't store negative entries. */ return ¬exist_; } db_[path] = new_entry; return &db_[path]; } } const HashCacheEntry* HashCache::get_entry_with_statinfo_and_hash(const FileName* path, int max_writers, int fd, const struct stat64 *stat_ptr, bool store, off_t* stored_bytes, bool skip_statinfo_update) { TRACK(FB_DEBUG_HASH, "path=%s, max_writers=%d, fd=%d, stat=%s, store=%s, skip_update=%s", D(path), max_writers, fd, D(stat_ptr), D(store), D(skip_statinfo_update)); if (path->writers_count() > max_writers) { /* The file could be written while calculating the hash, don't take that risk. */ return &dontknow_; } if (db_.count(path) > 0) { HashCacheEntry& entry = db_[path]; if (!update_hash(path, fd, stat_ptr, &entry, store, stored_bytes, skip_statinfo_update)) { db_.erase(path); return ¬exist_; } if (!path->is_in_read_only_location() && entry.info.type() == NOTEXIST) { /* For non-system locations don't store negative entries. */ db_.erase(path); return ¬exist_; } return &entry; } else { struct HashCacheEntry new_entry {FileInfo(DONTKNOW)}; if (!update_hash(path, fd, stat_ptr, &new_entry, store, stored_bytes, skip_statinfo_update)) { return ¬exist_; } if (!path->is_in_read_only_location() && new_entry.info.type() == NOTEXIST) { /* For non-system locations don't store negative entries. */ return ¬exist_; } db_[path] = new_entry; return &db_[path]; } } bool HashCache::get_statinfo(const FileName* path, bool *is_dir, ssize_t *size) { TRACK(FB_DEBUG_HASH, "path=%s", D(path)); if (path->is_in_ignore_location()) { return false; } else if (path->is_in_read_only_location()) { /* For system files go through our cache, as if we were interested in the hash too. */ const HashCacheEntry *entry = get_entry_with_statinfo(path, -1, nullptr); if (entry->info.type() == NOTEXIST) { return false; } if (is_dir) { *is_dir = entry->info.type() == ISDIR; } if (size && entry->info.type() != ISDIR) { *size = entry->info.size(); } return true; } else { /* For non-system files just stat() the file, completely bypassing the cache. Looking up and * updating the cache entry would just be a waste of CPU time since next time (when we do care * about the checksum) we'll have to update it anyway. */ struct stat64 st; if (stat64(path->c_str(), &st) == -1 || (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode))) { return false; } if (is_dir) { *is_dir = S_ISDIR(st.st_mode); } if (size && !S_ISDIR(st.st_mode)) { *size = st.st_size; } return true; } } bool HashCache::get_hash(const FileName* path, int max_writers, Hash *hash, bool *is_dir, ssize_t *size, int fd, const struct stat64 *stat_ptr) { TRACK(FB_DEBUG_HASH, "path=%s, max_writers=%d, fd=%d, stat=%s", D(path), max_writers, fd, D(stat_ptr)); if (path->is_in_ignore_location()) { return false; } const HashCacheEntry *entry = get_entry_with_statinfo_and_hash(path, max_writers, fd, stat_ptr, false, nullptr); if (entry->info.type() == NOTEXIST || entry->info.type() == DONTKNOW) { return false; } if (is_dir) { *is_dir = entry->info.type() == ISDIR; } if (entry->info.type() != ISDIR && size) { *size = entry->info.size(); } *hash = entry->info.hash(); return true; } bool HashCache::store_and_get_hash(const FileName* path, int max_writers, Hash *hash, off_t* stored_bytes, int fd, const struct stat64 *stat_ptr) { TRACK(FB_DEBUG_HASH, "path=%s, max_writers=%d, fd=%d, stat=%s", D(path), max_writers, fd, D(stat_ptr)); if (path->is_in_ignore_location()) { return false; } const HashCacheEntry *entry = get_entry_with_statinfo_and_hash(path, max_writers, fd, stat_ptr, true, stored_bytes); if (!entry) { return false; } *hash = entry->info.hash(); return true; } bool HashCache::file_info_matches(const FileName *path, const FileInfo& query) { TRACK(FB_DEBUG_HASH, "path=%s, query=%s", D(path), D(query)); if (path->is_in_ignore_location()) { /* Information about files in the ignore locations should not be stored in the cache. * Return false to not use this cache entry, while we could return true, because we should * not care. */ return false; } const HashCacheEntry *entry = get_entry_with_statinfo(path, -1, nullptr); /* We do have an up-to-date stat information now. Check if the query matches it. */ switch (query.type()) { case DONTKNOW: assert(0 && "shouldn't query the HashCache to see if matches"); return true; case EXIST: if (entry->info.type() == NOTEXIST) { return false; } break; case NOTEXIST: return (entry->info.type() == NOTEXIST); case NOTEXIST_OR_ISREG: if (entry->info.type() == NOTEXIST) { return true; } else if (entry->info.type() == ISREG) { if (query.size() >= 0 && query.size() != entry->info.size()) { return false; } } else { return false; } break; case ISREG: if (entry->info.type() != ISREG) { return false; } if (query.size() >= 0 && query.size() != entry->info.size()) { return false; } break; case ISDIR: if (entry->info.type() != ISDIR) { return false; } break; } if ((query.mode() & query.mode_mask()) != (entry->info.mode() & query.mode_mask())) { return false; } /* Everything matches so far. If the query doesn't contain a hash then it's a match. */ if (!query.hash_known()) { return true; } assert(query.type() == ISREG || query.type() == ISDIR || query.type() == NOTEXIST_OR_ISREG); assert((query.type() == NOTEXIST_OR_ISREG && entry->info.type() == ISREG) || entry->info.type() == query.type()); /* We need to compare the hash. The current cache entry does not necessarily contain this * information, because it's expensive to compute it so we defer it as long as possible. But if * the entry already contains it then save some time by not looking it up in the cache again. */ if (!entry->info.hash_known()) { entry = get_entry_with_statinfo_and_hash(path, 0, -1, nullptr, false, nullptr, true /* don't stat again */); if ((entry->info.type() != ISREG && entry->info.type() != ISDIR) || !entry->info.hash_known()) { /* Could not get the hash possibly because the file/directory is open for writing. */ return false; } } return entry->info.hash() == query.hash(); } const HashCacheEntry HashCache::notexist_ {FileInfo(NOTEXIST)}; const HashCacheEntry HashCache::dontknow_ {FileInfo(DONTKNOW)}; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const HashCacheEntry& hce, const int level) { (void)level; /* unused */ return std::string("{HashCacheEntry info=") + d(hce.info) + ", mtime={" + d(hce.mtime.tv_sec) + "," + d(hce.mtime.tv_nsec) + "}" + ", inode=" + d(hce.inode) + ", is_stored=" + d(hce.is_stored) + "}"; } std::string d(const HashCacheEntry *hce, const int level) { if (hce) { return d(*hce, level); } else { return "{HashCacheEntry NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/hash_cache.h000066400000000000000000000202661447164520700204610ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_HASH_CACHE_H_ #define FIREBUILD_HASH_CACHE_H_ #include #include #include #include #include #include "firebuild/file_info.h" #include "firebuild/file_name.h" #include "firebuild/hash.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { struct HashCacheEntry { FileInfo info {}; struct timespec mtime {}; ino_t inode {}; /* skip device, it's unlikely to change */ bool is_stored {}; /* it's known to be present in the blob cache because we stored it earlier */ }; /** * This class implements a global (that is, once per firebuild process) in-memory cache of file * hashes. * * This cache stores the hash of files and directories that are found, along with some statinfo that * lets determine if the hash needs to be refreshed. * * Internally, different strategies are used for files under system (read-only) locations (as per * the config file) and for non-system (read-write) locations. The public API completely hides this * and provides a unified interface for both types. * * For system locations we expect that they don't change during Firebuild's lifetime. Once cached, * the actual file is no longer checked. Nonexisting files are also cached. * * (Note though: Currently as soon as someone asks for the file's size or permissions, we compute * its checksum too. It's very unlikely that a build procedure stat()s a system file successfully * and then does not read it. In turn, the code becomes simpler, it doesn't have to cope with files * whose size is already cached but the checksum isn't yet. This might change in the future.) * * For non-system locations we always begin by stat()ing the file, and the cached checksum is * forgotten in case of statinfo mismatch. Accordingly, negative entries aren't cached, it just * wouldn't make sense. */ class HashCache { public: HashCache() {} ~HashCache(); /** * Get some stat information (currently the file type and size) from the cache. This method * doesn't compute and doesn't return the hash. * * @param path file's path * @param[out] is_dir optionally store if path is a dir * @param[out] size optionally store the size if it's a regular file * @return false if not a regular file or directory */ bool get_statinfo(const FileName* path, bool *is_dir, ssize_t *size); /** * Get some stat information (currently the file type and size) as well as the hash from the * cache. * * @param path file's path * @param max_writers maximum allowed number of writers to this file * @param[out] hash hash to retrive/calculate * @param[out] is_dir optionally store if path is a dir * @param[out] size optionally store the size if it's a regular file * @param fd if >= 0 then read the file from there * @param stat_ptr optionally the file's parameters already stat()'ed * @return false if not a regular file or directory */ bool get_hash(const FileName* path, int max_writers, Hash *hash, bool *is_dir = nullptr, ssize_t *size = nullptr, int fd = -1, const struct stat64 *stat_ptr = nullptr); /** * Return the hash of a regular file. Also store this file in the blob cache. * * @param path file's path * @param max_writers maximum allowed number of writers to this file * @param[out] hash hash to retrive/calculate * @param[out] stored_bytes bytes stored to the blob cache * @param fd if >= 0 then read the file from there * @param stat_ptr optionally the file's parameters already stat()'ed * @return false if not a regular file or directory */ bool store_and_get_hash(const FileName* path, int max_writers, Hash *hash, off_t* stored_bytes, int fd, const struct stat64 *stat_ptr); /** * Check if the given FileInfo query matches the file system. * * @param path file's path * @param query the query to match against * @return whether the query matches the file */ bool file_info_matches(const FileName *path, const FileInfo& query); private: tsl::hopscotch_map db_ = {}; /** * Returns an up-to-date HashCacheEntry corresponding to the given file. * * It's either of type NOTEXIST if path doesn't correspond to a regular file or directory, or of * type ISREG or ISDIR containing some stat information (currently the size in case of ISREG). * * The hash is also returned if it's cached, but if it wasn't cached then it will not be present * in the returned structure, it is not computed by this method. * * The returned pointer is always non-NULL, readonly, and only valid until the next operation on * HashCache. * * @param path file's path * @param fd if >= 0 then read the file from there * @param stat_ptr optionally the file's parameters already stat()'ed * @return the requested information about the file */ const HashCacheEntry* get_entry_with_statinfo(const FileName* path, int fd, const struct stat64 *stat_ptr); /** * Returns an up-to-date HashCacheEntry corresponding to the given file. * * It's either of type NOTEXIST if path doesn't correspond to a regular file or directory, or of * type ISREG or ISDIR containing some stat information (currently the size in case of ISREG) and * the hash. * * The returned pointer is always non-NULL, readonly, and only valid until the next operation on * HashCache. * * @param path file's path * @param max_writers maximum allowed number of writers to this file * @param fd if >= 0 then read the file from there * @param stat_ptr optionally the file's parameters already stat()'ed * @param store whether to store the file in the blob cache * @param[out] stored_bytes bytes stored to the blob cache * @param skip_statinfo_update assume that the stat info is up-to-date * @return the requested information about the file */ const HashCacheEntry* get_entry_with_statinfo_and_hash(const FileName* path, int max_writers, int fd, const struct stat64 *stat_ptr, bool store, off_t* stored_bytes, bool skip_statinfo_update = false); /** * A singleton structure representing a file system path that does not point to a regular file or * directory. get_entry_...() might return its address. */ static const HashCacheEntry notexist_; /** * A singleton structure representing a file system path with an unknown state for the supervisor. * get_entry_...() might return its address. */ static const HashCacheEntry dontknow_; DISALLOW_COPY_AND_ASSIGN(HashCache); }; /* singleton */ extern HashCache *hash_cache; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const HashCacheEntry& hce, const int level = 0); std::string d(const HashCacheEntry *hce, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_HASH_CACHE_H_ firebuild-0.8.2/src/firebuild/linear_buffer.h000066400000000000000000000127011447164520700212110ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /** * Linear buffer implementation optimized for minimizing memory reallocations. */ #ifndef FIREBUILD_LINEAR_BUFFER_H_ #define FIREBUILD_LINEAR_BUFFER_H_ #include #include #include #include "common/platform.h" #include "firebuild/cxx_lang_utils.h" #include "firebuild/debug.h" namespace firebuild { class LinearBuffer { public: LinearBuffer() : size_(8 * 1024), buffer_(reinterpret_cast(malloc(size_))), data_start_offset_(0), length_(0) {} ~LinearBuffer() {free(buffer_);} const char * data() const { return &buffer_[data_start_offset_]; } size_t length() const {return length_;} /** * Read data from fd and append it to the buffer. * * Note that in howmuch < 0 case the input buffer may not contain all the data the writer on * the other side has written. * * @param fd file descriptior to read from * @param howmuch number of bytes to be read, or in case howmuch is < 0, then read all bytes from * the input buffer * @return number of bytes read */ ssize_t read(int fd, ssize_t howmuch) { TRACK(FB_DEBUG_COMM, "fd=%s, howmuch=%" PRIssize, D_FD(fd), howmuch); assert_cmp(howmuch, !=, 0); if (howmuch >= 0) { /* Read at most the specified amount, in one step. (Note: fd is nonblocking.) */ ensure_space(howmuch); auto received = ::read(fd, &buffer_[data_start_offset_ + length_], howmuch); if (received > 0) { length_ += received; } return received; } else { /* Read as much as we can. (Note: fd is nonblocking.) * Try to use as few system calls as possible on average, see #417. * So, begin with a reasonably large read() that will most often result in a short read() * and then this is the only syscall we needed to perform. */ ensure_space(8 * 1024); /* Now we have at least 8kB of free space to read to, but maybe even more. * Try to read as much as we can, it cannot hurt. */ const ssize_t attempt1 = size_ - data_start_offset_ - length_; auto received1 = ::read(fd, &buffer_[data_start_offset_ + length_], attempt1); if (received1 <= 0) { /* EOF, or nothing to read right now, or other error. */ return received1; } length_ += received1; if (received1 < attempt1) { /* Short read: return what we already have. */ return received1; } /* Full read: need to continue reading. * We could expand the buffer and read in a loop until we have everything. But maybe let's * just query the incoming data size and read the rest in one step, using two syscalls. */ const ssize_t attempt2 = readable_bytes(fd); if (attempt2 <= 0) { /* EOF, or nothing to read right now, or other error. Don't report this, report what we * read in the previous step. */ return received1; } ensure_space(attempt2); auto received2 = ::read(fd, &buffer_[data_start_offset_ + length_], attempt2); if (received2 <= 0) { /* EOF, or nothing to read right now, or other error. Don't report this, report what we * read in the previous step. Or can we assert that this never happens? */ return received1; } length_ += received2; return received1 + received2; } } /** Discard howmuch bytes from the beginning of the data. */ void discard(const size_t howmuch) { TRACK(FB_DEBUG_COMM, "howmuch=%" PRIsize, howmuch); assert_cmp(howmuch, <=, length_); length_ -= howmuch; if (length_ == 0) { data_start_offset_ = 0; } else { data_start_offset_ += howmuch; } } private: size_t size_; char * buffer_; size_t data_start_offset_; size_t length_; void ensure_space(ssize_t howmuch) { TRACK(FB_DEBUG_COMM, "howmuch=%" PRIssize, howmuch); assert_cmp(howmuch, >=, 0); if (data_start_offset_ > 256 * 1024) { /* In the unlucky case of not processing all the data for many read cycles move it to the * beginning to the buffer to don't inflate the buffer unnecessarily. */ memmove(buffer_, buffer_ + data_start_offset_, length_); data_start_offset_ = 0; } const size_t needed_size = data_start_offset_ + length_ + howmuch; if (size_ < needed_size) { size_ = (needed_size > size_ * 2) ? needed_size : size_ * 2; buffer_ = reinterpret_cast(realloc(buffer_, size_)); } } ssize_t readable_bytes(int fd) { int n; if (ioctl(fd, FIONREAD, &n) < 0) { return -1; } else { return n; } } DISALLOW_COPY_AND_ASSIGN(LinearBuffer); }; } /* namespace firebuild */ #endif // FIREBUILD_LINEAR_BUFFER_H_ firebuild-0.8.2/src/firebuild/message_processor.cc000066400000000000000000001531071447164520700222750ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/message_processor.h" #include #include #include #include #include #include #include #include "common/firebuild_common.h" #include "firebuild/config.h" #include "firebuild/debug.h" #include "firebuild/connection_context.h" #include "firebuild/epoll.h" #include "firebuild/exe_matcher.h" #include "firebuild/file_name.h" #include "firebuild/execed_process.h" #include "firebuild/execed_process_cacher.h" #include "firebuild/pipe.h" #include "firebuild/pipe_recorder.h" #include "firebuild/process.h" #include "firebuild/process_factory.h" #include "firebuild/process_debug_suppressor.h" #include "firebuild/process_tree.h" #include "firebuild/process_fbb_adaptor.h" #include "firebuild/utils.h" #include "./fbbcomm.h" #include "firebuild/fbbfp.h" #include "firebuild/fbbstore.h" namespace firebuild { static void reject_exec_child(int fd_conn) { FBBCOMM_Builder_scproc_resp sv_msg; sv_msg.set_dont_intercept(true); sv_msg.set_shortcut(false); send_fbb(fd_conn, 0, reinterpret_cast(&sv_msg)); } void MessageProcessor::accept_exec_child(ExecedProcess* proc, int fd_conn, int fd0_reopen) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, proc, "fd_conn=%s, fd0_reopen=%s", D_FD(fd_conn), D_FD(fd0_reopen)); /* We build up an FBB referring to this value, so it has to be valid until we send that FBB. */ const int stdin_fileno = STDIN_FILENO; FBBCOMM_Builder_scproc_resp sv_msg; /* These two have the same number of items and they correspond to each other. * "reopened_dups" is for the "reopen_fd_fifos" array in FBB "scproc_resp", * "fifo_fds" is for the ancillary data. */ std::vector reopened_dups = {}; std::vector fifo_fds = {}; proc_tree->insert(proc); proc->initialize(); if (shortcut_allow_list_matcher && !shortcut_allow_list_matcher->match(proc)) { proc->disable_shortcutting_only_this("Executable is not allowed to be shortcut"); execed_process_cacher->not_shortcutting(); } if (dont_intercept_matcher->match(proc)) { /* Executables that should not be intercepted. */ proc->disable_shortcutting_bubble_up("Executable set to not be intercepted"); execed_process_cacher->not_shortcutting(); sv_msg.set_dont_intercept(true); } else if (dont_shortcut_matcher->match(proc)) { if (quirks & FB_QUIRK_LTO_WRAPPER && proc->args().size() > 0 && proc->args()[0] == "make" && proc->parent_exec_point() && proc->parent_exec_point()->executable()->without_dirs() == "lto-wrapper" ) { FB_DEBUG(FB_DEBUG_PROC, "Allow shortcutting lto-wrapper's make (lto-wrapper quirk)"); } else { /* Executables that are known not to be shortcuttable. */ proc->disable_shortcutting_bubble_up("Executable set to be not shortcut"); execed_process_cacher->not_shortcutting(); } } /* Check for executables that we prefer not to shortcut. */ if (skip_cache_matcher->match(proc)) { proc->disable_shortcutting_only_this("Executable matches skip_cache"); execed_process_cacher->not_shortcutting(); } /* If we still potentially can, and prefer to cache / shortcut this process, * register the cacher object and calculate the process's fingerprint. */ if (proc->can_shortcut()) { if (!execed_process_cacher->fingerprint(proc)) { proc->disable_shortcutting_bubble_up("Could not fingerprint the process"); } } std::vector inherited_files = proc->inherited_files(); for (inherited_file_t& inherited_file : inherited_files) { if (inherited_file.type == FD_PIPE_OUT) { /* There may be incoming data from the (transitive) parent(s), drain it. * Do it before trying to shortcut. */ auto pipe = proc->get_fd(inherited_file.fds[0])->pipe(); assert(pipe); pipe->drain(); } } /* Try to shortcut the process. */ std::vector fds_appended_to, seekable_fds; std::vector seekable_fds_size; bool shortcutting_succeeded = proc->shortcut(&fds_appended_to); if (shortcutting_succeeded) { sv_msg.set_shortcut(true); sv_msg.set_exit_status(proc->fork_point()->exit_status()); sv_msg.set_fds_appended_to(fds_appended_to); if (fd0_reopen >= 0) { close(fd0_reopen); } } else { sv_msg.set_shortcut(false); /* parent forked, thus a new set of fds is needed to track outputs */ /* For popen(..., "w") pipes we couldn't reopen its stdin in the short-lived forked process, * so connect our Pipe object with the stdin of the child process here. * (The stdout side of a popen(..., "r") child is handled below by the generic * code that reopens all inherited outgoing pipes.) */ if (fd0_reopen >= 0) { fifo_fds.push_back(fd0_reopen); /* alloca()'s lifetime is the entire function, not just the brace-block. This is what we * need because the data has to live until the send_fbb() below. */ auto dups = reinterpret_cast( alloca(sizeof(FBBCOMM_Builder_scproc_resp_reopen_fd))); dups->init(); dups->set_fds(&stdin_fileno, 1); reopened_dups.push_back(reinterpret_cast(dups)); } // TODO(rbalint) skip reopening fd if parent's other forked processes closed the fd // without writing to it const int jobserver_fd_w = proc->jobserver_fd_w(); for (inherited_file_t& inherited_file : inherited_files) { if (inherited_file.type == FD_PIPE_OUT) { if (inherited_file.fds[0] == jobserver_fd_w && inherited_file.fds.size() == 1) { /* Skip reopening the jobserver pipe. */ continue; } auto file_fd_old = proc->get_shared_fd(inherited_file.fds[0]); auto pipe = file_fd_old->pipe(); assert(pipe); /* As per #689, reopening the pipes causes different behavior than without firebuild. With * firebuild, across an exec they no longer share the same "open file description" and thus * the fcntl flags. Perform this unduping from the exec parent, i.e. modify the FileFDs to * point to a new FileOFD. */ auto fds = proc->fds(); int fd = inherited_file.fds[0]; auto file_fd = std::make_shared(fd, file_fd_old->flags(), pipe, file_fd_old->opened_by()); (*fds)[fd] = file_fd; for (size_t i = 1; i < inherited_file.fds.size(); i++) { fd = inherited_file.fds[i]; auto file_fd_dup = std::make_shared(fd, file_fd, false); (*fds)[fd] = file_fd_dup; } /* Create a new unnamed pipe. */ int fifo_fd[2]; int ret = fb_pipe2(fifo_fd, file_fd->flags() & ~O_ACCMODE); (void)ret; assert(ret == 0); if (epoll->is_added_fd(fifo_fd[0])) { fifo_fd[0] = epoll->remap_to_not_added_fd(fifo_fd[0]); } bump_fd_age(fifo_fd[0]); /* The supervisor needs nonblocking fds for the pipes. */ fcntl(fifo_fd[0], F_SETFL, O_NONBLOCK); /* Find the recorders belonging to the parent process. We need to record to all those, * plus create a new recorder for ourselves (unless shortcutting is already disabled). */ auto recorders = proc->parent() ? pipe->proc2recorders[proc->parent_exec_point()] : std::vector>(); if (proc->can_shortcut()) { inherited_file.recorder = std::make_shared(proc); recorders.push_back(inherited_file.recorder); } pipe->add_fd1_and_proc(fifo_fd[0], file_fd.get(), proc, std::move(recorders)); FB_DEBUG(FB_DEBUG_PIPE, "reopening process' fd: "+ d(inherited_file.fds[0]) + " as new fd1: " + d(fifo_fd[0]) + " of " + d(pipe)); fifo_fds.push_back(fifo_fd[1]); /* alloca()'s lifetime is the entire function, not just the brace-block. This is what we * need because the data has to live until the send_fbb() below. * Calling alloca() from a loop is often frowned upon because it can quickly eat up the * stack. Here we only need a tiny amount of data, typically less than 10 integers in all * the alloca()d areas combined. */ auto dups = reinterpret_cast( alloca(sizeof(FBBCOMM_Builder_scproc_resp_reopen_fd))); dups->init(); dups->set_fds(inherited_file.fds); reopened_dups.push_back(reinterpret_cast(dups)); } else if (inherited_file.type == FD_FILE) { int fd = inherited_file.fds[0]; auto file_fd = proc->get_shared_fd(fd); /* The current offset won't matter for writes. */ if (!(file_fd->flags() & O_APPEND)) { seekable_fds.push_back(fd); seekable_fds_size.push_back(inherited_file.start_offset); } } sv_msg.set_seekable_fds(seekable_fds); sv_msg.set_seekable_fds_size(seekable_fds_size); } sv_msg.set_reopen_fds(reopened_dups); /* inherited_files was updated with the recorders, save the new version */ proc->set_inherited_files(inherited_files); if (debug_flags != 0) { sv_msg.set_debug_flags(debug_flags); } } /* Send "scproc_resp", possibly with attached fds to reopen. */ send_fbb(fd_conn, 0, reinterpret_cast(&sv_msg), fifo_fds.data(), fifo_fds.size()); /* Close the sides that we transferred to the interceptor. This includes the stdin of a * popen(... "w") child, as well as the inherited outgoing pipes of every process. */ for (int fd : fifo_fds) { close(fd); } } /* This is run when we've received both the parent's "popen_parent" and the child's "scproc_query" * message, no matter in what order they arrived. */ static void accept_popen_child(Process* unix_parent, const pending_popen_t *pending_popen) { ExecedProcess *proc = pending_popen->child; /* This is for the special treatment of the fd if the process does another popen(). */ unix_parent->AddPopenedProcess(pending_popen->fd, proc); /* The short-lived forked process was added in proc_new_process_msg() when "scproc_query" arrived. * * Now we create the Pipe object and register its file handles for the execed process. * * TODO We should ideally register it to new process's exec parent (the short-lived fork of the * popening process) too. However, it really doesn't matter. */ int up[2], down[2]; int fd_send_to_parent; int fd0_reopen = -1; int flags = pending_popen->type_flags; if (is_rdonly(flags)) { /* For popen(..., "r") (parent reads <- child writes) create only the parent-side backing Unix * pipe, and the Pipe object. The child-side backing Unix pipe will be created in * accept_exec_child() when reopening the inherited outgoing pipes. */ FB_DEBUG(FB_DEBUG_PROC, "This is a popen(..., \"r...\") child"); if (fb_pipe2(down, flags & ~O_ACCMODE) < 0) { assert(0 && "pipe2() failed"); } bump_fd_age(down[0]); bump_fd_age(down[1]); FB_DEBUG(FB_DEBUG_PROC, "down[0]: " + d_fd(down[0]) + ", down[1]: " + d_fd(down[1])); fd_send_to_parent = down[0]; if (!(flags & O_NONBLOCK)) { /* The supervisor needs nonblocking fds for the pipes. */ fcntl(down[1], F_SETFL, flags | O_NONBLOCK); } #ifdef __clang_analyzer__ /* Scan-build reports a false leak for the correct code. This is used only in static * analysis. It is broken because all shared pointers to the Pipe must be copies of * the shared self pointer stored in it. */ auto pipe = std::make_shared(down[1] /* server fd */, unix_parent); #else auto pipe = (new Pipe(down[1] /* server fd */, unix_parent))->shared_ptr(); #endif /* The reading side of this pipe is in the popening (parent) process. */ auto ffd0 = std::make_shared(pending_popen->fd /* client fd */, (flags & ~O_ACCMODE) | O_RDONLY, pipe->fd0_shared_ptr(), unix_parent /* creator */, true /* close_on_popen */); unix_parent->add_filefd(pending_popen->fd /* client fd */, ffd0); /* The writing side of this pipe is in the forked and the execed processes. * We're lazy and we don't register it for the forked process, no one cares. */ auto ffd1 = std::make_shared(STDOUT_FILENO /* client fd */, (flags & ~O_ACCMODE) | O_WRONLY, pipe->fd1_shared_ptr(), unix_parent /* creator */, false /* close_on_popen */); proc->add_filefd(STDOUT_FILENO /* client fd */, ffd1); } else { /* For popen(..., "w") (parent writes -> child reads) create both backing Unix unnamed * pipes, as well as the Pipe object handling them. */ FB_DEBUG(FB_DEBUG_PROC, "This is a popen(..., \"w...\") child"); if (fb_pipe2(up, flags & ~O_ACCMODE) < 0 || fb_pipe2(down, flags & ~O_ACCMODE) < 0) { assert(0 && "pipe2() failed"); } if (epoll->is_added_fd(up[0])) { up[0] = epoll->remap_to_not_added_fd(up[0]); } bump_fd_age(up[0]); bump_fd_age(up[1]); bump_fd_age(down[0]); bump_fd_age(down[1]); FB_DEBUG(FB_DEBUG_PROC, "up[0]: " + d_fd(up[0]) + ", up[1]: " + d_fd(up[1]) + ", down[0]: " + d_fd(down[0]) + ", down[1]: " + d_fd(down[1])); fd_send_to_parent = up[1]; if (!(flags & O_NONBLOCK)) { /* The supervisor needs nonblocking fds for the pipes. */ fcntl(up[0], F_SETFL, flags | O_NONBLOCK); fcntl(down[1], F_SETFL, flags | O_NONBLOCK); } #ifdef __clang_analyzer__ /* Scan-build reports a false leak for the correct code. This is used only in static * analysis. It is broken because all shared pointers to the Pipe must be copies of * the shared self pointer stored in it. */ auto pipe = std::make_shared(down[1] /* server fd */, unix_parent); #else auto pipe = (new Pipe(down[1] /* server fd */, unix_parent))->shared_ptr(); #endif /* The reading side of this pipe is in the forked and the execed processes. * We're lazy and we don't register it for the forked process, no one cares. */ auto ffd0 = std::make_shared(STDIN_FILENO /* client fd */, (flags & ~O_ACCMODE) | O_RDONLY, pipe->fd0_shared_ptr(), unix_parent /* creator */, false /* close_on_popen */); proc->add_filefd(STDIN_FILENO /* client fd */, ffd0); /* The (so far only) writing side of this pipe is in the popening (parent) process. */ auto ffd1 = std::make_shared(pending_popen->fd /* client fd */, (flags & ~O_ACCMODE) | O_WRONLY, pipe->fd1_shared_ptr(), unix_parent /* creator */, true /* close_on_popen */); unix_parent->add_filefd(pending_popen->fd /* client fd */, ffd1); auto recorders = std::vector>(); pipe->add_fd1_and_proc(up[0] /* server fd */, ffd1.get(), proc, recorders); /* This is an incoming pipe in the child process that needs to be reopened because we * couldn't catch the pipe() call inside popen() and thus we couldn't do it yet. * Add this to the "reopen_fd_fifos" array of "scproc_resp", and to the ancillary data. */ fd0_reopen = down[0]; } /* ACK the parent, using a "popen_fd" message with the fd attached as ancillary data. * Then close that fd. */ FBBCOMM_Builder_popen_fd msg; send_fbb(pending_popen->parent_conn, pending_popen->ack_num, reinterpret_cast(&msg), &fd_send_to_parent, 1); close(fd_send_to_parent); MessageProcessor::accept_exec_child(proc, pending_popen->child_conn, fd0_reopen); proc_tree->DropPendingPopen(unix_parent); unix_parent->set_has_pending_popen(false); } static void accept_fork_child(Process* parent, int parent_fd, int parent_ack, Process** child_ref, int pid, int child_fd, int child_ack) { TRACK(FB_DEBUG_PROC, "parent_fd=%s, parent_ack=%d, parent=%s pid=%d child_fd=%s child_ack=%d", D_FD(parent_fd), parent_ack, D(parent), pid, D_FD(child_fd), child_ack); auto proc = ProcessFactory::getForkedProcess(pid, parent); proc_tree->insert(proc); *child_ref = proc; ack_msg(parent_fd, parent_ack); ack_msg(child_fd, child_ack); } /** * Process message coming from interceptor * @param fb_conn file desctiptor of the connection */ static void proc_new_process_msg(const FBBCOMM_Serialized *fbbcomm_buf, uint16_t ack_id, int fd_conn, Process** new_proc) { TRACK(FB_DEBUG_PROC, "fd_conn=%s, ack_id=%d", D_FD(fd_conn), ack_id); int tag = fbbcomm_buf->get_tag(); if (tag == FBBCOMM_TAG_scproc_query) { auto ic_msg = reinterpret_cast(fbbcomm_buf); auto pid = ic_msg->get_pid(); auto ppid = ic_msg->get_ppid(); const char* ic_version = ic_msg->get_version(); if (ic_version && strcmp(ic_version, FIREBUILD_VERSION) != 0) { fb_error("Mismatched interceptor version: " + std::string(ic_version)); abort(); } Process *unix_parent = NULL; LaunchType launch_type = LAUNCH_TYPE_OTHER; int type_flags; Process *parent = NULL; std::vector>* fds = nullptr; /* Locate the parent in case of execve or alike. This includes the * case when the outermost intercepted process starts up (no * parent will be found) or when this outermost process does an * exec (an exec parent will be found then). */ parent = proc_tree->pid2proc(pid); if (parent) { /* This PID was already seen, i.e. this process is the result of an exec*(), * or a posix_spawn*() where we've already seen and processed the * "posix_spawn_parent" message. */ assert_cmp(parent->state(), !=, FB_PROC_FINALIZED); if (parent->state() == FB_PROC_TERMINATED) { fds = parent->pass_on_fds(); } else { /* Queue the ExecedProcess until parent's connection is closed */ fds = new std::vector>(); auto proc = ProcessFactory::getExecedProcess( ic_msg, parent, fds); proc_tree->QueueExecChild(parent->pid(), fd_conn, proc); *new_proc = proc; return; } } else if (ppid == getpid()) { /* This is the first intercepted process. */ parent = proc_tree->root(); fds = parent->pass_on_fds(); } else { /* Locate the parent in case of system/popen/posix_spawn, but not * when the first intercepter process starts up. */ unix_parent = proc_tree->pid2proc(ppid); if (!unix_parent) { /* The parent could not be found. There could be one or more statically linked binaries in * the exec() - fork() chain. There is not much the supervisor can do, with so much missing * information. Let the child continue unintercepted and notice the missing popen/system() * child later. */ reject_exec_child(fd_conn); return; } /* Verify that the child was expected and get inherited fds. */ std::vector args = ic_msg->get_arg_as_vector(); fds = unix_parent->pop_expected_child_fds(args, &launch_type, &type_flags); if (!fds) { fds = new std::vector>(); } if (unix_parent->posix_spawn_pending()) { /* This is a posix_spawn*() child, but we haven't yet seen and processed the * "posix_spawn_parent" message. Defer processing the child until "posix_spawn_parent" * is processed first. * Don't set the parent yet because we haven't created that ForkedProcess object yet. * Also don't set fds, we couldn't because that depends on the file actions. We'll * set these when handling the "posix_spawn_parent" message. */ auto proc = ProcessFactory::getExecedProcess( ic_msg, nullptr, nullptr); proc_tree->QueuePosixSpawnChild(ppid, fd_conn, proc); *new_proc = proc; delete fds; return; } /* This is a system or popen child. */ /* Add a ForkedProcess for the forked child we never directly saw. */ parent = new ForkedProcess(pid, ppid, unix_parent, fds); if (launch_type == LAUNCH_TYPE_POPEN) { /* The new exec child should not inherit the fd connected to the unix_parent's popen()-ed * stream. The said fd is not necessarily open. */ int child_fileno = is_wronly(type_flags) ? STDIN_FILENO : STDOUT_FILENO; parent->handle_force_close(child_fileno); /* The new exec child also does not inherit parent's popen()-ed fds. * See: glibc/libio/iopopen.c: * POSIX states popen shall ensure that any streams from previous popen() * calls that remain open in the parent process should be closed in the new * child process. [...] */ for (auto& file_fd : *parent->fds()) { if (file_fd && file_fd->close_on_popen()) { parent->handle_close(file_fd->fd()); } } } /* For the intermediate ForkedProcess where posix_spawn()'s file_actions were executed, * we still had all the fds, even the close-on-exec ones. Now it's time to close them. */ fds = parent->pass_on_fds(); parent->set_state(FB_PROC_TERMINATED); proc_tree->insert(parent); /* Now we can ack the previous posix_spawn()'s second message. */ if (launch_type == LAUNCH_TYPE_POSIX_SPAWN) { proc_tree->AckParent(unix_parent->pid()); } } /* Add the ExecedProcess. */ auto proc = ProcessFactory::getExecedProcess( ic_msg, parent, fds); if (launch_type == LAUNCH_TYPE_SYSTEM) { unix_parent->set_system_child(proc); } else if (launch_type == LAUNCH_TYPE_POPEN) { /* Entry must have been created at the "popen" message */ pending_popen_t *pending_popen = proc_tree->Proc2PendingPopen(unix_parent); assert(pending_popen); /* Fill in the new fields */ assert_null(pending_popen->child); pending_popen->child = proc; pending_popen->child_conn = fd_conn; /* If the "popen_parent" message has already arrived then accept the popened child, * which will also ACK the parent. * Otherwise this will be done whenever the "popen_parent" message arrives. */ if (pending_popen->fd >= 0) { accept_popen_child(unix_parent, pending_popen); } *new_proc = proc; return; } MessageProcessor::accept_exec_child(proc, fd_conn); *new_proc = proc; } else if (tag == FBBCOMM_TAG_fork_child) { auto ic_msg = reinterpret_cast(fbbcomm_buf); auto pid = ic_msg->get_pid(); auto ppid = ic_msg->get_ppid(); auto pending_ack = proc_tree->PPid2ParentAck(ppid); /* The supervisor needs up to date information about the fork parent in the ProcessTree * when the child Process is created. To ensure having up to date information all the * messages must be processed from the fork parent up to ForkParent and only then can * the child Process created in the ProcessTree and let the child process continue execution. */ if (!pending_ack) { /* queue fork_child data and delay processing messages on this socket */ proc_tree->QueueForkChild(pid, fd_conn, ppid, ack_id, new_proc); } else { auto pproc = proc_tree->pid2proc(ppid); assert(pproc); /* record new process */ accept_fork_child(pproc, pending_ack->sock, pending_ack->ack_num, new_proc, pid, fd_conn, ack_id); proc_tree->DropParentAck(ppid); } } } static void proc_ic_msg(const FBBCOMM_Serialized *fbbcomm_buf, uint16_t ack_num, int fd_conn, Process* proc) { TRACKX(FB_DEBUG_COMM, 1, 1, Process, proc, "fd_conn=%s, tag=%s, ack_num=%d", D_FD(fd_conn), fbbcomm_tag_to_string(fbbcomm_buf->get_tag()), ack_num); int tag = fbbcomm_buf->get_tag(); assert(proc); switch (tag) { case FBBCOMM_TAG_fork_parent: { auto parent_pid = proc->pid(); auto fork_child_sock = proc_tree->Pid2ForkChildSock(parent_pid); if (!fork_child_sock) { /* wait for child */ proc_tree->QueueParentAck(parent_pid, ack_num, fd_conn); } else { /* record new child process */ accept_fork_child(proc, fd_conn, ack_num, fork_child_sock->fork_child_ref, fork_child_sock->child_pid, fork_child_sock->sock, fork_child_sock->ack_num); proc_tree->DropQueuedForkChild(parent_pid); } return; } case FBBCOMM_TAG_exec_failed: { // FIXME(rbalint) check exec parameter and record what needs to be // checked when shortcutting the process proc->set_exec_pending(false); break; } case FBBCOMM_TAG_rusage: { auto ic_msg = reinterpret_cast(fbbcomm_buf); proc->resource_usage(ic_msg->get_utime_u(), ic_msg->get_stime_u()); break; } case FBBCOMM_TAG_system: { auto ic_msg = reinterpret_cast(fbbcomm_buf); assert_null(proc->system_child()); /* system(cmd) launches a child of argv = ["sh", "-c", "--", cmd], with "--" being optional */ auto expected_child = new ExecedProcessEnv(proc->pass_on_fds(false), LAUNCH_TYPE_SYSTEM); if (ic_msg->has_cmd()) { expected_child->set_sh_c_command(ic_msg->get_cmd()); } proc->set_expected_child(expected_child); break; } case FBBCOMM_TAG_system_ret: { assert(proc->system_child()); auto ic_msg = reinterpret_cast(fbbcomm_buf); /* system() implicitly waits for the child to finish. */ int ret = ic_msg->get_ret(); if (ret == -1 || !WIFEXITED(ret)) { proc->system_child()->exec_point()->disable_shortcutting_bubble_up_to_excl( proc->system_child()->fork_point()->exec_point(), "Process started by system() exited abnormally or the exit status could not be" " collected"); } else { proc->system_child()->fork_point()->set_exit_status(WEXITSTATUS(ret)); } proc->system_child()->set_been_waited_for(); if (!proc->system_child()->fork_point()->can_ack_parent_wait()) { /* The process has actually quit (otherwise the interceptor * couldn't send us the system_ret message), but the supervisor * hasn't seen this event yet. Thus we have to slightly defer * sending the ACK. */ proc->system_child()->set_on_finalized_ack(ack_num, fd_conn); proc->set_system_child(NULL); return; } /* Can be ACK'd straight away. */ proc->set_system_child(NULL); break; } case FBBCOMM_TAG_popen: { auto ic_msg = reinterpret_cast(fbbcomm_buf); assert(proc_tree->Proc2PendingPopen(proc) == nullptr); int type_flags = ic_msg->get_type_flags(); auto fds = proc->pass_on_fds(false); /* popen(cmd) launches a child of argv = ["sh", "-c", "--", cmd], with "--" being optional */ auto expected_child = new ExecedProcessEnv(fds, LAUNCH_TYPE_POPEN); // FIXME what if !has_cmd() ? expected_child->set_sh_c_command(ic_msg->get_cmd()); expected_child->set_type_flags(type_flags); proc->set_expected_child(expected_child); pending_popen_t pending_popen; pending_popen.type_flags = type_flags; // FIXME why set it at two places? proc_tree->QueuePendingPopen(proc, pending_popen); proc->set_has_pending_popen(true); break; } case FBBCOMM_TAG_popen_parent: { auto ic_msg = reinterpret_cast(fbbcomm_buf); /* Entry must have been created at the "popen" message */ pending_popen_t *pending_popen = proc_tree->Proc2PendingPopen(proc); assert(pending_popen); /* Fill in the new fields */ assert(pending_popen->fd == -1); pending_popen->fd = ic_msg->get_fd(); pending_popen->parent_conn = fd_conn; pending_popen->ack_num = ack_num; /* If the child's "scproc_query" message has already arrived then accept the popened child, * which will also ACK the parent. * Otherwise this will be done whenever the child's "scproc_query" message arrives.*/ if (pending_popen->child) { accept_popen_child(proc, pending_popen); } return; } case FBBCOMM_TAG_popen_failed: { auto ic_msg = reinterpret_cast(fbbcomm_buf); // FIXME what if !has_cmd() ? delete(proc->pop_expected_child_fds( std::vector({"sh", "-c", "--", ic_msg->get_cmd()}), nullptr, nullptr, true)); break; } case FBBCOMM_TAG_pclose: { auto ic_msg = reinterpret_cast(fbbcomm_buf); if (!ic_msg->has_error_no()) { /* pclose() is essentially an fclose() first, then a waitpid(), but the interceptor * sends an extra close message in advance thus here the fd is already tracked as closed. */ ExecedProcess *child = proc->PopPopenedProcess(ic_msg->get_fd()); assert(child); int ret = ic_msg->get_ret(); if (ret == -1 || !WIFEXITED(ret)) { child->exec_point()->disable_shortcutting_bubble_up_to_excl( child->fork_point()->exec_point(), "Process started by popen() exited abnormally or the exit status could not be" " collected"); } else { child->fork_point()->set_exit_status(WEXITSTATUS(ret)); } child->set_been_waited_for(); if (!child->fork_point()->can_ack_parent_wait()) { /* We haven't seen the process quitting yet. Defer sending the ACK. */ child->set_on_finalized_ack(ack_num, fd_conn); return; } /* Else we can ACK straight away. */ } break; } case FBBCOMM_TAG_posix_spawn: { auto ic_msg = reinterpret_cast(fbbcomm_buf); auto expected_child = new ExecedProcessEnv(proc->pass_on_fds(false), LAUNCH_TYPE_POSIX_SPAWN); std::vector argv = ic_msg->get_arg_as_vector(); expected_child->set_argv(argv); proc->set_expected_child(expected_child); proc->set_posix_spawn_pending(true); /* The actual forked process might perform some file operations according to * posix_spawn()'s file_actions. Pre-open the files to be written. */ for (size_t i = 0; i < ic_msg->get_file_actions_count(); i++) { const FBBCOMM_Serialized *action = ic_msg->get_file_actions_at(i); switch (action->get_tag()) { case FBBCOMM_TAG_posix_spawn_file_action_open: { /* A successful open to a particular fd, silently closing the previous file if any. */ const FBBCOMM_Serialized_posix_spawn_file_action_open *action_open = reinterpret_cast(action); int flags = action_open->get_flags(); if (is_write(flags)) { const FileName* file_name = proc->get_absolute( AT_FDCWD, action_open->get_pathname(), action_open->get_pathname_len()); if (file_name) { /* Pretend that the parent opened the file for writing and not the fork child. * This is not accurate, but the fork child does not exist yet. A parallel * process opening the file for writing would disable shortcutting the same way. */ file_name->open_for_writing(proc->exec_point()); } } break; } default: /* Only opens are handled (as pre_opens). */ break; } } break; } case FBBCOMM_TAG_posix_spawn_parent: { auto ic_msg = reinterpret_cast(fbbcomm_buf); /* First, do the basic fork() */ auto pid = ic_msg->get_pid(); auto fork_child = ProcessFactory::getForkedProcess(pid, proc); proc_tree->insert(fork_child); /* The actual forked process might perform some file operations according to * posix_spawn()'s file_actions. Do the corresponding administration. */ for (size_t i = 0; i < ic_msg->get_file_actions_count(); i++) { const FBBCOMM_Serialized *action = ic_msg->get_file_actions_at(i); switch (action->get_tag()) { case FBBCOMM_TAG_posix_spawn_file_action_open: { /* A successful open to a particular fd, silently closing the previous file if any. */ auto action_open = reinterpret_cast(action); const char *pathname = action_open->get_pathname(); const size_t pathname_len = action_open->get_pathname_len(); int fd = action_open->get_fd(); int flags = action_open->get_flags(); mode_t mode = action_open->get_mode(); fork_child->handle_force_close(fd); fork_child->handle_open(AT_FDCWD, pathname, pathname_len, flags, mode, fd, 0, -1, 0, false, false); /* Revert the effect of "pre-opening" paths to be written in the posix_spawn message.*/ if (is_write(flags)) { const FileName* file_name = fork_child->get_absolute(AT_FDCWD, pathname, pathname_len); if (file_name) { file_name->close_for_writing(); } } break; } case FBBCOMM_TAG_posix_spawn_file_action_close: { /* A close attempt, maybe successful, maybe failed, we don't know. See glibc's * sysdeps/unix/sysv/linux/spawni.c: * Signal errors only for file descriptors out of range. * sysdeps/posix/spawni.c: * Only signal errors for file descriptors out of range. * whereas signaling the error means to abort posix_spawn and thus not reach * this code here. */ auto action_close = reinterpret_cast(action); int fd = action_close->get_fd(); fork_child->handle_force_close(fd); break; } case FBBCOMM_TAG_posix_spawn_file_action_closefrom: { /* A successful closefrom. */ auto action_closefrom = reinterpret_cast (action); int lowfd = action_closefrom->get_lowfd(); fork_child->handle_closefrom(lowfd); break; } case FBBCOMM_TAG_posix_spawn_file_action_dup2: { /* A successful dup2. * Note that as per https://austingroupbugs.net/view.php?id=411 and glibc's * implementation, oldfd==newfd clears the close-on-exec bit (here only, * not in a real dup2()). */ auto action_dup2 = reinterpret_cast(action); int oldfd = action_dup2->get_oldfd(); int newfd = action_dup2->get_newfd(); if (oldfd == newfd) { fork_child->handle_clear_cloexec(oldfd); } else { fork_child->handle_dup3(oldfd, newfd, 0, 0); } break; } case FBBCOMM_TAG_posix_spawn_file_action_chdir: { /* A successful chdir. */ auto action_chdir = reinterpret_cast(action); const char *pathname = action_chdir->get_pathname(); fork_child->handle_set_wd(pathname); break; } case FBBCOMM_TAG_posix_spawn_file_action_fchdir: { /* A successful fchdir. */ auto action_fchdir = reinterpret_cast(action); int fd = action_fchdir->get_fd(); fork_child->handle_set_fwd(fd); break; } default: assert(false); } } proc->set_posix_spawn_pending(false); auto posix_spawn_child_sock = proc_tree->Pid2PosixSpawnChildSock(proc->pid()); if (posix_spawn_child_sock) { /* The child has already appeared, but had to wait for this "posix_spawn_parent" message. * Let the child continue (respond to the pending "scproc_query" with "scproc_resp"). */ auto posix_spawn_child = posix_spawn_child_sock->incomplete_child; fork_child->set_exec_child(posix_spawn_child); posix_spawn_child->set_parent(fork_child); posix_spawn_child->set_fds(fork_child->pass_on_fds()); MessageProcessor::accept_exec_child(posix_spawn_child, posix_spawn_child_sock->sock); proc_tree->DropQueuedPosixSpawnChild(proc->pid()); } else { /* The child hasn't appeared yet. Register a pending exec, just like we do at exec*() * calls. This lets us detect a statically linked binary launched by posix_spawn(), * exactly the way we do at a regular exec*(), i.e. successfully wait*()ing for a child * that is in exec_pending state. */ std::vector arg = ic_msg->get_arg_as_vector(); delete(proc->pop_expected_child_fds(arg, nullptr)); fork_child->set_exec_pending(true); } fork_child->set_state(FB_PROC_TERMINATED); /* In either case, ACK the "posix_spawn_parent" message, don't necessarily wait for the * child to appear. */ break; } case FBBCOMM_TAG_posix_spawn_failed: { auto ic_msg = reinterpret_cast(fbbcomm_buf); std::vector arg = ic_msg->get_arg_as_vector(); delete(proc->pop_expected_child_fds(arg, nullptr, nullptr, true)); proc->set_posix_spawn_pending(false); /* The actual forked process might perform some file operations according to * posix_spawn()'s file_actions. Revert the pre-opening of the files to be written. */ for (size_t i = 0; i < ic_msg->get_file_actions_count(); i++) { const FBBCOMM_Serialized *action = ic_msg->get_file_actions_at(i); switch (action->get_tag()) { case FBBCOMM_TAG_posix_spawn_file_action_open: { /* A successful open to a particular fd, silently closing the previous file if any. */ auto action_open = reinterpret_cast(action); int flags = action_open->get_flags(); if (is_write(flags)) { const FileName* file_name = proc->get_absolute( AT_FDCWD, action_open->get_pathname(), action_open->get_pathname_len()); if (file_name) { file_name->close_for_writing(); } } break; } default: /* Only opens are handled (as pre_opens). */ break; } } break; } case FBBCOMM_TAG_wait: { auto ic_msg = reinterpret_cast(fbbcomm_buf); const int pid = ic_msg->get_pid(); Process *child = proc_tree->pid2proc(pid); assert(child); int status; bool exited; if (ic_msg->has_si_code()) { /* The intercepted call was waitid() actually. */ status = ic_msg->get_si_status(); exited = ic_msg->get_si_code() == CLD_EXITED; } else { const int wstatus = ic_msg->get_wstatus(); status = WEXITSTATUS(wstatus); exited = WIFEXITED(wstatus); } if (exited) { child->fork_point()->set_exit_status(status); } else { child->exec_point()->disable_shortcutting_bubble_up_to_excl( child->fork_point()->exec_point(), "Process exited abnormally"); } child->set_been_waited_for(); if (child->exec_pending()) { /* If the supervisor believes an exec is pending in a child proces while the parent * actually successfully waited for the child, it means that the child didn't sign in to * the supervisor, presumably because it is statically linked. See #324 for details. */ child->exec_point()->disable_shortcutting_bubble_up( "Process did not sign in to supervisor, perhaps statically linked or failed to link"); /* Need to also clear the exec_pending state for Process::any_child_not_finalized() * and finalize this never-seen process. */ child->set_exec_pending(false); child->reset_file_fd_pipe_refs(); child->maybe_finalize(); /* Ack it straight away. */ } else if (!child->fork_point()->can_ack_parent_wait()) { /* We haven't seen the process quitting yet. Defer sending the ACK. */ child->set_on_finalized_ack(ack_num, fd_conn); return; } /* Else we can ACK straight away. */ break; } case FBBCOMM_TAG_pipe_request: { ProcessFBBAdaptor::handle(proc, reinterpret_cast(fbbcomm_buf), fd_conn); break; } case FBBCOMM_TAG_pipe_fds: { PFBBA_HANDLE(proc, pipe_fds, fbbcomm_buf); break; } case FBBCOMM_TAG_exec: { auto ic_msg = reinterpret_cast(fbbcomm_buf); proc->update_rusage(ic_msg->get_utime_u(), ic_msg->get_stime_u()); // FIXME(rbalint) save exec parameters proc->set_exec_pending(true); break; } case FBBCOMM_TAG_pre_open: { PFBBA_HANDLE(proc, pre_open, fbbcomm_buf); break; } case FBBCOMM_TAG_open: { PFBBA_HANDLE_ACKED(proc, open, fbbcomm_buf, fd_conn, ack_num); /* ACK is sent by the msg handler if needed. */ return; } case FBBCOMM_TAG_freopen: { PFBBA_HANDLE_ACKED(proc, freopen, fbbcomm_buf, fd_conn, ack_num); /* ACK is sent by the msg handler if needed. */ return; } case FBBCOMM_TAG_dlopen: { PFBBA_HANDLE_ACKED(proc, dlopen, fbbcomm_buf, fd_conn, ack_num); /* ACK is sent by the msg handler if needed. */ return; } case FBBCOMM_TAG_close: { PFBBA_HANDLE(proc, close, fbbcomm_buf); break; } case FBBCOMM_TAG_closefrom: { PFBBA_HANDLE(proc, closefrom, fbbcomm_buf); break; } case FBBCOMM_TAG_close_range: { PFBBA_HANDLE(proc, close_range, fbbcomm_buf); break; } case FBBCOMM_TAG_truncate: { PFBBA_HANDLE(proc, truncate, fbbcomm_buf); break; } case FBBCOMM_TAG_unlink: { PFBBA_HANDLE(proc, unlink, fbbcomm_buf); break; } case FBBCOMM_TAG_mkdir: { PFBBA_HANDLE(proc, mkdir, fbbcomm_buf); break; } case FBBCOMM_TAG_rmdir: { PFBBA_HANDLE(proc, rmdir, fbbcomm_buf); break; } case FBBCOMM_TAG_dup3: { PFBBA_HANDLE(proc, dup3, fbbcomm_buf); break; } case FBBCOMM_TAG_dup: { PFBBA_HANDLE(proc, dup, fbbcomm_buf); break; } case FBBCOMM_TAG_rename: { PFBBA_HANDLE(proc, rename, fbbcomm_buf); break; } case FBBCOMM_TAG_symlink: { PFBBA_HANDLE(proc, symlink, fbbcomm_buf); break; } case FBBCOMM_TAG_fcntl: { PFBBA_HANDLE(proc, fcntl, fbbcomm_buf); break; } case FBBCOMM_TAG_ioctl: { PFBBA_HANDLE(proc, ioctl, fbbcomm_buf); break; } case FBBCOMM_TAG_umask: { PFBBA_HANDLE(proc, umask, fbbcomm_buf); break; } case FBBCOMM_TAG_chdir: { PFBBA_HANDLE(proc, chdir, fbbcomm_buf); break; } case FBBCOMM_TAG_fchdir: { PFBBA_HANDLE(proc, fchdir, fbbcomm_buf); break; } case FBBCOMM_TAG_read_from_inherited: { PFBBA_HANDLE(proc, read_from_inherited, fbbcomm_buf); break; } case FBBCOMM_TAG_write_to_inherited: { PFBBA_HANDLE(proc, write_to_inherited, fbbcomm_buf); break; } case FBBCOMM_TAG_seek_in_inherited: { PFBBA_HANDLE(proc, seek_in_inherited, fbbcomm_buf); break; } case FBBCOMM_TAG_inherited_fd_offset: { PFBBA_HANDLE(proc, inherited_fd_offset, fbbcomm_buf); break; } case FBBCOMM_TAG_recvmsg_scm_rights: { PFBBA_HANDLE(proc, recvmsg_scm_rights, fbbcomm_buf); break; } case FBBCOMM_TAG_link: { proc->exec_point()->disable_shortcutting_bubble_up("Creating a hard link is not supported"); break; } case FBBCOMM_TAG_fstatat: { PFBBA_HANDLE(proc, fstatat, fbbcomm_buf); break; } case FBBCOMM_TAG_faccessat: { PFBBA_HANDLE(proc, faccessat, fbbcomm_buf); break; } case FBBCOMM_TAG_fchmodat: { PFBBA_HANDLE(proc, fchmodat, fbbcomm_buf); break; } #ifdef __linux__ case FBBCOMM_TAG_memfd_create: { PFBBA_HANDLE(proc, memfd_create, fbbcomm_buf); break; } case FBBCOMM_TAG_timerfd_create: { PFBBA_HANDLE(proc, timerfd_create, fbbcomm_buf); break; } case FBBCOMM_TAG_epoll_create: { PFBBA_HANDLE(proc, epoll_create, fbbcomm_buf); break; } case FBBCOMM_TAG_eventfd: { PFBBA_HANDLE(proc, eventfd, fbbcomm_buf); break; } case FBBCOMM_TAG_signalfd: { PFBBA_HANDLE(proc, signalfd, fbbcomm_buf); break; } #endif case FBBCOMM_TAG_getrandom: { auto ic_msg = reinterpret_cast(fbbcomm_buf); const unsigned int flags = ic_msg->get_flags_with_fallback(0); #ifdef GRND_RANDOM const std::string pathname(flags & GRND_RANDOM ? "/dev/random" : "/dev/urandom"); #else const std::string pathname("/dev/urandom"); (void)flags; #endif if (!FileName::Get(pathname)->is_in_ignore_location()) { proc->exec_point()->disable_shortcutting_bubble_up( deduplicated_string("Using " + pathname + " is not allowed").c_str()); } break; } case FBBCOMM_TAG_futime: { auto ic_msg = reinterpret_cast(fbbcomm_buf); const int fd = ic_msg->get_fd(); const FileFD* ffd = proc->get_fd(fd); if (!ic_msg->has_error_no() && ffd && is_write(ffd->flags()) && ic_msg->get_all_utime_now()) { /* The fd has been opened for writing and the access and modification times should be set to * current time which happens automatically when the process is shortcut. This is safe. */ } else { ExecedProcess* next_exec_level; if (quirks & FB_QUIRK_LTO_WRAPPER && proc->exec_point()->args().size() > 0 && proc->exec_point()->args()[0] == "touch" && (next_exec_level = proc->parent_exec_point()) // sh && (next_exec_level = next_exec_level->parent_exec_point()) // make && (next_exec_level = next_exec_level->parent_exec_point()) // lto-wrapper && next_exec_level->executable()->without_dirs() == "lto-wrapper" ) { FB_DEBUG(FB_DEBUG_PROC, "Allow shortcutting lto-wrapper's touch descendant " "(lto-wrapper quirk)"); } else { proc->exec_point()->disable_shortcutting_bubble_up( "Changing file timestamps is not supported"); } } break; } case FBBCOMM_TAG_utime: { proc->exec_point()->disable_shortcutting_bubble_up( "Changing file timestamps is not supported"); break; } case FBBCOMM_TAG_clock_gettime: { if (quirks & FB_QUIRK_IGNORE_TIME_QUERIES) { FB_DEBUG(FB_DEBUG_PROC, "Allow shortcutting time query " "(ignore-time-queries quirk)"); } else { proc->exec_point()->disable_shortcutting_bubble_up( "Time queries such as clock_gettime() prevent shortcutting unless the " "\"ignore-time-queries\" quirk is set."); } break; } case FBBCOMM_TAG_clone: { proc->exec_point()->disable_shortcutting_bubble_up("clone() is not supported"); break; } case FBBCOMM_TAG_socket: { PFBBA_HANDLE(proc, socket, fbbcomm_buf); break; } case FBBCOMM_TAG_socketpair: { PFBBA_HANDLE(proc, socketpair, fbbcomm_buf); break; } case FBBCOMM_TAG_statfs: ProcessFBBAdaptor::handle( proc, reinterpret_cast(fbbcomm_buf)); break; case FBBCOMM_TAG_gethostname: /* Ignore gethostname. With a local cache it should not make a difference, while * in a shared cache the intention is to use cached results from other machines. */ break; case FBBCOMM_TAG_seccomp: /* Ignore seccomp(). The interposer always returns EINVAL error to keep interception * working. This breaks sandboxing, but builds can run arbitrary commands anyway. */ break; case FBBCOMM_TAG_fb_debug: case FBBCOMM_TAG_fb_error: case FBBCOMM_TAG_fchownat: case FBBCOMM_TAG_fpathconf: case FBBCOMM_TAG_getdomainname: case FBBCOMM_TAG_lockf: case FBBCOMM_TAG_pathconf: case FBBCOMM_TAG_readlink: case FBBCOMM_TAG_scproc_resp: case FBBCOMM_TAG_sysconf: { // TODO(rbalint) break; } case FBBCOMM_TAG_gen_call: { auto msg = reinterpret_cast(fbbcomm_buf); const int error = msg->get_error_no_with_fallback(0); proc->exec_point()->disable_shortcutting_bubble_up( deduplicated_string( std::string(error == 0 ? "" : "failed") + msg->get_call() + " is not supported" + (error == 0 ? "" : " (error: " + d(error) + ")")).c_str()); break; } default: { fb_error("Unknown FBB message tag: " + std::to_string(tag)); assert(0 && "Unknown message FBB message tag!"); } } if (ack_num != 0) { ack_msg(fd_conn, ack_num); } } /* NOLINT(readability/fn_size) */ void MessageProcessor::ic_conn_readcb(const struct epoll_event* event, void *ctx) { auto conn_ctx = reinterpret_cast(ctx); auto proc = conn_ctx->proc; auto &buf = conn_ctx->buffer(); size_t full_length; const msg_header * header; ProcessDebugSuppressor debug_suppressor(proc); if (!Epoll::ready_for_read(event)) { FB_DEBUG(FB_DEBUG_COMM, "socket " + d_fd(Epoll::event_fd(event)) + " hung up (" + d(proc) + ")"); delete conn_ctx; return; } int read_ret = buf.read(Epoll::event_fd(event), -1); if (read_ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { /* Try again later. */ return; } } if (read_ret <= 0) { FB_DEBUG(FB_DEBUG_COMM, "socket " + d_fd(Epoll::event_fd(event)) + " hung up (" + d(proc) + ")"); delete conn_ctx; return; } do { if (buf.length() < sizeof(*header)) { /* Header is still incomplete, try again later. */ return; } else { header = reinterpret_cast(buf.data()); full_length = sizeof(*header) + header->msg_size; if (buf.length() < full_length) { /* Have partial message, more data is needed. */ return; } } /* Have at least one full message. */ auto fbbcomm_msg = reinterpret_cast(buf.data() + sizeof(*header)); if (!proc) { /* Now the message is complete, the debug suppression can be correctly set. */ debug_suppressed = ProcessFactory::peekProcessDebuggingSuppressed(fbbcomm_msg); } if (FB_DEBUGGING(FB_DEBUG_COMM)) { if (!debug_suppressed) { FB_DEBUG(FB_DEBUG_COMM, "fd " + d_fd( Epoll::event_fd(event)) + ": (" + d(proc) + ")"); if (header->ack_id) { fprintf(stderr, "ack_num: %d\n", header->ack_id); } fbbcomm_msg->debug(stderr); fflush(stderr); } } /* Process the messaage. */ if (proc) { proc_ic_msg(fbbcomm_msg, header->ack_id, Epoll::event_fd(event), proc); } else { /* Fist interceptor message */ proc_new_process_msg( fbbcomm_msg, header->ack_id, Epoll::event_fd(event), &conn_ctx->proc); /* Reset suppression which was set peeking at the message. */ debug_suppressed = false; } buf.discard(full_length); } while (buf.length() > 0); } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/message_processor.h000066400000000000000000000024631447164520700221350ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_MESSAGE_PROCESSOR_H_ #define FIREBUILD_MESSAGE_PROCESSOR_H_ #include "firebuild/execed_process.h" #include "firebuild/epoll.h" namespace firebuild { /** Handles incoming FBB messages from the interceptor */ class MessageProcessor { public: static void accept_exec_child(ExecedProcess* proc, int fd_conn, int fd0_reopen = -1); static void ic_conn_readcb(const struct epoll_event* event, void *ctx); }; } /* namespace firebuild */ #endif // FIREBUILD_MESSAGE_PROCESSOR_H_ firebuild-0.8.2/src/firebuild/obj_cache.cc000066400000000000000000000516241447164520700204500ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/obj_cache.h" #include #include #include #include #include #include #include #include #include #include #include #include "firebuild/blob_cache.h" #include "firebuild/config.h" #include "firebuild/debug.h" #include "firebuild/execed_process_cacher.h" #include "firebuild/hash.h" #include "firebuild/fbbfp.h" #include "firebuild/fbbstore.h" #include "firebuild/subkey.h" #include "firebuild/utils.h" namespace firebuild { /* singleton */ ObjCache *obj_cache; ObjCache::ObjCache(const std::string &base_dir) : base_dir_(base_dir) { mkdir(base_dir_.c_str(), 0700); } /* /x/xx// */ static size_t kObjCachePathLength = 1 + 1 + 1 + 2 + 1 + Hash::kAsciiLength + 1 + Subkey::kAsciiLength; /* * Constructs the directory name where the cached files are to be * stored, or read from. Optionally creates the necessary subdirectories * within the cache's base directory. * * Example: with base="base", key's ASCII representation being "key", and * create_dirs=true, it creates the directories "base/k", "base/k/ke" * and "base/k/ke/key" and returns the latter. */ static void construct_cached_dir_name(const std::string &base, const Hash &key, bool create_dirs, char* path) { char ascii[Hash::kAsciiLength + 1]; key.to_ascii(ascii); char *end = path; memcpy(end, base.c_str(), base.length()); end += base.length(); *end++ = '/'; *end++ = ascii[0]; if (create_dirs) { *end = '\0'; mkdir(path, 0700); } *end++ = '/'; *end++ = ascii[0]; *end++ = ascii[1]; if (create_dirs) { *end = '\0'; mkdir(path, 0700); } *end++ = '/'; memcpy(end, ascii, sizeof(ascii)); if (create_dirs) { mkdir(path, 0700); } } /* * Constructs the filename where the cached file is to be stored, or * read from. Optionally creates the necessary subdirectories within the * cache's base directory. * * Example: with base="base", key's ASCII representation being "key", * subkey's ASCII representation being "subkey", and create_dirs=true, it * creates the directories "base/k", "base/k/ke" and "base/k/ke/key" and * returns "base/k/ke/key/subkey". */ static void construct_cached_file_name(const std::string &base, const Hash &key, const char* const subkey, bool create_dirs, char* path) { construct_cached_dir_name(base, key, create_dirs, path); path[base.length() + kObjCachePathLength - Subkey::kAsciiLength - 1] = '/'; memcpy(&path[base.length() + kObjCachePathLength - Subkey::kAsciiLength], subkey, Subkey::kAsciiLength + 1); } bool ObjCache::store(const Hash &key, const FBBSTORE_Builder * const entry, off_t stored_blob_bytes, const FBBFP_Serialized * const debug_key) { TRACK(FB_DEBUG_CACHING, "key=%s, stored_blob_bytes=%" PRIoff, D(key), stored_blob_bytes); if (FB_DEBUGGING(FB_DEBUG_CACHING)) { FB_DEBUG(FB_DEBUG_CACHING, "ObjCache: storing entry, key " + d(key)); } if (FB_DEBUGGING(FB_DEBUG_CACHE) && debug_key) { /* Place a human-readable version of the key in the cache, for easier debugging. */ char* path_debug = reinterpret_cast(alloca(base_dir_.length() + kObjCachePathLength - Subkey::kAsciiLength + 1 +strlen(kDirDebugJson) + 1)); construct_cached_dir_name(base_dir_, key, true, path_debug); path_debug[base_dir_.length() + kObjCachePathLength - Subkey::kAsciiLength - 1] = '/'; memcpy(&path_debug[base_dir_.length() + kObjCachePathLength - Subkey::kAsciiLength], kDirDebugJson, strlen(kDirDebugJson) + 1); FILE *f = fopen(path_debug, "wx"); if (f) { debug_key->debug(f); execed_process_cacher->update_cached_bytes(ftell(f)); fclose(f); } } const char* tmpfile_end = "/new.XXXXXX"; char* tmpfile = static_cast(malloc(base_dir_.length() + strlen(tmpfile_end) + 1)); memcpy(tmpfile, base_dir_.c_str(), base_dir_.length()); memcpy(tmpfile + base_dir_.length(), tmpfile_end, strlen(tmpfile_end) + 1); int fd_dst = mkstemp(tmpfile); /* opens with O_RDWR */ if (fd_dst == -1) { fb_perror("Failed mkstemp() for storing cache object"); assert(0); free(tmpfile); return false; } // FIXME Do we need to split large files into smaller writes? // FIXME add basic error handling // FIXME Is it faster if we alloca() for small sizes instead of malloc()? // FIXME Is it faster to ftruncate() the file to the desired size, then mmap, // then serialize to the mapped memory, then ftruncate() again to the actual size? size_t len = entry->measure(); if (stored_blob_bytes + len > max_entry_size) { FB_DEBUG(FB_DEBUG_CACHING, "Could not store entry in cache because it would exceed max_entry_size"); free(tmpfile); close(fd_dst); return false; } char *entry_serial = reinterpret_cast(malloc(len)); entry->serialize(entry_serial); fb_write(fd_dst, entry_serial, len); close(fd_dst); /* Create randomized object file */ char* path_dst = reinterpret_cast(alloca(base_dir_.length() + kObjCachePathLength + 1)); struct timespec time; clock_gettime(CLOCK_REALTIME, &time); /* Store both seconds and nanoseconds in 64 bits. * The seconds since the epoch is stored in the first 34 bits * that will be enough until 2514 and the nanoseconds are stored in the next 30. */ Subkey subkey = Subkey((static_cast(time.tv_sec) << 30) + static_cast(time.tv_nsec)); if (FB_DEBUGGING(FB_DEBUG_DETERMINISTIC_CACHE)) { /* Debugging: Instead of a randomized filename (which is fast to generate) use the content's * hash for a deterministic filename. */ XXH128_hash_t entry_hash = XXH3_128bits(entry_serial, len); XXH128_canonical_t canonical; XXH128_canonicalFromHash(&canonical, entry_hash); /* Use only the first part of the digest in for the subkey. */ // TODO(rbalint) switching to XXH64_hash_t somehow does not match xxh64sum's output, debug that subkey = Subkey(canonical.digest); } construct_cached_file_name(base_dir_, key, subkey.c_str(), true, path_dst); free(entry_serial); if (fb_renameat2(AT_FDCWD, tmpfile, AT_FDCWD, path_dst, RENAME_NOREPLACE) == -1) { if (errno == EEXIST) { FB_DEBUG(FB_DEBUG_CACHING, "cache object is already stored"); unlink(tmpfile); return true; } else { fb_perror("Failed rename() while storing cache object"); assert(0); unlink(tmpfile); free(tmpfile); return false; } } else { execed_process_cacher->update_cached_bytes(len); } if (FB_DEBUGGING(FB_DEBUG_CACHING)) { FB_DEBUG(FB_DEBUG_CACHING, " subkey " + d(subkey)); } free(tmpfile); if (FB_DEBUGGING(FB_DEBUG_CACHE)) { /* Place a human-readable version of the value in the cache, for easier debugging. */ char* path_debug = reinterpret_cast(alloca(base_dir_.length() + kObjCachePathLength + strlen(kDebugPostfix) + 1)); memcpy(path_debug, path_dst, base_dir_.length() + kObjCachePathLength); memcpy(&path_debug[base_dir_.length() + kObjCachePathLength], kDebugPostfix, strlen(kDebugPostfix) + 1); FILE *f = fopen(path_debug, "wx"); if (f) { entry->debug(f); execed_process_cacher->update_cached_bytes(ftell(f)); fclose(f); } } return true; } bool ObjCache::retrieve(const Hash &key, const char* const subkey, uint8_t ** entry, size_t * entry_len) { TRACK(FB_DEBUG_CACHING, "key=%s, subkey=%s", D(key), D(subkey)); if (FB_DEBUGGING(FB_DEBUG_CACHING)) { FB_DEBUG(FB_DEBUG_CACHING, "ObjCache: retrieving entry, key " + d(key) + " subkey " + d(subkey)); } char* path = reinterpret_cast(alloca(base_dir_.length() + kObjCachePathLength + 1)); construct_cached_file_name(base_dir_, key, subkey, false, path); return retrieve(path, entry, entry_len); } bool ObjCache::retrieve(const char* path, uint8_t ** entry, size_t * entry_len) { TRACK(FB_DEBUG_CACHING, "path=%s", D(path)); int fd = open(path, O_RDONLY); if (fd == -1) { fb_perror("open"); assert(0); return false; } struct stat64 st; if (fstat64(fd, &st) == -1) { fb_perror("Failed fstat retrieving cache object"); assert(0); close(fd); return false; } else if (!S_ISREG(st.st_mode)) { FB_DEBUG(FB_DEBUG_CACHING, "not a regular file"); assert(0); close(fd); return false; } uint8_t *p = NULL; if (st.st_size > 0) { /* Zero bytes can't be mmapped, we're fine with p == NULL then. * Although a serialized entry probably can't be 0 bytes long. */ p = reinterpret_cast(mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0)); if (p == MAP_FAILED) { fb_perror("mmap"); assert(0); close(fd); return false; } } else { fb_error("0-sized cache entry: " + std::string(path)); assert(st.st_size <= 0); close(fd); return false; } close(fd); *entry_len = st.st_size; *entry = p; return true; } void ObjCache::mark_as_used(const Hash &key, const char* const subkey) { TRACK(FB_DEBUG_CACHING, "key=%s, subkey=%s", D(key), D(subkey)); char* path = reinterpret_cast(alloca(base_dir_.length() + kObjCachePathLength + 1)); construct_cached_file_name(base_dir_, key, subkey, false, path); /* Touch the used file. */ struct timespec times[2] = {{0, UTIME_OMIT}, {0, UTIME_NOW}}; utimensat(AT_FDCWD, path, times, 0); } /** * Return the list of subkeys for the given key in the order to be tried for shortcutting. * * The last created subkey is returned first. * * // FIXME replace with some iterator-like approach? */ static std::vector list_subkeys_internal(const char* path) { DIR *dir = opendir(path); if (dir == NULL) { return std::vector(); } std::vector ret; struct dirent *dirent; if (!FB_DEBUGGING(FB_DEBUG_CACHE)) { while ((dirent = readdir(dir)) != NULL) { if (Subkey::valid_ascii(dirent->d_name)) { ret.push_back(Subkey(dirent->d_name)); } } struct { bool operator()(const Subkey a, const Subkey b) const { return b < a; } } reverse_order; std::sort(ret.begin(), ret.end(), reverse_order); } else { /* Use the subkey's timestamp for sorting since with FB_DEBUG_CACHE the subkey * is generated from the file's content, not the creation timestamp. */ /* Note: Since using a subkey for shortcutting also sets mtime this ordering * may not match the ordering without debugging. */ std::vector> subkey_timestamp_pairs; struct stat st; while ((dirent = readdir(dir)) != NULL) { if (Subkey::valid_ascii(dirent->d_name) && fstatat(dirfd(dir), dirent->d_name, &st, 0) == 0) { subkey_timestamp_pairs.push_back({Subkey(dirent->d_name), st.st_mtim}); } } struct { bool operator()(const std::pair a, const std::pair b) const { return timespeccmp(&(b.second), &(a.second), <); } } reverse_order; std::sort(subkey_timestamp_pairs.begin(), subkey_timestamp_pairs.end(), reverse_order); for (auto pair : subkey_timestamp_pairs) { ret.push_back(pair.first); } } closedir(dir); return ret; } /** * Return the list of subkeys for the given key in the order to be tried for shortcutting. * * The last created subkey is returned first. * * // FIXME replace with some iterator-like approach? */ std::vector ObjCache::list_subkeys(const Hash &key) { TRACK(FB_DEBUG_CACHING, "key=%s", D(key)); char* path = reinterpret_cast(alloca(base_dir_.length() + kObjCachePathLength + 1)); construct_cached_dir_name(base_dir_, key, false, path); return list_subkeys_internal(path); } static void gc_collect_obj_timestamp_sizes_internal( const std::string& path, std::vector* obj_timestamp_sizes) { DIR * dir = opendir(path.c_str()); if (dir == NULL) { return; } /* Visit dirs recursively and collect all the files named as valid subkeys. */ struct dirent *dirent; while ((dirent = readdir(dir)) != NULL) { const char* name = dirent->d_name; if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { continue; } switch (fixed_dirent_type(dirent, dir, path)) { case DT_DIR: { gc_collect_obj_timestamp_sizes_internal(path + "/" + name, obj_timestamp_sizes); break; } case DT_REG: { struct stat st; if (Subkey::valid_ascii(name) && fstatat(dirfd(dir), name, &st, 0) == 0) { obj_timestamp_sizes->push_back({path + "/" + name, st.st_mtim, st.st_size}); } break; } default: /* Just ignore the file which is not a cache object named as a valid subkey. */ break; } } closedir(dir); } std::vector ObjCache::gc_collect_sorted_obj_timestamp_sizes() { std::vector obj_timestamp_sizes; gc_collect_obj_timestamp_sizes_internal(base_dir_, &obj_timestamp_sizes); struct { bool operator()(const obj_timestamp_size_t& a, const obj_timestamp_size_t& b) const { return timespeccmp(&(b.ts), &(a.ts), <); } } reverse; std::sort(obj_timestamp_sizes.begin(), obj_timestamp_sizes.end(), reverse); return obj_timestamp_sizes; } off_t ObjCache::gc_collect_total_objects_size() { return recursive_total_file_size(base_dir_); } void ObjCache::gc_obj_cache_dir(const std::string& path, tsl::hopscotch_set* referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes) { DIR * dir = opendir(path.c_str()); if (dir == NULL) { return; } /* Visit dirs recursively and check all the files. */ bool valid_ascii_found = false; struct dirent *dirent; std::vector entries_to_delete; std::vector subdirs_to_visit; while ((dirent = readdir(dir)) != NULL) { const char* name = dirent->d_name; if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { continue; } switch (fixed_dirent_type(dirent, dir, path)) { case DT_DIR: { subdirs_to_visit.push_back(name); break; } case DT_REG: { if (Subkey::valid_ascii(name)) { /* Good, will process this later using list_subkeys_internal() to process the subkeys * in the order they would be used for shortcutting. */ valid_ascii_found = true; } else { /* Regular file, but not named as expected for a cache object. */ const char* debug_postfix = nullptr; if (strcmp(name, kDirDebugJson) == 0) { if (FB_DEBUGGING(FB_DEBUG_CACHE)) { /* Keeping directory debuuging file, it may be removed with the otherwise empty dir * later. */ *debug_bytes += file_size(dir, name); } else { entries_to_delete.push_back(name); } } else if ((debug_postfix = strstr(name, kDebugPostfix))) { /* Files for debugging cache entries.*/ if (FB_DEBUGGING(FB_DEBUG_CACHE)) { const size_t name_len = debug_postfix - name; assert_cmp(name_len, <, FB_PATH_BUFSIZE); char related_name[FB_PATH_BUFSIZE]; memcpy(related_name, name, name_len); related_name[name_len] = '\0'; struct stat st; if (fstatat(dirfd(dir), related_name, &st, 0) == 0) { /* Keeping debugging file that has related object. If the object gets removed * the debugging file will be removed with it, too. */ *debug_bytes += file_size(dir, name); } else { /* Removing old debugging file later to not break next readdir(). */ entries_to_delete.push_back(name); } } else { /* Removing old debugging file later to not break next readdir(). */ entries_to_delete.push_back(name); } } else { fb_error("Regular file among cache objects has unexpected name, keeping it: " + path + "/" + name); *unexpected_file_bytes += file_size(dir, name); } } break; } default: fb_error("File's type is unexpected, it is not a directory nor a regular file: " + path + "/" + name); } } /* This actually deletes entries from here, the ObjCache, * just uses the implementation in BlobCache. */ BlobCache::delete_entries(path, entries_to_delete, kDebugPostfix, debug_bytes); for (const auto& subdir : subdirs_to_visit) { gc_obj_cache_dir(path + "/" + subdir, referenced_blobs, cache_bytes, debug_bytes, unexpected_file_bytes); } /* Process valid entries. */ if (valid_ascii_found) { std::vector entries = list_subkeys_internal(path.c_str()); int usable_entries = 0; for (const Subkey& entry : entries) { uint8_t* entry_buf; size_t entry_len; if (usable_entries >= shortcut_tries) { /* This entry will never be tried. */ struct stat st; if (fstatat(dirfd(dir), entry.c_str(), &st, AT_SYMLINK_NOFOLLOW) == 0) { if (unlinkat(dirfd(dir), entry.c_str(), 0) == 0) { execed_process_cacher->update_cached_bytes(-st.st_size); } else { fb_perror("unlinkat"); } } else { fb_perror("fstatat"); } continue; } if (retrieve((path + "/" + entry.c_str()).c_str(), &entry_buf, &entry_len)) { if (execed_process_cacher->is_entry_usable(entry_buf, referenced_blobs)) { /* The entry is usable and the referenced blobs were collected. */ munmap(entry_buf, entry_len); usable_entries++; *cache_bytes += entry_len; } else { /* This entry is not usable, remove it. */ munmap(entry_buf, entry_len); if (unlinkat(dirfd(dir), entry.c_str(), 0) == 0) { execed_process_cacher->update_cached_bytes(-entry_len); } else { fb_perror("unlinkat"); } } } else { fb_error("File's type is unexpected, it is not a directory nor a regular file: " + path + "/" + entry.c_str()); *unexpected_file_bytes += file_size(nullptr, (path + "/" + entry.c_str()).c_str()); } } } /* Remove empty directory. */ rewinddir(dir); bool has_valid_entries = false, has_dir_debug_json = false; while ((dirent = readdir(dir)) != NULL) { const char* name = dirent->d_name; /* skip "." and ".." */ if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { continue; } if ((strcmp(name, kDirDebugJson) == 0)) { has_dir_debug_json = true; continue; } has_valid_entries = true; break; } if (!has_valid_entries && path != base_dir_) { if (has_dir_debug_json) { struct stat st; if (fstatat(dirfd(dir), kDirDebugJson, &st, AT_SYMLINK_NOFOLLOW) == 0) { if (unlinkat(dirfd(dir), kDirDebugJson, 0) == 0) { execed_process_cacher->update_cached_bytes(-st.st_size); *debug_bytes -= st.st_size; } else { fb_perror("unlinkat"); } } else { fb_perror(kDebugPostfix); } } /* The directory is now empty. It can be removed. */ rmdir(path.c_str()); } closedir(dir); } void ObjCache::gc(tsl::hopscotch_set* referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes) { gc_obj_cache_dir(base_dir_, referenced_blobs, cache_bytes, debug_bytes, unexpected_file_bytes); } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/obj_cache.h000066400000000000000000000127011447164520700203030ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_OBJ_CACHE_H_ #define FIREBUILD_OBJ_CACHE_H_ #include #include #include #include "firebuild/subkey.h" #include "firebuild/hash.h" #include "firebuild/fbbfp.h" #include "firebuild/fbbstore.h" namespace firebuild { typedef struct obj_timestamp_size_ { std::string obj {""}; struct timespec ts {0, 0}; off_t size {0}; } obj_timestamp_size_t; /** * obj-cache is a weird caching structure where a key can contain * multiple values. More precisely, a key contains a list of subkeys, * and a (key, subkey) pair points to a value. * * In practice, one ProcessFingerprint can have multiple * ProcessInputsOutputs associated with it. The key is the hash of * ProcessFingerprint's serialization. The subkey happens to be the hash * of ProcessInputsOutputs's serialization, although it could easily be * anything else. * * Currently the backend is the filesystem. The multiple values are * stored as separate file of a given directory. The list of subkeys is * retrieved by listing the directory. * * E.g. ProcessFingerprint1's hash in ASCII is "fingerprint1". Underneath * it there are two values: ProcessInputsOutputs1's hash in ASCII is * "inputsoutputs1",ProcessInputsOutputs2's hash in ASCII is * "inputsoutputs2". The directory structure is: * - f/fi/fingerprint1/inputsoutputs1 * - f/fi/fingerprint1/inputsoutputs2 */ class ObjCache { public: explicit ObjCache(const std::string &base_dir); ~ObjCache(); /** * Store a serialized entry in obj-cache. * * @param key The key * @param entry The entry to serialize and store * @param stored_blob_bytes Total size of blobs referenced by this obj * @param debug_key Optionally the key as pb for debugging purposes * @return Whether succeeded */ bool store(const Hash &key, const FBBSTORE_Builder * const entry, off_t stored_blob_bytes, const FBBFP_Serialized * const debug_key); /** * Retrieve an entry from the obj-cache. * * @param key The key * @param subkey The subkey * @param[out] entry mmap()-ed cache entry. It is the caller's responsibility to munmap() it later. * @param[out] entry_len entry's length in bytes * @return Whether succeeded */ bool retrieve(const Hash &key, const char * const subkey, uint8_t ** entry, size_t * entry_len); bool retrieve(const char* path, uint8_t ** entry, size_t * entry_len); void mark_as_used(const Hash &key, const char * const subkey); std::vector list_subkeys(const Hash &key); /** * Garbage collect the object cache * @param referenced_blobs blobs referenced from the object cache entries. It is updated while * processing the cache objects. * @param[in,out] cache_bytes increased by every found and kept obj's size * @param[in,out] debug_bytes increased by every found and kept debug file's size * @param[in,out] unexpected_file_bytes increased by every found and kept file's size that has unexpected name, i.e. it is not used as a cache object, nor a debug file */ void gc(tsl::hopscotch_set* referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes); /* Returns {object path, timestamp, size} ordered by decreasing timestamp. */ std::vector gc_collect_sorted_obj_timestamp_sizes(); /** Returns total size of all stored objects including debug and invalid entries. */ off_t gc_collect_total_objects_size(); private: /** * Garbage collect an object cache directory * @param path object cache directory's absolute path * @param referenced_blobs blobs referenced from the object cache entries. It is updated while * processing the cache objects. * @param[in,out] cache_bytes increased by every found and kept obj's size * @param[in,out] debug_bytes increased by every found and kept debug file's size * @param[in,out] unexpected_file_bytes increased by every found and kept file's size that has unexpected name, i.e. it is not used as a cache object, nor a debug file */ void gc_obj_cache_dir(const std::string& path, tsl::hopscotch_set* referenced_blobs, off_t* cache_bytes, off_t* debug_bytes, off_t* unexpected_file_bytes); /* Including the "objs" subdir. */ std::string base_dir_; static constexpr char kDebugPostfix[] = "_debug.json"; static constexpr char kDirDebugJson[] = "%_directory_debug.json"; }; /* singleton */ extern ObjCache *obj_cache; } /* namespace firebuild */ #endif // FIREBUILD_OBJ_CACHE_H_ firebuild-0.8.2/src/firebuild/pipe.cc000066400000000000000000000473571447164520700175200ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/pipe.h" #include #include #ifndef __APPLE__ #include #endif #include #include #include #include #include #include #include #include "common/firebuild_common.h" #include "firebuild/debug.h" #include "firebuild/epoll.h" #include "firebuild/execed_process.h" #include "firebuild/file_fd.h" #include "firebuild/pipe_recorder.h" #include "firebuild/process.h" #include "firebuild/process_debug_suppressor.h" /** Timeout for closing a pipe after all fd1 ends are closed and a new hasn't been opened. */ const int kFd1ReopenTimeoutMs = 100; namespace firebuild { static void maybe_finish(Pipe* pipe) { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, pipe, ""); if (!pipe->finished()) { if (pipe->conn2fd1_ends.size() == 0) { if (pipe->buffer_empty()) { pipe->finish(); } else { pipe->set_send_only_mode(true); } } } } struct Fd0Deleter { void operator()(Pipe* pipe) const { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, pipe, ""); /* The last FileFD referencing the pipe's fd1 ends is gone, which means all processes that * could write to this pipe terminated. */ maybe_finish(pipe); pipe->reset_fd0_ptrs_self_ptr_(); } }; struct Fd1Deleter { void operator()(Pipe* pipe) const { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, pipe, ""); /* The last FileFD referencing the pipe's fd0 ends is gone, which means all processes that * could read from this pipe terminated. */ maybe_finish(pipe); pipe->reset_fd1_ptrs_self_ptr_(); } }; void Pipe::fd1_timeout_cb(void *arg) { Pipe* pipe = reinterpret_cast(arg); ProcessDebugSuppressor debug_suppressor(pipe->creator()); pipe->fd1_timeout_id_ = -1; if (++pipe->fd1_timeout_round_ >= 2) { /* At least kFd1ReopenTimeout time elapsed since the * the pipe lost the last fd1 end and all non timer events have been processed after that. */ pipe->finish(); } else { /* Add it again, it is not persistent. */ pipe->fd1_timeout_id_ = epoll->add_timer(kFd1ReopenTimeoutMs, fd1_timeout_cb, pipe); } } Pipe::Pipe(int fd0_conn, Process* creator) : fd0_conn(fd0_conn), conn2fd1_ends(), ffd2fd1_ends(), proc2recorders(), id_(id_counter_++), send_only_mode_(false), fd0_shared_ptr_generated_(false), fd1_shared_ptr_generated_(false), fd1_timeout_round_(0), buf_(), fd0_ptrs_held_self_ptr_(nullptr), fd1_ptrs_held_self_ptr_(nullptr), shared_self_ptr_(this), creator_(creator) { TRACKX(FB_DEBUG_PIPE, 0, 1, Pipe, this, "fd0_conn=%s, creator=%s", D_FD(fd0_conn), D(creator)); } Pipe::~Pipe() { TRACKX(FB_DEBUG_PIPE, 1, 0, Pipe, this, ""); if (fd1_timeout_id_ >= 0) { epoll->del_timer(fd1_timeout_id_); } } std::shared_ptr Pipe::fd0_shared_ptr() { assert(!fd0_shared_ptr_generated_); fd0_ptrs_held_self_ptr_ = shared_self_ptr_; fd0_shared_ptr_generated_ = true; return std::shared_ptr(this, Fd0Deleter()); } std::shared_ptr Pipe::fd1_shared_ptr() { assert(!fd1_shared_ptr_generated_); fd1_ptrs_held_self_ptr_ = shared_self_ptr_; fd1_shared_ptr_generated_ = true; return std::shared_ptr(this, Fd1Deleter()); } void Pipe::add_fd1_and_proc(int fd1_conn, FileFD* file_fd, ExecedProcess *proc, std::vector> recorders) { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, "fd1_conn=%s, proc=%s, #recorders=%" PRIsize, D_FD(fd1_conn), D(proc), recorders.size()); #ifdef FB_EXTRA_DEBUG assert(conn2fd1_ends.count(fd1_conn) == 0); #endif assert(!finished()); if (fd1_timeout_id_ >= 0) { epoll->del_timer(fd1_timeout_id_); fd1_timeout_id_ = -1; } auto fd1_end = new pipe_end({fd1_conn, {file_fd}, recorders}); conn2fd1_ends[fd1_conn] = fd1_end; ffd2fd1_ends[file_fd] = fd1_end; if (!send_only_mode_) { epoll->add_fd(fd1_conn, EPOLLIN, Pipe::pipe_fd1_read_cb, this); } proc2recorders[proc] = recorders; } void Pipe::pipe_fd0_write_cb(const struct epoll_event* event, void *arg) { auto pipe = reinterpret_cast(arg); ProcessDebugSuppressor debug_suppressor(pipe->creator()); TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, pipe, "fd=%s", D_FD(Epoll::event_fd(event))); if (!Epoll::ready_for_write(event)) { /* Clean up pipe. */ pipe->finish(); return; } switch (pipe->send_buf()) { case FB_PIPE_WOULDBLOCK: { /* waiting to be able to send more data on fd0 */ assert(pipe->send_only_mode()); break; } case FB_PIPE_FD0_EPIPE: { /* Clean up pipe. */ pipe->finish(); break; } case FB_PIPE_SUCCESS: { if (pipe->buffer_empty() && pipe->conn2fd1_ends.size() == 0) { if (!pipe->fd1_ptrs_held_self_ptr_) { /* There are no active fd1 ends nor fd1 references to this pipe. There can't be any more * incoming data. */ pipe->finish(); } else { /* There are references held to fd1 which means that a process may show up inheriting * the open pipe end. Set up a timer to finish() the pipe if the new process does * not register with the supervisor possibly because it is a static binary. */ pipe->fd1_timeout_round_ = 0; assert(pipe->fd1_timeout_id_ < 0); pipe->fd1_timeout_id_ = epoll->add_timer(kFd1ReopenTimeoutMs, fd1_timeout_cb, pipe); } } break; } default: assert(0 && "unexpected result from send_buf()"); } } void Pipe::close_one_fd1(int fd) { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, "fd=%s", D_FD(fd)); auto it = conn2fd1_ends.find(fd); if (it == conn2fd1_ends.end()) { return; } auto fd1_end = it->second; for (auto file_fd : fd1_end->file_fds) { ffd2fd1_ends.erase(file_fd); } conn2fd1_ends.erase(it); epoll->maybe_del_fd(fd); close(fd); delete(fd1_end); if (conn2fd1_ends.size() == 0) { if (buffer_empty()) { if (!fd1_ptrs_held_self_ptr_) { finish(); } else { /* There are references held to fd1 which means that a process may show up inheriting * the open pipe end. Set up a timer to finish() the pipe if the new process does * not register with the supervisor possibly because it is a static binary. */ fd1_timeout_round_ = 0; assert(fd1_timeout_id_ < 0); fd1_timeout_id_ = epoll->add_timer(kFd1ReopenTimeoutMs, fd1_timeout_cb, this); } } else { /* Let the pipe send out the remaining data. */ set_send_only_mode(true); } } } void Pipe::handle_close(FileFD* file_fd) { pipe_end* fd1_end = get_fd1_end(file_fd); /* The close message may be processed later than detecting the closure of the pipe end, but * when close arrives earlier the end needs to be drained and closed. */ if (fd1_end) { if (fd1_end->file_fds.size() == 1) { /* This was the last open fd, it is safe to drain it. */ drain_fd1_end(file_fd); } else { ffd2fd1_ends.erase(file_fd); fd1_end->file_fds.erase(file_fd); } } } void Pipe::handle_dup(FileFD* old_file_fd, FileFD* new_file_fd) { pipe_end* fd1_end = get_fd1_end(old_file_fd); /* The dup message may be processed later than detecting the closure of the pipe end, * but when a dup arrives and there is an associated end the end should be associated * with the new FileFD, too. */ if (fd1_end) { ffd2fd1_ends[new_file_fd] = fd1_end; fd1_end->file_fds.insert(new_file_fd); } } void Pipe::finish() { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, ""); if (finished()) { assert(!shared_self_ptr_); return; } FB_DEBUG(FB_DEBUG_PIPE, "cleaning up " + d(this)); /* clean up all events */ for (auto it : conn2fd1_ends) { FB_DEBUG(FB_DEBUG_PIPE, "closing pipe fd1: " + d_fd(it.first)); epoll->maybe_del_fd(it.first); close(it.first); delete it.second; } conn2fd1_ends.clear(); ffd2fd1_ends.clear(); pipe_op_result send_ret; do { send_ret = send_buf(); } while (!buffer_empty() && send_ret == FB_PIPE_SUCCESS); FB_DEBUG(FB_DEBUG_PIPE, "closing pipe fd0: " + d_fd(fd0_conn)); epoll->maybe_del_fd(fd0_conn); close(fd0_conn); fd0_conn = -1; if (fd1_timeout_id_ >= 0) { epoll->del_timer(fd1_timeout_id_); fd1_timeout_id_ = -1; } shared_self_ptr_.reset(); } void Pipe::pipe_fd1_read_cb(const struct epoll_event* event, void *arg) { auto pipe = reinterpret_cast(arg); ProcessDebugSuppressor debug_suppressor(pipe->creator()); TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, pipe, "fd=%s", D_FD(Epoll::event_fd(event))); if (!Epoll::ready_for_read(event)) { pipe->close_one_fd1(Epoll::event_fd(event)); return; } auto result = pipe->forward(Epoll::event_fd(event), false); switch (result) { case FB_PIPE_WOULDBLOCK: { /* waiting to be able to send more data on fd0 */ assert(pipe->send_only_mode()); break; } case FB_PIPE_FD0_EPIPE: { pipe->finish(); break; } case FB_PIPE_FD1_EOF: { pipe->close_one_fd1(Epoll::event_fd(event)); break; } case FB_PIPE_SUCCESS: { assert(!pipe->send_only_mode()); break; } default: assert(0 && "unexpected result from forward()"); } } void Pipe::set_send_only_mode(const bool mode) { TRACKX(FB_DEBUG_PIPE, 1, 0, Pipe, this, "mode=%s", D(mode)); assert(!finished()); if (mode != send_only_mode_) { FB_DEBUG(FB_DEBUG_PIPE, std::string(mode ? "en" : "dis") + "abling send only mode on " + d(this)); if (mode) { for (auto it : conn2fd1_ends) { epoll->del_fd(it.first); } /* should try again writing when fd0 becomes writable */ if (epoll->is_added_fd(fd0_conn)) { fd0_conn = epoll->remap_to_not_added_fd(fd0_conn); bump_fd_age(fd0_conn); } epoll->add_fd(fd0_conn, EPOLLOUT, Pipe::pipe_fd0_write_cb, this); } else { tsl::hopscotch_map conn2fd1_ends_to_remap_first = {}; for (auto it : conn2fd1_ends) { if (epoll->is_added_fd(it.first)) { conn2fd1_ends_to_remap_first.insert({it.first, it.second}); } else { epoll->add_fd(it.first, EPOLLIN, Pipe::pipe_fd1_read_cb, this); } } for (auto it : conn2fd1_ends_to_remap_first) { int new_conn = epoll->remap_to_not_added_fd(it.first); bump_fd_age(new_conn); conn2fd1_ends.erase(it.first); conn2fd1_ends[new_conn] = it.second; epoll->add_fd(new_conn, EPOLLIN, Pipe::pipe_fd1_read_cb, this); } /* Should not be woken up by fd0 staying writable until data arrives. */ epoll->del_fd(fd0_conn); } send_only_mode_ = mode; } else { FB_DEBUG(FB_DEBUG_PIPE, "send only mode already " + std::string(mode ? "en" : "dis") + "abled on " + d(this)); } } pipe_op_result Pipe::send_buf() { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, ""); assert(!finished()); if (!buffer_empty()) { /* There is data to be forwarded. */ ssize_t sent; do { sent = write(fd0_conn, buf_.data(), buf_.length()); FB_DEBUG(FB_DEBUG_PIPE, "sent " + d(sent) + " bytes via fd: " + d_fd(fd0_conn) + " of " + d(this)); if (sent == -1) { if (errno == EAGAIN || errno == EWOULDBLOCK) { /* This pipe should not receive more data. */ set_send_only_mode(true); return FB_PIPE_WOULDBLOCK; } else { if (errno == EPIPE) { FB_DEBUG(FB_DEBUG_PIPE, "ret: FB_PIPE_FD0_EPIPE"); return FB_PIPE_FD0_EPIPE; } else { // TODO(rbalint) handle some errors fb_perror("write"); return FB_PIPE_FD0_EPIPE; } } } else if (sent == 0) { /* This should be handled by EPIPE. */ assert(0 && "fd0_conn is closed, but not with EPIPE error"); return FB_PIPE_FD0_EPIPE; } else { buf_.discard(sent); if (buffer_empty()) { /* Buffer emptied, pipe can receive more data. */ if (send_only_mode_) { /* This pipe can now receive more data. */ set_send_only_mode(false); } } } } while (sent > 0 && !buffer_empty()); } return FB_PIPE_SUCCESS; } pipe_op_result Pipe::forward(int fd1, bool drain) { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, "fd1=%s, drain=%s", D_FD(fd1), D(drain)); pipe_op_result send_ret; if (finished()) { return FB_PIPE_FINISHED; } auto fd1_end = conn2fd1_ends[fd1]; assert(fd1_end); /* This loop tries to forward as much data as possible without blocking using the fast tee() * and splice() calls and then detects which end is blocked by trying to read to the buffer then * trying to send it. In case of being called with drain == true fd0 blocking does not break the * loop, but it continues to read all data available on fd1. Otherwise fd0 blocking disables read * callbacks - which would just fill the buffer - until the buffer is emptied and the data is * sent. */ do { int received; #ifdef __linux__ /* Try splice and tee first. */ if (buffer_empty()) { do { /* Forward data first to block the reader less. */ if (PipeRecorder::has_active_recorder(fd1_end->recorders)) { /* We want to record the data. Forward it using tee() which will leave it in the pipe. */ received = tee(fd1, fd0_conn, SIZE_MAX, SPLICE_F_NONBLOCK); if (received == -1 || received == 0) { /* EOF of other error on one of the fds, let the slow path figure that out. */ break; } else { FB_DEBUG(FB_DEBUG_PIPE, "sent " + d(received) + " bytes from fd: " + d_fd(fd1) + " to fd: " + d_fd(fd0_conn) + " using tee"); /* Save the data, consuming it from the pipe. */ PipeRecorder::record_data_from_unix_pipe(&fd1_end->recorders, fd1, received); } } else { /* We do not want to record the data. Forward it using splice() which consumes it from the * pipe. */ received = splice(fd1, NULL, fd0_conn, NULL, SIZE_MAX, SPLICE_F_NONBLOCK); if (received == -1 || received == 0) { /* EOF of other error on one of the fds, let the slow path figure that out. */ break; } else { FB_DEBUG(FB_DEBUG_PIPE, "sent " + d(received) + " bytes to fd: " + d_fd(fd0_conn) + " using splice"); } } } while (received > 0); } #endif /* Read one round to the buffer and try to send it. */ received = buf_.read(fd1, -1); if (received == -1) { if (errno == EAGAIN || errno == EWOULDBLOCK) { /* Try emptying the buffer if there is any data to send. */ return send_buf(); } else { /* Try emptying the buffer if there is any data to send. */ send_buf(); /* unexpected error, this fd1 connection can be closed irrespective to send_buf()'s * result */ return FB_PIPE_FD1_EOF; } } else if (received == 0) { FB_DEBUG(FB_DEBUG_PIPE, "received EOF from fd: " + d_fd(fd1)); /* Try emptying the buffer if there is any data to send. */ send_buf(); /* pipe end is closed */ return FB_PIPE_FD1_EOF; } else { FB_DEBUG(FB_DEBUG_PIPE, "received " + d(received) + " bytes from fd: " + d_fd(fd1)); /* Locate the new data in the buffer. */ ssize_t bufsize = buf_.length(); assert(bufsize >= received); const char * buf_to_save = buf_.data() + bufsize - received; /* Record it. */ PipeRecorder::record_data_from_buffer(&fd1_end->recorders, buf_to_save, received); /* Try to send it, too. */ send_ret = send_buf(); } } while (drain && (send_ret != FB_PIPE_FD0_EPIPE)); if (send_ret == FB_PIPE_FD0_EPIPE || send_ret == FB_PIPE_SUCCESS) { return send_ret; } /* sending is blocked */ assert(conn2fd1_ends.size() > 0); return FB_PIPE_WOULDBLOCK; } void Pipe::drain_fd1_end(FileFD* file_fd) { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, ""); if (finished()) { return; } auto fd1_end = get_fd1_end(file_fd); if (!fd1_end) { return; } int fd = fd1_end->fd; switch (forward(fd, true)) { case FB_PIPE_FD1_EOF: { /* This close will not finish the pipe, since there must be an fd1 ptr held, passed to this function. */ close_one_fd1(fd); break; } case FB_PIPE_FD0_EPIPE: { if (fd0_conn >= 0) { /* Clean up pipe. */ finish(); } break; } default: ffd2fd1_ends.erase(file_fd); fd1_end->file_fds.erase(file_fd); } } void Pipe::drain() { TRACKX(FB_DEBUG_PIPE, 1, 1, Pipe, this, ""); if (finished()) { return; } bool restart_iteration; tsl::hopscotch_set visited_fds; do { restart_iteration = false; for (auto pair : ffd2fd1_ends) { pipe_end* fd1_end = pair.second; assert(fd1_end); int fd = fd1_end->fd; if (!visited_fds.insert(fd).second) { /* Don't forward traffic again on already visited fds (after restarting iteration). */ continue; } switch (forward(fd, true)) { case FB_PIPE_FD1_EOF: { /* This close will not finish the pipe, since there must be an fd1 ptr held, passed to this function. */ close_one_fd1(fd); /* The iterator is invalid now, restart iteration */ restart_iteration = true; break; } case FB_PIPE_FD0_EPIPE: { if (fd0_conn >= 0) { /* Clean up pipe. */ finish(); /* Break the loop and exit to not touch the invalid iterator again. */ return; } break; } default: /* Nothing to do, the fd1 end may keep operating. */ break; } if (restart_iteration) { break; } } } while (restart_iteration); } void Pipe::add_data_from_fd(int fd, size_t len) { if (len > 0) { buf_.read(fd, len); /* Pipe might represent one of the top process's files inherited for writing, which might even * be a regular file (e.g. in case of "firebuild command args > outfile"). We can't directly * call set_send_only_mode() on that. So instead call send_buf(), it'll automatically take care * of it. */ send_buf(); } } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const Pipe& pipe, const int level) { std::string ret = "{Pipe #" + d(pipe.id()); if (level <= 0) { if (!pipe.finished()) { ret += ", fd1s:"; for (const auto& it : pipe.conn2fd1_ends) { ret += " " + d_fd(it.first); } ret += ", fd0: " + d_fd(pipe.fd0_conn); } else { ret += ", finished"; } ret += ", creator=" + d(pipe.creator(), level + 1); } ret += "}"; return ret; } std::string d(const Pipe *pipe, const int level) { if (pipe) { return d(*pipe, level); } else { return "{Pipe NULL}"; } } int Pipe::id_counter_ = 0; } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/pipe.h000066400000000000000000000324251447164520700173500ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PIPE_H_ #define FIREBUILD_PIPE_H_ #include #ifndef __APPLE__ #include #endif #include #include #include #include #include #include #include #include #include "firebuild/cxx_lang_utils.h" #include "firebuild/debug.h" #include "firebuild/epoll.h" #include "firebuild/linear_buffer.h" #include "firebuild/pipe_recorder.h" extern firebuild::Epoll *epoll; namespace firebuild { class FileFD; class ExecedProcess; class Process; typedef struct _pipe_end { /** fd number of this fd1 pipe end (where we get the data from) */ int fd; /* FileFDs associated with this pipe end keeping a(n fd1) reference to this pipe. */ tsl::hopscotch_set file_fds; /** Cache files to save the captured data to */ std::vector> recorders; } pipe_end; /** * Result codes of operations performed on pipe ends. */ typedef enum { /** Pipe's fd0 end would block forwarding more data */ FB_PIPE_WOULDBLOCK, /** Pipe's fd0 end got EPIPE */ FB_PIPE_FD0_EPIPE, /** One of pipe's fd1 reached EOF */ FB_PIPE_FD1_EOF, /** The pipe end can accept more data */ FB_PIPE_SUCCESS, /** Pipe is already finished, it is not operational. */ FB_PIPE_FINISHED } pipe_op_result; /** * A single Pipe object represents what would be a single Unix unnamed pipe (fifo) without the * interceptor mimicking it for the intercepted program. The interceptor routes the data written * to the pipe through the supervisor to be able to record it. Pipe is also used to catch the * initial stdout and stderr of the topmost intercepted process (typically the terminal). * * A Pipe may have multiple source file descriptors (fd1-s), that could be written to by multiple * Processes, due to dup(), fork() and alike. Each of them are converted to a separate named pipe * towards the supervisor, because it needs to record which process wrote the data. * The supervisor-side file descriptors of these channels are tracked in conn2fd1_ends, via the * add_fd1_and_proc() helper method. * * The fd0 and fd1 naming in the supervisor reflects that in the intercepted programs those ends * are connected to the pipefd[0] and pipefd[1] of pipe()'s output parameter. See pipe(2). * * Each Pipe has a single fd0 end in the supervisor. While filefd[0] can also be read from via * multiple file descriptors, even by multiple intercepted processes, the supevisor does not track * those separately because those are inputs to the intercepted processes and it cannot be reliably * separated on the supervisor's side which process consumed which part of the data. (As a result * expected process inputs read from pipes or inherited file descriptors cannot be used when * shortcutting a single process.) * * Forwarding data on the supervisor's side can be event-triggered or forced by calling * Pipe::forward(): * - For the event-triggered method there is an epoll callback registered on each pipe end. * fd0 and fd1 ends have different event handlers due fd0 can only be written to, and fd1-s can * only be read. In Pipe's default state (send_only_mode_ == false) the fd1 ends' callback is * active and whenever there is incoming data on an fd1 end it is written to the fd0 end * (and saved if the process the data came from can be shortcut). The data is not buffered if * it can be immediately sent. In this mode fd0's callback is disabled. * * If the incoming data can't be immediately sent via fd0 because fd0 would block * the pipe enters send_only_mode_, enables the callback on fd0 to be notified when fd0 becomes * writable again, and disables callbacks on fd1-s to not receive more data to the internal * buffer (buf_), where the data in flight is saved. * * In send_only_mode_ only writes to fd0 are triggered by fd events and the Pipe stays in this * mode until the internal buffer is emptied. Then the fd0 callback is disabled and all fd1 * callbacks are enabled again. send_only_mode_ is set to false. * * - Pipe::forward(int fd1, bool drain) can be used to reading from an fd1 end with or without * draining it. It tries to read once, or all the readable data in case of draining it. * Pipe::forward() reads from fd1 irrespective to the send_only_mode_ state, possibly adding more * data to the already used buffer. Drain mode is used when trying to receive all sent * data from a process that exec()-ed or terminated. * * Pipe ends lifecycle: * - Fd1 ends can be closed independently. When one fd1 end is closed the file descriptor is closed, * the callback on it is disabled and freed. When the last fd1 is closed there may still be data * in the buffer to send. In that case the pipe switches to send_only_mode_ and keeps forwarding * the data to fd0 until all the data is sent or received EPIPE on fd0. * Even when the last fd1 gets closed the pipe stays active and a new fd1 can be added to it. This * sequence of events can occur when the supervisor detects the closure of the fd1 fds before a * new intercepted process shows up for which one fd1 end needs to be reopened. As a result pipes * are finished after all fd1 ends are closed and there are no fd1-side references are kept by * processes. * It is also possible that there is an fd1-side reference kept in the supervisor, but the new * process that would inherit it never shows up, for example because it is statically linked * thus it is not intercepted. For that case when all fd1 ends are closed the pipe starts a timer * and waits a preset time and for the processing of all non timer events. If no new fd1 end is * added until this final cutoff time the pipe is finished. * - When fd0 end is closed the whole Pipe can be finish()-ed discarding the buffered data and * closing all fd1 ends. This is detected when receiving EPIPE on fd0. * The forward() and send_buf() functions don't change the Pipe ends, it is the responsibility of * the caller of forward() and send_buf() based on the Pipe operation result. */ class Pipe { public: Pipe(int fd0_conn, Process* creator); ~Pipe(); /** * Shared_ptr of this Pipe for fd0-side references. */ std::shared_ptr fd0_shared_ptr(); /** * Shared_ptr of this Pipe for fd1-side references. */ std::shared_ptr fd1_shared_ptr(); /** * Shared_ptr of this Pipe for not fd0- or fd1-side references. */ std::shared_ptr shared_ptr() {return shared_self_ptr_;} /** * fd number of the fd0 end (where we forward the data to). */ int fd0_conn; /** * Fd1 ends indexed by local connection file descriptor. * During fd1 end's lifetime this maps the supervisor-side connections to the fd1 end. * When and EOF is detected and the fd1 end is cleaned up and the connection is closed * the pipe_end reference is also removed from this map. */ tsl::hopscotch_map conn2fd1_ends; /** * Fd1 ends indexed by FileFD (pointer) * During fd1 end's lifetime this maps the intercepted process' file descriptor as tracked in * the supervisor to fd1 ends. * When and EOF is detected and the fd1 end is cleaned up the pipe_end reference is also removed * from this map. The FileFD can still be tracked as being open, because the message about the * close() or dup() may arrive later than the EOF being detected. */ tsl::hopscotch_map ffd2fd1_ends; /** * PipeRecorders indexed by ExecedProcess (pointer) * * For a given exec point, tells which PipeRecorders record(ed) the subset of the Pipe * corresponding to the given ExecProcess. * Somewhat similar to conn2fd1_ends and ffd2fd1_ends, but this one has to live on until the * process is stored in the cache, when pipe_end might no longer be around. Used for track the * recorders across an exec(), as well as storing in the cache what a process wrote to a pipe. */ std::unordered_map>> proc2recorders; void add_fd1_and_proc(int fd1, FileFD*, ExecedProcess *proc, std::vector> recorders); /** * Try to send some of the data that's in the buffers. Also flips send_only_mode (and thus * configures epoll) according to whether further sending is needed. * * The Pipe might represent a regular file that the top process inherited for writing. In this case * this method should successfully write the entire buffer, and thus not call set_send_only_mode(). */ pipe_op_result send_buf(); bool buffer_empty() { return buf_.length() == 0; } void reset_fd0_ptrs_self_ptr_() {fd0_ptrs_held_self_ptr_.reset();} void reset_fd1_ptrs_self_ptr_() {fd1_ptrs_held_self_ptr_.reset();} /** * Flip whether we wish to only send data from the Pipe's buffer (which we want if the buffer is * nonempty) or if we wish to read (and probably immediately send that). Also configure epoll * accordingly. * * Note: This method can't be called if the current Pipe represents one of regular files the top * process inherited for writing. E.g. if you execute: * firebuild command args > outfile * then care has to be taken not to call this method on "outfile". * This is because epoll_ctl() doesn't support regular files. */ void set_send_only_mode(bool mode); bool send_only_mode() {return send_only_mode_;} int id() const {return id_;} const Process * creator() const {return creator_;} /** * Read from fd1 and try to forward it to fd0 * @param fd1 connection to read from * @param drain false: read() available data only once true: read till EOF * @return result of the read or write operation, whichever could be executed last */ pipe_op_result forward(int fd1, bool drain); /** * Drain one fd1 end corresponding to file_fd and remove file_fd references from ffd2fd1_ends and * fd1 end's file_fds if they were present. * */ void drain_fd1_end(FileFD* file_fd); /** * Drain all fd1 ends. */ void drain(); /** * Handle closing a pipe end file descriptor in the intercepted process. * * Also drain the pipe end if this was the last open fd. */ void handle_close(FileFD* file_fd); void handle_dup(FileFD* old_file_fd, FileFD* new_file_fd); /** Close all ends of the pipe */ void finish(); /** All ends are closed and the pipe is not functional anymore, just exists because there are * references to it. */ bool finished() const { return fd0_conn == -1; } /** * Add the contents of the given file to the Pipe's buffer. This is used when shortcutting a * process, the cached data is injected into the Pipe. */ void add_data_from_fd(int fd, size_t len); private: /* Unique Pipe ID, for debugging */ int id_; /** Switch send only mode */ bool send_only_mode_:1; bool fd0_shared_ptr_generated_:1; bool fd1_shared_ptr_generated_:1; /** Number of times the fd1 timeout callback visited the pipe. */ unsigned int fd1_timeout_round_:3; LinearBuffer buf_; int fd1_timeout_id_ = -1; /** * Shared self pointer used by fd0 references to clean oneself up only after finish() and keep * track of fd0 references separately . */ std::shared_ptr fd0_ptrs_held_self_ptr_; /** * Shared self pointer used by fd1 references to clean oneself up only after finish() and keep * track of fd1 references separately. */ std::shared_ptr fd1_ptrs_held_self_ptr_; /** Shared self pointer kept until the pipe is finish()-ed */ std::shared_ptr shared_self_ptr_; /** The process that created this pipe, or NULL if it represents a pipe or terminal line * inherited from the external world. */ Process* creator_; /** Global counter, so that each Pipe object gets a unique ID. */ static int id_counter_; static void pipe_fd0_write_cb(const struct epoll_event* event, void *arg); static void pipe_fd1_read_cb(const struct epoll_event* event, void *arg); static void fd1_timeout_cb(void *arg); pipe_end* get_fd1_end(FileFD* file_fd) { auto it = ffd2fd1_ends.find(file_fd); if (it != ffd2fd1_ends.end()) { return it->second; } else { return nullptr; } } void close_one_fd1(int fd); DISALLOW_COPY_AND_ASSIGN(Pipe); }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const Pipe& pipe, const int level = 0); std::string d(const Pipe *pipe, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_PIPE_H_ firebuild-0.8.2/src/firebuild/pipe_recorder.cc000066400000000000000000000174541447164520700214000ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/pipe_recorder.h" #include #include #include #include #include #include "common/firebuild_common.h" #include "common/platform.h" #include "firebuild/execed_process.h" #include "firebuild/hash.h" #include "firebuild/pipe.h" #include "firebuild/utils.h" namespace firebuild { PipeRecorder::PipeRecorder(const ExecedProcess *for_proc) : for_proc_(for_proc), id_(id_counter_++) { TRACKX(FB_DEBUG_PIPE, 0, 1, PipeRecorder, this, "for_proc=%s", D(for_proc)); } void PipeRecorder::open_backing_file() { TRACKX(FB_DEBUG_PIPE, 1, 0, PipeRecorder, this, ""); if (asprintf(&filename_, "%s/pipe.XXXXXX", base_dir_) < 0) { fb_perror("asprintf"); assert(0 && "asprintf"); } fd_ = mkstemp(filename_); /* opens with O_RDWR */ if (fd_ < 0) { fb_perror("mkstemp"); free(filename_); filename_ = NULL; assert(0 && "mkstemp"); } } void PipeRecorder::add_data_from_buffer(const char *buf, ssize_t len) { TRACKX(FB_DEBUG_PIPE, 1, 1, PipeRecorder, this, "len=%" PRIssize, len); assert(!deactivated_); assert(!abandoned_); assert_cmp(len, >, 0); if (fd_ < 0) { open_backing_file(); } #ifndef NDEBUG ssize_t saved = #endif fb_write(fd_, buf, len); assert_cmp(saved, ==, len); offset_ += len; assert_cmp(offset_, >, 0); } void PipeRecorder::add_data_from_unix_pipe(int pipe_fd, ssize_t len) { TRACKX(FB_DEBUG_PIPE, 1, 1, PipeRecorder, this, "pipe_fd=%d, len=%" PRIssize, pipe_fd, len); assert(!deactivated_); assert(!abandoned_); assert_cmp(len, >, 0); if (fd_ < 0) { open_backing_file(); } /* Writing to a regular file. Also the caller must make sure by a preceding tee(2) call that * the given amount of data is readily available. So we're not expecting short writes. */ #ifndef NDEBUG ssize_t saved = #endif #ifdef __linux__ splice(pipe_fd, NULL, fd_, NULL, len, 0); #else fb_copy_file_range(pipe_fd, NULL, fd_, NULL, len, 0); #endif assert_cmp(saved, ==, len); offset_ += len; assert_cmp(offset_, >, 0); } void PipeRecorder::add_data_from_regular_fd(int fd_in, loff_t off_in, ssize_t len) { TRACKX(FB_DEBUG_PIPE, 1, 1, PipeRecorder, this, "fd_in=%d, off_in=%" PRIloff ", len=%" PRIssize, fd_in, off_in, len); assert(fd_in >= 0); assert(!deactivated_); assert(!abandoned_); assert_cmp(len, >, 0); if (fd_ < 0) { open_backing_file(); } ssize_t saved = fb_copy_file_range(fd_in, &off_in, fd_, NULL, len, 0); if (saved == -1) { fb_perror("copy_file_range"); abort(); } assert_cmp(saved, ==, len); offset_ += len; assert_cmp(offset_, >, 0); } bool PipeRecorder::store(bool *is_empty_out, Hash *key_out, off_t* stored_bytes) { TRACKX(FB_DEBUG_PIPE, 1, 1, PipeRecorder, this, ""); assert(!deactivated_); assert(!abandoned_); bool ret; if (fd_ >= 0) { /* Some data was seen. Place it in the blob cache, get its hash. */ *is_empty_out = false; ret = blob_cache->move_store_file(filename_, fd_, offset_, key_out); /* Note: move_store_file() closed the fd_. */ fd_ = -1; *stored_bytes = ret ? offset_ : 0; } else { /* No data was seen at all. */ *is_empty_out = true; ret = true; } free(filename_); filename_ = NULL; abandoned_ = true; return ret; } void PipeRecorder::abandon() { TRACKX(FB_DEBUG_PIPE, 1, 1, PipeRecorder, this, ""); assert(!abandoned_); if (fd_ >= 0) { close(fd_); unlink(filename_); fd_ = -1; } free(filename_); filename_ = NULL; abandoned_ = true; } void PipeRecorder::deactivate() { TRACKX(FB_DEBUG_PIPE, 1, 1, PipeRecorder, this, ""); assert(!deactivated_); if (fd_ >= 0) { close(fd_); unlink(filename_); fd_ = -1; } free(filename_); filename_ = NULL; deactivated_ = true; } bool PipeRecorder::has_active_recorder( const std::vector>& recorders) { for (size_t i = 0; i < recorders.size(); i++) { if (!recorders[i]->deactivated_) { return true; } } return false; } void PipeRecorder::record_data_from_buffer(std::vector> *recorders, const char *buf, ssize_t len) { TRACK(FB_DEBUG_PIPE, "#recorders=%" PRIsize ", len=%" PRIssize, recorders->size(), len); assert_cmp(len, >, 0); // FIXME Would it be faster to call add_data_from_buffer() for the first active recorder only, // and then do add_data_from_regular_fd() (i.e. copy_file_range()) for the rest? for (std::shared_ptr& recorder : *recorders) { if (!recorder->deactivated_) { recorder->add_data_from_buffer(buf, len); } } } void PipeRecorder::record_data_from_unix_pipe(std::vector> *recorders, int fd, ssize_t len) { TRACK(FB_DEBUG_PIPE, "#recorders=%" PRIsize ", fd=%d, len=%" PRIssize, recorders->size(), fd, len); #ifdef FB_EXTRA_DEBUG assert(has_active_recorder(*recorders)); #endif assert_cmp(len, >, 0); /* The first active recorder consumes the data from the pipe. */ size_t i; for (i = 0; i < recorders->size(); i++) { if (!(*recorders)[i]->deactivated_) { (*recorders)[i]->add_data_from_unix_pipe(fd, len); break; } } size_t first_active = i++; /* The remaining active recorders copy from the first one's backing file. */ for (; i < recorders->size(); i++) { if (!(*recorders)[i]->deactivated_) { (*recorders)[i]->add_data_from_regular_fd((*recorders)[first_active]->fd_, (*recorders)[first_active]->offset_ - len, len); } } } void PipeRecorder::record_data_from_regular_fd( std::vector> *recorders, int fd, ssize_t len) { TRACK(FB_DEBUG_PIPE, "#recorders=%" PRIsize ", fd=%d, len=%" PRIssize, recorders->size(), fd, len); assert_cmp(len, >, 0); for (std::shared_ptr& recorder : *recorders) { if (!recorder->deactivated_) { recorder->add_data_from_regular_fd(fd, 0, len); } } } void PipeRecorder::set_base_dir(const char *dir) { free(base_dir_); base_dir_ = strdup(dir); mkdir(base_dir_, 0700); } std::string PipeRecorder::d_internal(const int level) const { (void)level; std::string ret = "{PipeRecorder #" + d(id_) + ", " + d(offset_) + " bytes"; if (abandoned_) { ret += ", abandoned"; } else if (deactivated_) { ret += ", deactivated"; } else { ret += " so far"; } ret += "}"; return ret; } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const PipeRecorder& recorder, const int level) { return recorder.d_internal(level); } std::string d(const PipeRecorder *recorder, const int level) { if (recorder) { return d(*recorder, level); } else { return "{PipeRecorder NULL}"; } } int PipeRecorder::id_counter_ = 0; char *PipeRecorder::base_dir_ = NULL; } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/pipe_recorder.h000066400000000000000000000424571447164520700212430ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PIPE_RECORDER_H_ #define FIREBUILD_PIPE_RECORDER_H_ #include #include #include #include #include "firebuild/debug.h" #include "firebuild/blob_cache.h" #include "firebuild/hash.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { class ExecedProcess; class Pipe; /* * Pipe Recording Overview * * As described in Pipe's documentation, a Pipe instance represents what would have been a single * unnamed Unix pipe (or the terminal) if we didn't intervene. * * It might receive traffic from different actual Unix file descriptors (due to how replacing that * pipe with named pipes works). They need to be stored in the cache differently. * * Example: * * - Proc0 creates a pipe, writes Text0 into it. * * The Pipe objects is created, the pipe's traffic is routed through it. No PipeRecorder yet, * Text0 is forwarded but not recorded. This is because no matter where we will shortcut in a * later run, this data won't be replayed. * * - Proc0 exec()s or fork()+exec()s Proc1, Proc1 writes Text1. * * Pipe is one of Proc1's inherited_files of type FD_PIPE_OUT. A PipeRecorder instance Rec1 is * created when Proc1 is accepted, and is placed at the corresponding inherited_file's recorder. * Furthermore, in the Pipe object, for the pipe_end that corresponds to Proc1's reopened fd, this * recorder is added as the only recorder. When Text1 is seen, Rec1 opens the backing File1 and * stores Text1. * * - Proc1 exec()s or fork()+exec()s Proc2, Proc2 writes Text2. * * Pipe is one of Proc2's inherited_files of type FD_PIPE_OUT. A PipeRecorder instance Rec2 is * created when Proc2 is accepted, and is placed at the corresponding inherited_file's recorder. * Furthermore, in the Pipe object, for the pipe_end that corresponds to Proc2's reopened fd, this * recorder is added too, resulting in two recorders: Rec1 and Rec2. Text2 is recorded by both, * i.e. into File1 and File2. * * - Proc2 exec()s or fork()+exec()s Proc3, Proc3 writes Text3. * * At the corresponding inherited_file of Proc3, Rec3 is the recorder. At the Pipe, corresponding * to Proc3's reopened fd, there are now three recorders Rec1, Rec2 and Rec3. Text3 is recorded by * all, i.e. into File1, File2 and File3. * * Now File1 contains Text1+Text2+Text3, File2 contains Text2+Text3, and File3 contains Text3. * That is, each cache entry recorded exactly what was written to the given Pipe, from below the * given exec point in the process tree. * * This example is also shown in this picture. Solid arrows denote pointer-like relationship in the * supervisor's memory. Dashed arrows on the left side show the data that travels through the Unix * pipes, from the actual intercepted processes that are represented by those Process boxes. Dashed * arrows on the right side show how this data travels further inside the supervisor. * * ┌───────── Pipe ──────────┐ * │ pipe_ends: │ * │ ┌───────────────────┐ │ * ┌╌╌ Text0 ╌╌╌╌╌╌╌╌│╌>│ recorders: (none) │ │ * ┆ │ ├───────────────────┤ │ * ┆ ┌╌╌ Text1 ╌╌╌╌╌╌│╌>│ recorders: │ │ * ┆ ┆ │ │ - Rec1 │╌╌│╌┐ * ┆ ┆ │ ├───────────────────┤ │ ┆ * ┆ ┆ ┌╌╌ Text2 ╌╌╌╌│╌>│ recorders: │ │ ┆ * ┆ ┆ ┆ │ │ - Rec1 │╌╌│╌┆╌┐ * ┆ ┆ ┆ │ │ - Rec2 │╌╌│╌┆╌┆╌╌╌┐ * ┆ ┆ ┆ │ ├───────────────────┤ │ ┆ ┆ ┆ * ┆ ┆ ┆ ┌╌╌ Text3 ╌╌│╌>│ recorders: │ │ ┆ ┆ ┆ * ┆ ┆ ┆ ┆ │ │ - Rec1 │╌╌│╌┆╌┆╌┐ ┆ * ┆ ┆ ┆ ┆ │ │ - Rec2 │╌╌│╌┆╌┆╌┆╌┆╌┐ * ┆ ┆ ┆ ┆ │ │ - Rec3 │╌╌│╌┆╌┆╌┆╌┆╌┆╌┐ * ┌────────────── Proc0 ─────────────┐ ┆ ┆ ┆ ┆ │ └───────────────────┘ │ ┆ ┆ ┆ ┆ ┆ ┆ * │ named pipe │╌╌┘ ┆ ┆ ┆ └─────────────────────────┘ ┆ ┆ ┆ ┆ ┆ ┆ * └──────────────────────────────────┘ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ * │ ┆ ┆ ┆ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ Text1 ╌╌┘ ┆ ┆ ┆ ┆ ┆ * │ ┆ ┆ ┆ ┆ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ Text2 ╌╌┘ ┆ ┆ ┆ ┆ * v ┆ ┆ ┆ ┆ ┆ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ Text3 ╌╌┘ ┆ ┆ ┆ * ┌────────────── Proc1 ─────────────┐ ┆ ┆ ┆ v v v ┆ ┆ ┆ * │ named pipe │╌╌╌╌┘ ┆ ┆ ┌── Rec1 ──┐ ┌───── File1 ─────┐ ┆ ┆ ┆ * │ inherited FD_PIPE_OUT's recorder │────────────>│ file │───>│ Text1Text2Text3 │ ┆ ┆ ┆ * └──────────────────────────────────┘ ┆ ┆ └──────────┘ └─────────────────┘ ┆ ┆ ┆ * │ ┆ ┆ ┆ ┆ ┆ * │ ┆ ┆ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ Text2 ╌╌┘ ┆ ┆ * v ┆ ┆ ┆ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ Text3 ╌╌┘ ┆ * ┌────────────── Proc2 ─────────────┐ ┆ ┆ v v ┆ * │ named pipe │╌╌╌╌╌╌┘ ┆ ┌── Rec2 ──┐ ┌───── File2 ─────┐ ┆ * │ inherited FD_PIPE_OUT's recorder │────────────>│ file │───>│ Text2Text3 │ ┆ * └──────────────────────────────────┘ ┆ └──────────┘ └─────────────────┘ ┆ * │ ┆ ┆ * │ ┆ ┆ * v ┆ ┌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ Text3 ╌╌┘ * ┌────────────── Proc3 ─────────────┐ ┆ v * │ named pipe │╌╌╌╌╌╌╌╌┘ ┌── Rec3 ──┐ ┌───── File3 ─────┐ * │ inherited FD_PIPE_OUT's recorder │────────────>│ file │───>│ Text3 │ * └──────────────────────────────────┘ └──────────┘ └─────────────────┘ * * In pipe_end, there's an array of recorders. Each incoming piece of data might need to be * registered for multiple processes (or none at all) as something that happened to the given * process transitively. E.g. Text2 is printed directly by Proc2, yet Proc1 also printed this * indirectly, via the help of a child process. * * If an execed process inherits a pipe (rather than creating it on its own), exactly one of these * recorders is the special one (stored in inherited_file's recorder field) which records the * traffic generated by this execed process, transitively. This has to be stored in the cache as the * data transitively printed by this process, and replayed if we shortcut the process. */ /* * The static record_data_*() methods take multiple PipeRecorder objects and operate on them * simultaneously, rather than on one PipeRecorder at a time. This was designed to hide from the * caller that sometimes the desired action is not the same for all the recorders (e.g. the first * one fetches the data from the pipe, the next ones clone it across files). This was also designed * in preparation for the following TODO: PipeRecorder objects that happen to see the exact same * traffic should share the underlying the file, and then clone on demand. */ /** * PipeRecorder represents a certain traffic (data stream), a subset of the entire traffic that a * Pipe object sees, corresponding to what happens in the subtree under an exec point. (See "Pipe * Recording Overview" above for detailed explanation.) * * This object records the traffic, and eventually stores in the cache with the computed hash. * * Opening the backing file is delayed until actual traffic is encountered. It is handled as a * special case if the recorder sees no traffic at all, this won't be stored in the cache. * * Currently each PipeRecorder object has its own backing file. This should be improved so * that PipeRecorder objects that happen to see the exact same traffic share the same backing * file. */ class PipeRecorder { public: explicit PipeRecorder(const ExecedProcess *for_proc); ~PipeRecorder() { if (!abandoned_) abandon(); } /** * If there was traffic, store it in the cache and return is_empty_out=false, key_out=[the_hash]. * If there was no traffic, set is_empty_out=true, key_out is undefined. * Set the recorder to abandoned state. * Returns false in case of failure. */ bool store(bool *is_empty_out, Hash *key_out, off_t* stored_bytes); /** Close the backing fd, drop the data that was written so far. Set to deactivated state. */ void deactivate(); /** Close the backing fd, drop the data that was written so far. Set to abandoned state. */ void abandon(); /** * Returns whether any of the given recorders is active, i.e. still records data. */ static bool has_active_recorder(const std::vector>& recorders); /** * Record the given data, from an in-memory buffer, to all the given recorders that are still active. * * See pipe_recorder.h for the big picture, as well as the design rationale behind this static * method taking multiple PipeRecorders at once. */ static void record_data_from_buffer(std::vector> *recorders, const char *buf, ssize_t len); /** * Record the given data, from the given Unix pipe, to all the given recorders that are still * active. * * The recorders array must contain at least one active recorder. * * The Unix pipe must have the given amount of data readily available, as guaranteed by a previous * tee(2) call. The data is consumed from the pipe. * * See pipe_recorder.h for the big picture, as well as the design rationale behind this static * method taking multiple PipeRecorders at once. */ static void record_data_from_unix_pipe(std::vector> *recorders, int fd, ssize_t len); /** * Record the given data, from the beginning of the given regular file, to all the given recorders * that are still active. * * The current seek offset is irrelevant. len must match the file's size. * * (This is used when replaying and bubbling up pipe traffic.) * * See in pipe_recorder.h for the big picture, as well as the design rationale behind this static * method taking multiple PipeRecorders at once. */ static void record_data_from_regular_fd(std::vector> *recorders, int fd, ssize_t len); static void set_base_dir(const char *dir); /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d_internal(const int level = 0) const; private: /** * Perform the delayed opening of the backing file. * To be called the first time when there's data to record. */ void open_backing_file(); /** * Add non-empty data to this PipeRecorder from a memory buffer, using write(). * * Internal private helper. Callers should call the static record_*() methods instead. */ void add_data_from_buffer(const char *buf, ssize_t len); /** * Add non-empty data to this PipeRecorder from a pipe, using splice(). * * The Unix pipe must have the given amount of data readily available, as guaranteed by a previous * tee(2) call. The data is consumed from the pipe. * * Internal private helper. Callers should call the static record_*() methods instead. */ void add_data_from_unix_pipe(int pipe_fd, ssize_t len); /** * Add non-empty data to this PipeRecorder, by copying it from another file using copy_file_range(). * * The current seek offset in fd_in is irrelevant. * * Internal private helper. Callers should call the static record_*() methods instead. */ void add_data_from_regular_fd(int fd_in, loff_t off_in, ssize_t len); /* The ExecedProcess we're recording for, i.e. the ExecedProcess that created this PipeRecorder to * add to its inherited_files array. Data written to the Pipe by this process or a descendant will * be recorded by this PipeRecorder, data written to the Pipe by an ancestor of this process * won't. Used for debugging only. */ const ExecedProcess *for_proc_; /* The name of the backing file, if currently opened. */ char *filename_ = NULL; /* The fd, -1 if not yet opened or already closed. */ int fd_ {-1}; /* The amount of data written so far. */ loff_t offset_ {0}; /* A deactivated PipeRecorder has thrown away all the data it has ever received, and won't record * any new data. However, it might still receive data "to record", which will silently be dropped. * This is useful when a process can no longer be shortcut, so there's no point in recording the * data as seen from its point of view, however, there's still pipe traffic going on. */ bool deactivated_ {false}; /* An abandoned PipeRecorder has either stored or thrown away the data it received so far. It * asserts that it doesn't receive any new data to record. */ bool abandoned_ {false}; /* Unique PipeRecorder ID, for debugging. */ int id_; static int id_counter_; /* The location to work with, including the "tmp" subdir. */ static char *base_dir_; DISALLOW_COPY_AND_ASSIGN(PipeRecorder); }; /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const PipeRecorder& recorder, const int level = 0); std::string d(const PipeRecorder *recorder, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_PIPE_RECORDER_H_ firebuild-0.8.2/src/firebuild/process.cc000066400000000000000000002075531447164520700202350ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/process.h" #include #include #ifdef __linux__ #include #include #endif #include #include #ifdef __linux__ #include #include #include #endif #include #include #include #include #include "common/firebuild_common.h" #include "common/platform.h" #include "firebuild/pipe_recorder.h" #include "firebuild/config.h" #include "firebuild/execed_process.h" #include "firebuild/forked_process.h" #include "firebuild/execed_process_env.h" #include "firebuild/process_tree.h" #include "firebuild/debug.h" #include "firebuild/utils.h" namespace firebuild { static int fb_pid_counter; Process::Process(const int pid, const int ppid, const int exec_count, const FileName *wd, const mode_t umask, Process * parent, std::vector>* fds, const bool debug_suppressed) : parent_(parent), state_(FB_PROC_RUNNING), fb_pid_(fb_pid_counter++), pid_(pid), ppid_(ppid), exec_count_(exec_count), wd_(wd), umask_(umask), fds_(fds), fork_children_(), expected_child_(), debug_suppressed_(debug_suppressed), exec_child_(NULL) { TRACKX(FB_DEBUG_PROC, 0, 1, Process, this, "pid=%d, ppid=%d, parent=%s", pid, ppid, D(parent)); } Process* Process::fork_parent() { return fork_point() ? fork_point()->parent() : nullptr; } const Process* Process::fork_parent() const { return fork_point() ? fork_point()->parent() : nullptr; } Process* Process::last_exec_descendant() { Process* ret = this; while (ret->exec_child()) { ret = ret->exec_child(); } return ret; } const Process* Process::last_exec_descendant() const { const Process* ret = this; while (ret->exec_child()) { ret = ret->exec_child(); } return ret; } void Process::update_rusage(const int64_t utime_u, const int64_t stime_u) { ExecedProcess* ep = exec_point(); if (ep) { ep->add_utime_u(utime_u); ep->add_stime_u(stime_u); } } void Process::resource_usage(const int64_t utime_u, const int64_t stime_u) { update_rusage(utime_u, stime_u); } /* This is a static function operating on any std::vector>, * without requiring a Process object. */ std::shared_ptr Process::add_filefd(std::vector>* fds, const int fd, std::shared_ptr ffd) { TRACK(FB_DEBUG_PROC, "fd=%d", fd); if (fds->size() <= static_cast(fd)) { fds->resize(fd + 1, nullptr); } if ((*fds)[fd]) { firebuild::fb_error("Fd " + d(fd) + " is already tracked as being open."); } /* the shared_ptr takes care of cleaning up the old fd if needed */ (*fds)[fd] = ffd; return ffd; } /* This is the member function operating on `this` Process. */ std::shared_ptr Process::add_filefd(const int fd, std::shared_ptr ffd) { TRACK(FB_DEBUG_PROC, "fd=%d", fd); return Process::add_filefd(fds_, fd, ffd); } void Process::add_pipe(std::shared_ptr pipe) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "pipe=%s", D(pipe.get())); exec_point()->add_pipe(pipe); } void Process::drain_all_pipes() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); for (auto file_fd : *fds_) { if (!file_fd || !is_wronly(file_fd->flags())) { continue; } auto pipe = file_fd->pipe().get(); if (pipe) { pipe->drain_fd1_end(file_fd.get()); } } } std::vector>* Process::pass_on_fds(const bool execed) const { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "execed=%s", D(execed)); const int fds_size = fds_->size(); auto ret_fds = new std::vector>(fds_size); int last_fd = -1; for (int i = 0; i < fds_size; i++) { const FileFD* const raw_file_fd = (*fds_)[i].get(); if (raw_file_fd != nullptr) { if (!(execed && raw_file_fd->cloexec())) { /* The operations on the fds in the new process don't affect the fds in the parent, * thus create a copy of the parent's FileFD pointed to by a new shared pointer. */ (*ret_fds)[i] = std::make_shared(*raw_file_fd); last_fd = i; if (execed && raw_file_fd->close_on_popen()) { /* The newly exec()-ed process will not close inherited popen()-ed fds on pclose() */ (*ret_fds)[i]->set_close_on_popen(false); } } } } if (last_fd + 1 < fds_size) { /* A few of the last elements of the ret_fds vector is not used. Cut the size to the used * ones. */ ret_fds->resize(last_fd + 1); } return ret_fds; } void Process::AddPopenedProcess(int fd, ExecedProcess *proc) { fd2popen_child_[fd] = proc; } int Process::handle_pre_open(const int dirfd, const char * const ar_name, const size_t ar_len) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "dirfd=%d, ar_name=%s", dirfd, D(ar_name)); const FileName* name = get_absolute(dirfd, ar_name, ar_len); if (!name) { exec_point()->disable_shortcutting_bubble_up( "Could not find file name to mark as opened for writing"); return -1; } else { name->open_for_writing(this->exec_point()); return 0; } } int Process::handle_open(const int dirfd, const char * const ar_name, const size_t ar_len, const int flags, const mode_t mode, const int fd, const int error, int fd_conn, const int ack_num, const bool pre_open_sent, const bool tmp_file) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "dirfd=%d, ar_name=%s, flags=%d, mode=0%03o, pre_open_sent=%d, fd=%d, error=%d, " "fd_conn=%s, ack_num=%d", dirfd, D(ar_name), flags, mode, fd, pre_open_sent, error, D_FD(fd_conn), ack_num); /* O_TMPFILE is actually multiple bits, 0x410000 */ #ifdef O_TMPFILE const bool o_tmpfile_set = (flags & O_TMPFILE) == O_TMPFILE; #else const bool o_tmpfile_set = false; #endif const FileName* name = get_absolute(dirfd, ar_name, ar_len); if (!name) { // FIXME don't disable shortcutting if openat() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up("Invalid dirfd passed to openat()"); if (ack_num != 0) { ack_msg(fd_conn, ack_num); } return -1; } if (fd >= 0) { if (o_tmpfile_set) { /* open(..., O_TMPFILE) does not really open a file, but creates an unnamed temporary file in * the specified directory. Treat it like it was a memory backed file and later register the * directory's usage. */ add_filefd(fd, std::make_shared(fd, flags, FD_SPECIAL, this)); } else { add_filefd(fd, std::make_shared(name, fd, flags, this)); } } if (pre_open_sent) { /* When pre_open is sent the not interceptor nor the supervisor knew the outcome of open, but * the path's refcount is increased in the supervisor (+1). * In handle_open() std::make_shared(name, fd, flags, this) increases the refcount again * due to the FileFD construction if the file really got opened (+2), or the refcount is not * touched in case of an error (still +1). In both cases the refcount has to be decremented to * reflect actual usage (+1 vs +0) after the open(). */ name->close_for_writing(); } /* If O_TMPFILE was set we register the parent directory and it is not treated as a temporary * dir here. */ FileUsageUpdate update = FileUsageUpdate::get_from_open_params(name, o_tmpfile_set ? O_RDWR | O_DIRECTORY : flags, mode & 07777 & ~umask(), error, o_tmpfile_set ? false : tmp_file); if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up("Could not register the opening of a file", *name); if (ack_num != 0) { ack_msg(fd_conn, ack_num); } return -1; } /* handle_open() is designed to possibly send an early ACK. However it doesn't do so currently, * until we figure out #878 / #879. It always sends the ACK just before returning. */ if (ack_num != 0) { ack_msg(fd_conn, ack_num); } return 0; } /* Handle freopen(). See #650 for some juicy details. */ int Process::handle_freopen(const char * const ar_name, const size_t ar_len, const int flags, const int oldfd, const int fd, const int error, int fd_conn, const int ack_num, const bool pre_open_sent) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "ar_name=%s, flags=%d, oldfd=%d, fd=%d, error=%d, fd_conn=%s, ack_num=%d", D(ar_name), flags, oldfd, fd, error, D_FD(fd_conn), ack_num); if (ar_name != NULL) { /* old_fd is always closed, even if freopen() fails. It comes from stdio's bookkeeping, we have * no reason to assume that this close attempt failed. */ handle_close(oldfd, 0); /* Register the opening of the new file, no matter if succeeded or failed. */ return handle_open(AT_FDCWD, ar_name, ar_len, flags, 0666, fd, error, fd_conn, ack_num, pre_open_sent, false); } else { /* Without a name pre_open should not have been sent. */ assert(!pre_open_sent); /* Find oldfd. */ FileFD *file_fd = get_fd(oldfd); if (!file_fd || file_fd->filename() == nullptr) { /* Can't find oldfd, or wasn't opened by filename. Don't know what to do. */ exec_point()->disable_shortcutting_bubble_up( "Could not figure out old file name for freopen(..., NULL)", oldfd); if (ack_num != 0) { ack_msg(fd_conn, ack_num); } return -1; } else { /* Remember oldfd's filename before closing oldfd. We've tried to reopen the same file. */ const FileName *filename = file_fd->filename(); /* old_fd is always closed, even if freopen() fails. It comes from stdio's bookkeeping, we have * no reason to assume that this close attempt failed. */ handle_close(oldfd, 0); /* Register the reopening, no matter if succeeded or failed. */ return handle_open(AT_FDCWD, filename->c_str(), filename->length(), flags, 0666, fd, error, fd_conn, ack_num, pre_open_sent, false); } } } /* close that fd if open, silently ignore if not open */ int Process::handle_force_close(const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d", fd); if (get_fd(fd)) { return handle_close(fd, 0); } return 0; } void Process::handle_close(FileFD * file_fd) { auto pipe = file_fd->pipe().get(); if (pipe) { /* There may be data pending, drain it and register closure. */ pipe->handle_close(file_fd); file_fd->set_pipe(nullptr); } (*fds_)[file_fd->fd()].reset(); } int Process::handle_close(const int fd, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, error=%d", fd, error); if (error == EIO) { exec_point()->disable_shortcutting_bubble_up("IO error closing fd", fd); return -1; } else if (error == EINTR) { /* We don't know if the fd was closed or not, see #723. */ exec_point()->disable_shortcutting_bubble_up("EINTR while closing fd", fd); return -1; } else if (error == EBADF) { /* Process closed an fd unknown to it. Who cares? */ assert(!get_fd(fd)); return 0; } else { FileFD* file_fd = get_fd(fd); if (!file_fd) { if (error == 0) { exec_point()->disable_shortcutting_bubble_up( "Process closed an unknown fd successfully, " "which means interception missed at least one open()", fd); return -1; } else { exec_point()->disable_shortcutting_bubble_up( "Process closed an unknown, but valid fd unsuccessfully, " "which could mean that interception missed at least one open()", fd); return -1; } } else { handle_close(file_fd); return 0; } } } int Process::handle_closefrom(const int lowfd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "lowfd=%d", lowfd); return handle_close_range(lowfd, UINT_MAX, 0, 0); } int Process::handle_close_range(const unsigned int first, const unsigned int last, const int flags, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "first=%u, last=%u, flags=%d, error=%d", first, last, flags, error); (void)flags; /* might be unused */ if (!error) { for (auto& file_fd : *fds_) { if (!file_fd) { continue; } unsigned int fd = static_cast(file_fd->fd()); if (fd >= first && fd <= last) { if (flags & CLOSE_RANGE_CLOEXEC) { /* Don't close, just set the cloexec bit. */ file_fd->set_cloexec(true); } else { handle_close(file_fd.get()); } } } } return 0; } int Process::handle_dlopen(const char * const absolute_filename, const size_t absolute_filename_len, const char * const looked_up_filename, const size_t looked_up_filename_len, const bool error, int fd_conn, int ack_num) { if (absolute_filename) { /* When failing to dlopen() a file assume it is not present. * This is a safe assumption for shortcutting purposes, since the cache entry * will require the the file to be missing to shortcut the process and if the file * is missing dlopen() would have failed for sure. */ return handle_open(AT_FDCWD, absolute_filename, absolute_filename_len, O_RDONLY, 0, -1, error ? ENOENT : 0, fd_conn, ack_num, false, false); } else if (looked_up_filename) { /* Failed dlopen() could not find the file on the search path.*/ /* TODO(rbalint) allow shortcutting the process and mark the file as missing on all the * search path entries as described in dlopen(3). */ (void)looked_up_filename_len; exec_point()->disable_shortcutting_bubble_up("Process failed to dlopen() ", looked_up_filename); return -1; } else { if (error) { exec_point()->disable_shortcutting_bubble_up("Process failed to dlopen() the main program"); return -1; } } return 0; } int Process::handle_truncate(const char * const ar_name, const size_t ar_len, const off_t length, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "ar_name=%s, length=%" PRIoff ", error=%d", D(ar_name), length, error); const FileName* name = get_absolute(AT_FDCWD, ar_name, ar_len); if (!name) { exec_point()->disable_shortcutting_bubble_up("Could not find file to truncate()"); return -1; } /* truncate() always sends pre_open. */ name->close_for_writing(); // FIXME Will be a bit tricky to implement shortcutting, see #637. (void)length; (void)error; exec_point()->disable_shortcutting_bubble_up("truncate() is not supported"); return 0; } int Process::handle_unlink(const int dirfd, const char * const ar_name, const size_t ar_len, const int flags, const int error, const bool pre_open_sent) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "dirfd=%d, ar_name=%s, flags=%d, error=%d, pre_open_sent=%d", dirfd, D(ar_name), flags, error, pre_open_sent); const FileName* name = get_absolute(dirfd, ar_name, ar_len); if (!name) { // FIXME don't disable shortcutting if unlinkat() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up("Invalid dirfd passed to unlinkat()"); return -1; } if (pre_open_sent) { name->close_for_writing(); } if (!error) { /* There is no need to call register_parent_directory(). * If the process created the file to unlink it is already registered and if it was present * before the process started then registering the file to unlink already implies the existence * of the parent dir. */ // FIXME When a directory is removed, register that it was an _empty_ directory FileUsageUpdate update = FileUsageUpdate(name, flags & AT_REMOVEDIR ? ISDIR : ISREG, true); if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the unlink or rmdir", *name); return -1; } } else if (error == ENOENT) { FileUsageUpdate update(name, NOTEXIST); if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up("Could not register the unlink or rmdir", *name); return -1; } } return 0; } int Process::handle_fstatat(const int fd, const char * const ar_name, const size_t ar_len, const int flags, const mode_t st_mode, const off_t st_size, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, ar_name=%s, flags=%d, st_mode=%d, st_size=%" PRIoff ", error=%d", fd, D(ar_name), flags, st_mode, st_size, error); const FileName *name; #ifndef AT_EMPTY_PATH (void)flags; #endif if (ar_name == nullptr #ifdef AT_EMPTY_PATH || (ar_name[0] == '\0' && (flags & AT_EMPTY_PATH)) #endif ) { /* Operating on an opened fd, i.e. fstat() or fstatat("", AT_EMPTY_PATH). */ FileFD *file_fd = get_fd(fd); if (!file_fd) { if (error == 0) { exec_point()->disable_shortcutting_bubble_up( "Process fstatat()ed an unknown fd successfully, " "which means interception missed at least one open()", fd); return -1; } else { /* Invalid fd passed to fstat(), or something like that. */ return 0; } } name = file_fd->filename(); if (!name) { /* Cannot find file name, maybe it's a pipe or similar. Take no action. */ return 0; } } else { /* Operating on a file reached by its name, like [l]stat(), or fstatat() with a non-empty * path relative to some dirfd (called 'fd' here). */ name = get_absolute(fd, ar_name, ar_len); if (!name) { // FIXME don't disable shortcutting if stat() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up( "Invalid dirfd or filename passed to stat() variant"); return -1; } } FileUsageUpdate update = FileUsageUpdate::get_from_stat_params(name, st_mode, st_size, error); if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up("Could not register fstatat() on", *name); return -1; } return 0; } int Process::handle_statfs(const char * const a_name, const size_t length, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "a_name=%s, error=%d", D(a_name), error); if (a_name == nullptr) { /* Operating on an opened fd, i.e. fstatfs(). */ if (quirks & FB_QUIRK_IGNORE_STATFS) { return 0; } else { exec_point()->disable_shortcutting_bubble_up( "fstatfs() family operating on fds is not supported"); return -1; } } /* Operating on a file reached by its name, made absolute by the interceptor. */ #ifdef FB_EXTRA_DEBUG assert(path_is_absolute(a_name)); #endif const FileName* name = FileName::Get(a_name, length); if (error == ENOENT) { FileUsageUpdate update(name, NOTEXIST); if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up("Could not register failed statfs()", *name); } } else if (!(quirks & FB_QUIRK_IGNORE_STATFS)) { // TODO(rbalint) add more supported cases exec_point()->disable_shortcutting_bubble_up( "Successful statfs() calls are not supported."); return -1; } return 0; } int Process::handle_faccessat(const int dirfd, const char * const ar_name, const size_t ar_name_len, const int mode, const int flags, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "dirfd=%d, ar_name=%s, mode=%d, flags=%d, error=%d", dirfd, D(ar_name), mode, flags, error); (void)flags; /* AT_EACCESS is currently ignored, we assume no setuid/setgid in the game. */ /* Note: faccessat() obviously cannot operate on an already opened file, doesn't support * AT_EMPTY_PATH. */ const FileName* name = get_absolute(dirfd, ar_name, ar_name_len); if (!name) { // FIXME don't disable shortcutting if chmod() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up( "Invalid dirfd or filename passed faccessat()"); return -1; } FileUsageUpdate update(name); if (mode == F_OK) { /* Checked if there's something at this name. It's effectively a stat(), without getting to know * the fields in "struct stat". */ if (!error) { update.set_initial_type(EXIST); } else { update.set_initial_type(NOTEXIST); } } else { /* We got to know something about some of the read, write, execute permission bits. We assume it * corresponds to the owner's permission bit. By this we assume that the current user isn't * root, that there isn't a setuid/setgid bit in the game, and that there isn't any read-only * filesystem involved. */ if (!error) { /* Something exists at the given location, and all of the requested bits are set. */ update.set_initial_type(EXIST); if (mode & R_OK) { update.set_initial_mode_bits(S_IRUSR, S_IRUSR); /* 0400 */ } if (mode & W_OK) { update.set_initial_mode_bits(S_IWUSR, S_IWUSR); /* 0200 */ } if (mode & X_OK) { update.set_initial_mode_bits(S_IXUSR, S_IXUSR); /* 0100 */ } } else if (error == EACCES) { /* The requested permissions aren't set. Unfortunately we don't know if the problem is with * this particular or some preceding path component, perhaps the access()ed file doesn't even * exist. Also, if multiple bits were requested then we don't know which of them are unset or * otherwise problematic, and which ones are set. * stat() the file, and remember the relevant bits from that information. */ struct stat64 st; if (stat64(name->c_str(), &st) < 0) { /* There's a permission error somewhere earlier along the PATH. Firebuild does not support * this error anywhere else, we always pretend the file doesn't exist. Do that here, too. * FIXME This will be fixed across Firebuild by #862. */ update.set_initial_type(NOTEXIST); } else { /* We know all the permission bits of the file. Record the ones that were requested by the * access() call. */ update.set_initial_type(EXIST); bool unset_bit_seen = false; if (mode & R_OK) { update.set_initial_mode_bits(st.st_mode & S_IRUSR, S_IRUSR); /* 0400 */ unset_bit_seen = unset_bit_seen || ((st.st_mode & S_IRUSR) == 0); } if (mode & W_OK) { update.set_initial_mode_bits(st.st_mode & S_IWUSR, S_IWUSR); /* 0200 */ unset_bit_seen = unset_bit_seen || ((st.st_mode & S_IWUSR) == 0); } if (mode & X_OK) { update.set_initial_mode_bits(st.st_mode & S_IXUSR, S_IXUSR); /* 0100 */ unset_bit_seen = unset_bit_seen || ((st.st_mode & S_IXUSR) == 0); } if (!unset_bit_seen) { /* We've stat()ed the file, and all the permission bits requested by access() are set. * Why did access() return EACCES then?? It's a mystery. */ exec_point()->disable_shortcutting_bubble_up("stat() returned unexpected permission bits " "for", *name); return -1; } } } else { /* The entry doesn't exist, or at least we cannot handle it. */ update.set_initial_type(NOTEXIST); } } if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the faccessat of a file", *name); return -1; } return 0; } int Process::handle_fchmodat(const int fd, const char * const ar_name, const size_t ar_len, const mode_t mode, const int flags, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, ar_name=%s, mode=%d, flags=%d, error=%d", fd, D(ar_name), mode, flags, error); (void)mode; /* No need to remember what we chmod to, we'll stat at the end. */ const FileName *name; ExecedProcess *register_from = exec_point(); /* Linux's fchmodat() doesn't support AT_EMPTY_PATH. The attempt to fix it at * https://patchwork.kernel.org/project/linux-fsdevel/patch/148830142269.7103.7429913851447595016.stgit@bahia/ * has apparently stalled. * * FreeBSD supports it. * * Let's just go on assuming it's supported to make our code consistent with fstatat(), future * fchownat() and possibly some other methods too. */ #ifndef AT_EMPTY_PATH (void)flags; #endif if (ar_name == nullptr #ifdef AT_EMPTY_PATH || (ar_name[0] == '\0' && (flags & AT_EMPTY_PATH)) #endif ) { /* Operating on an opened fd, i.e. fchmod() or fchmodat("", AT_EMPTY_PATH). */ FileFD *file_fd = get_fd(fd); if (!file_fd) { if (error == 0) { exec_point()->disable_shortcutting_bubble_up( "Process fchmodat()ed an unknown fd successfully, " "which means interception missed at least one open()", fd); return -1; } else { /* Invalid fd passed to fchmod(), or something like that. */ return 0; } } /* Disable shortcutting the processes that inherited this fd. If the fd was opened by the * current process (or a fork ancestor), as it's usually the case, then this is an empty set. * See #927. */ register_from = file_fd->opened_by() ? file_fd->opened_by()->exec_point() : nullptr; exec_point()->disable_shortcutting_bubble_up_to_excl( register_from, "fchmod() on an inherited fd not supported"); name = file_fd->filename(); if (!name) { /* Cannot find file name, maybe it's a pipe or similar. Take no further action. */ return 0; } } else { /* Operating on a file reached by its name, like [l]chmod(), or fchmodat() with a non-empty * path relative to some dirfd (called 'fd' here). */ name = get_absolute(fd, ar_name, ar_len); if (!name) { // FIXME don't disable shortcutting if chmod() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up( "Invalid dirfd or filename passed fchmodat()"); return -1; } } if (error) { if (register_from) { register_from->disable_shortcutting_bubble_up("Cannot register a failed fchmodat() on", *name); } return -1; } FileUsageUpdate update(name, EXIST, false, true); if (register_from) { if (!register_from->register_file_usage_update(name, update)) { register_from->disable_shortcutting_bubble_up("Could not register fchmodat() on", *name); return -1; } } return 0; } #ifdef __linux__ int Process::handle_memfd_create(const int flags, const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "flags=%d, fd=%d", flags, fd); add_filefd(fd, std::make_shared(fd, (flags & MFD_CLOEXEC) ? O_CLOEXEC : 0, FD_SPECIAL, this)); return 0; } int Process::handle_timerfd_create(const int flags, const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "flags=%d, fd=%d", flags, fd); add_filefd(fd, std::make_shared(fd, (flags & TFD_CLOEXEC) ? O_CLOEXEC : 0, FD_SPECIAL, this)); return 0; } #ifdef __linux__ int Process::handle_epoll_create(const int flags, const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "flags=%d, fd=%d", flags, fd); add_filefd(fd, std::make_shared(fd, (flags & EPOLL_CLOEXEC) ? O_CLOEXEC : 0, FD_SPECIAL, this)); return 0; } #endif int Process::handle_eventfd(const int flags, const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "flags=%d, fd=%d", flags, fd); add_filefd(fd, std::make_shared(fd, (flags & EFD_CLOEXEC) ? O_CLOEXEC : 0, FD_SPECIAL, this)); return 0; } int Process::handle_signalfd(const int oldfd, const int flags, const int newfd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "oldfd=%d, flags=%d newfd=%d", oldfd, flags, newfd); if (oldfd == -1) { add_filefd(newfd, std::make_shared(newfd, (flags & SFD_CLOEXEC) ? O_CLOEXEC : 0, FD_SPECIAL, this)); } else { /* Reusing old fd, nothing to do.*/ // TODO(rbalint) maybe the O_CLOEXEC flag could be updated, but the man page sounds like as // if the flags are not used when reusing the old fd. } return 0; } #endif int Process::handle_rmdir(const char * const ar_name, const size_t ar_name_len, const int error, const bool pre_open_sent) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "ar_name=%s, error=%d, pre_open_sent=%d", D(ar_name), error, pre_open_sent); return handle_unlink(AT_FDCWD, ar_name, ar_name_len, AT_REMOVEDIR, error, pre_open_sent); } int Process::handle_mkdir(const int dirfd, const char * const ar_name, const size_t ar_len, const int error, const bool tmp_dir) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "dirfd=%d, ar_name=%s, error=%d", dirfd, D(ar_name), error); const FileName* name = get_absolute(dirfd, ar_name, ar_len); if (!name) { // FIXME don't disable shortcutting if mkdirat() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up("Invalid dirfd passed to mkdirat()"); return -1; } FileUsageUpdate update = FileUsageUpdate::get_from_mkdir_params(name, error, tmp_dir); if (!exec_point()->register_file_usage_update(name, update)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the directory creation ", *name); return -1; } return 0; } void Process::handle_pipe_request(const int flags, const int fd_conn) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "flags=%d", flags); pending_pipe_t pending_pipe {}; /* Creating a pipe consists of multiple steps, but they are all guarded by a single lock in the * interceptor, so we can't have two pipe creations running in parallel in the same Process. */ assert(!proc_tree->Proc2PendingPipe(this)); /* Create an intercepted unnamed pipe, which is actually two unnamed pipes: one from the * interceptor up to the supervisor, and one from the supervisor back down to the interceptor. */ int up[2], down[2]; FBBCOMM_Builder_pipe_created response; if (fb_pipe2(up, flags) < 0) { response.set_error_no(errno); send_fbb(fd_conn, 0, reinterpret_cast(&response)); return; } if (fb_pipe2(down, flags) < 0) { response.set_error_no(errno); send_fbb(fd_conn, 0, reinterpret_cast(&response)); close(up[0]); close(up[1]); return; } if (epoll->is_added_fd(down[1])) { down[1] = epoll->remap_to_not_added_fd(down[1]); } /* Send the "pipe_created" message with two attached fds. * down[0] becomes pipefd[0], up[1] becomes pipefd[1] in the interceptor. */ int fds_to_send[2] = { down[0], up[1] }; send_fbb(fd_conn, 0, reinterpret_cast(&response), fds_to_send, 2); /* The endpoints we've just sent are no longer needed in the supervisor. */ close(down[0]); close(up[1]); /* Remember the other two fds, we'll need them in the "pipe_fds" step. * According to the supervisor terminology: * - fd0 corresponds to the original pipe's fd0, where we're writing to, which is now down[1]; * - fd1 corresponds to the original pipe's fd1, where we're reading from, which is now up[0]. */ pending_pipe.fd0 = down[1]; pending_pipe.fd1 = up[0]; pending_pipe.flags = flags; /* We need the supervisor end of theses pipes to be nonblocking. Maybe they already are. */ if (!(flags & O_NONBLOCK)) { fcntl(pending_pipe.fd0, F_SETFL, flags | O_NONBLOCK); fcntl(pending_pipe.fd1, F_SETFL, flags | O_NONBLOCK); } firebuild::bump_fd_age(pending_pipe.fd0); firebuild::bump_fd_age(pending_pipe.fd1); proc_tree->QueuePendingPipe(this, pending_pipe); /* To be continued in handle_pipe_fds() upon the next interceptor message. */ } void Process::handle_pipe_fds(const int fd0, const int fd1) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd0=%d fd1=%d", fd0, fd1); /* Continuing from a previous handle_pipe_request(). */ /* Creating a pipe consists of multiple steps, but they are all guarded by a single lock in the * interceptor, so we can't have two pipe creations running in parallel in the same Process. */ const pending_pipe_t *pending_pipe = proc_tree->Proc2PendingPipe(this); assert(pending_pipe != nullptr); /* Validate fds. */ if (get_fd(fd0)) { /* We already have this fd, probably missed a close(). */ exec_point()->disable_shortcutting_bubble_up( "Process created an fd which is known to be open, " "which means interception missed at least one close()", fd0); close(pending_pipe->fd0); close(pending_pipe->fd1); } else if (get_fd(fd1)) { /* we already have this fd, probably missed a close(). */ exec_point()->disable_shortcutting_bubble_up( "Process created an fd which is known to be open, " "which means interception missed at least one close()", fd1); close(pending_pipe->fd0); close(pending_pipe->fd1); } else { #ifdef __clang_analyzer__ /* Scan-build reports a false leak for the correct code. This is used only in static * analysis. It is broken because all shared pointers to the Pipe must be copies of * the shared self pointer stored in it. */ auto pipe = std::make_shared(pending_pipe->fd0, this); #else auto pipe = (new Pipe(pending_pipe->fd0, this))->shared_ptr(); #endif add_filefd(fd0, std::make_shared( fd0, (pending_pipe->flags & ~O_ACCMODE) | O_RDONLY, pipe->fd0_shared_ptr(), this, false)); auto ffd1 = std::make_shared(fd1, (pending_pipe->flags & ~O_ACCMODE) | O_WRONLY, pipe->fd1_shared_ptr(), this, false); add_filefd(fd1, ffd1); /* Empty recorders array. We don't start recording after a pipe(), this data wouldn't be * used anywhere. We only start recording after an exec(), to catch the traffic as seen * from that potential shortcutting point. */ auto recorders = std::vector>(); pipe->add_fd1_and_proc(pending_pipe->fd1, ffd1.get(), exec_point(), recorders); add_pipe(pipe); } /* Back to the default state. */ proc_tree->DropPendingPipe(this); } void Process::handle_socket(const int domain, const int type, const int protocol, const int ret, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "domain=%d, type=%d, protocol=%d, ret=%d, error=%d", domain, type, protocol, ret, error); /* Creating a socket is fine from shortcutting POV as long as no communication takes place * over it. The created fd needs to be tracked, though. */ (void)domain; (void)protocol; if (!error) { if (get_fd(ret)) { /* We already have this fd, probably missed a close(). */ exec_point()->disable_shortcutting_bubble_up( "Process created an fd which is known to be open, " "which means interception missed at least one close()", ret); handle_close(ret); } const int flags = O_RDWR #ifdef SOCK_CLOEXEC | ((type & SOCK_CLOEXEC) ? O_CLOEXEC : 0) #endif #ifdef SOCK_NONBLOCK | ((type & SOCK_NONBLOCK) ? O_NONBLOCK : 0) #endif | 0; add_filefd(ret, std::make_shared(ret, flags, FD_SPECIAL, this)); #if defined(SOCK_CLOEXEC) && defined(SOCK_NONBLOCK) switch (type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) { #elif defined(SOCK_CLOEXEC) #error "" #elif defined(SOCK_NONBLOCK) #else switch (type) { #endif #if defined(SOCK_DGRAM) case SOCK_DGRAM: #endif #if defined(SOCK_RAW) case SOCK_RAW: #endif #if defined(SOCK_PACKET) case SOCK_PACKET: #endif #if defined(SOCK_DGRAM) || defined(SOCK_RAW) || defined(SOCK_PACKET) { exec_point()->disable_shortcutting_bubble_up( "SOCK_DGRAM, SOCK_RAW and SOCK_PACKET sockets are not supported"); break; } #endif default: /* Other socket types are OK since they require connect/bind/listen operations before * data can be exchanged with other processes and those disable shortcutting. */ break; } } else { // TODO(rbalint) maybe add the result as a process input and allow shortcutting // in the same circumstances, for example when hitting EACCESS in restricted build // environment. exec_point()->disable_shortcutting_bubble_up("socket() call failed"); } } void Process::handle_socketpair(const int domain, const int type, const int protocol, const int fd0, const int fd1, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "domain=%d, type=%d, protocol=%d, fd0=%d, fd1=%d, error=%d", domain, type, protocol, fd0, fd1, error); /* Creating a socketpair is fine from shortcutting POV, it behaves like an anonymous file * (e.g. memfd). */ (void)domain; (void)protocol; #if !defined(SOCK_CLOEXEC) && !defined(SOCK_NONBLOCK) (void)type; #endif if (!error) { for (const int fd : {fd0, fd1}) { if (get_fd(fd)) { /* We already have this fd, probably missed a close(). */ exec_point()->disable_shortcutting_bubble_up( "Process created an fd which is known to be open, " "which means interception missed at least one close()", fd); handle_close(fd); } const int flags = O_RDWR #ifdef SOCK_CLOEXEC | ((type & SOCK_CLOEXEC) ? O_CLOEXEC : 0) #endif #ifdef SOCK_NONBLOCK | ((type & SOCK_NONBLOCK) ? O_NONBLOCK : 0) #endif | 0; add_filefd(fd, std::make_shared(fd, flags, FD_SPECIAL, this)); } } else { /* This is ulikely to happen and may not be deterministic .*/ exec_point()->disable_shortcutting_bubble_up("socketpair() call failed"); } } int Process::handle_dup3(const int oldfd, const int newfd, const int flags, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "oldfd=%d, newfd=%d, flags=%d, error=%d", oldfd, newfd, flags, error); switch (error) { case EBADF: case EBUSY: case EINTR: case EINVAL: case ENFILE: { /* dup() failed */ return 0; } case 0: default: break; } /* validate fd-s */ if (!get_fd(oldfd)) { /* we already have this fd, probably missed a close() */ exec_point()->disable_shortcutting_bubble_up( "Process created an fd which is known to be open, " "which means interception missed at least one close()", oldfd); return -1; } /* dup2()'ing an existing fd to itself returns success and does nothing */ if (oldfd == newfd) { return 0; } handle_force_close(newfd); add_filefd(newfd, std::make_shared(newfd, (*fds_)[oldfd], flags & O_CLOEXEC)); return 0; } int Process::handle_rename(const int olddirfd, const char * const old_ar_name, const size_t old_ar_len, const int newdirfd, const char * const new_ar_name, const size_t new_ar_len, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "olddirfd=%d, old_ar_name=%s, newdirfd=%d, new_ar_name=%s, error=%d", olddirfd, D(old_ar_name), newdirfd, D(new_ar_name), error); /* * Note: rename() is different from "mv" in at least two aspects: * * - Can't move to a containing directory, e.g. * rename("/home/user/file.txt", "/tmp"); (or ... "/tmp/") * fails, you have to specify the full new name like * rename("/home/user/file.txt", "/tmp/file.txt"); * instead. * * - If the source is a directory then the target can be an empty directory, * which will be atomically removed beforehand. E.g. if "mytree" is a directory then * mkdir("/tmp/target"); * rename("mytree", "/tmp/target"); * will result in the former "mytree/file.txt" becoming "/tmp/target/file.txt", * with no "mytree" component. "target" has to be an empty directory for this to work. */ const FileName* old_name = get_absolute(olddirfd, old_ar_name, old_ar_len); const FileName* new_name = get_absolute(newdirfd, new_ar_name, new_ar_len); /* Rename always sends pre_open, which pretends that the file was opened for writing. */ if (old_name) old_name->close_for_writing(); if (new_name) new_name->close_for_writing(); if (error) { switch (error) { case EEXIST: { FileUsageUpdate update(new_name); update.set_initial_type(EXIST); if (!exec_point()->register_file_usage_update(new_name, update)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the renaming (to)", *new_name); return -1; } return 0; } default: // TODO(rbalint) add detailed error handling exec_point()->disable_shortcutting_bubble_up("Failed rename() is not supported"); } return -1; } if (!old_name || !new_name) { // FIXME don't disable shortcutting if renameat() failed due to the invalid dirfd exec_point()->disable_shortcutting_bubble_up("Invalid dirfd passed to renameat()"); return -1; } struct stat64 st; if (lstat64(new_name->c_str(), &st) < 0 || !S_ISREG(st.st_mode)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the renaming of non-regular file", *old_name); return -1; } /* It's tricky because the renaming has already happened, there's supposedly nothing * at the old filename. Yet we need to register that we read that file with its * particular hash value. * FIXME we compute the hash twice, both for the old and new location. * FIXME refactor so that it plays nicer together with register_file_usage_update(). */ /* Register the opening for reading at the old location, although read the file's hash from the * new location. */ FileUsageUpdate update_old = FileUsageUpdate::get_oldfile_usage_from_rename_params(old_name, new_name, error); if (!exec_point()->register_file_usage_update(old_name, update_old)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the renaming (from)", *old_name); return -1; } /* Register the opening for writing at the new location */ // TODO(rbalint) fix error handling, it is way more complicated for rename than for open. FileUsageUpdate update_new = FileUsageUpdate::get_newfile_usage_from_rename_params(new_name, error); if (!exec_point()->register_file_usage_update(new_name, update_new)) { exec_point()->disable_shortcutting_bubble_up( "Could not register the renaming (to)", *new_name); return -1; } return 0; } int Process::handle_symlink(const char * const target, const int newdirfd, const char * const new_ar_name, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "target=%s, newdirfd=%d, new_ar_name=%s, error=%d", D(target), newdirfd, D(new_ar_name), error); if (!error) { exec_point()->disable_shortcutting_bubble_up( "Process created a symlink", " ([" + d(newdirfd) + "]" + d(new_ar_name) + " -> " + d(target) + ")"); return -1; } return 0; } int Process::handle_clear_cloexec(const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d", fd); if (!get_fd(fd)) { exec_point()->disable_shortcutting_bubble_up( "Process successfully cleared cloexec on fd which is known to be closed, " "which means interception missed at least one open()", fd); return -1; } (*fds_)[fd]->set_cloexec(false); return 0; } int Process::handle_fcntl(const int fd, const int cmd, const int arg, const int ret, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, cmd=%d, arg=%d, ret=%d, error=%d", fd, cmd, arg, ret, error); switch (cmd) { case F_DUPFD: return handle_dup3(fd, ret, 0, error); case F_DUPFD_CLOEXEC: return handle_dup3(fd, ret, O_CLOEXEC, error); case F_SETFD: if (error == 0) { if (!get_fd(fd)) { exec_point()->disable_shortcutting_bubble_up( "Process successfully fcntl'ed on fd which is known to be closed, " "which means interception missed at least one open()", fd); return -1; } (*fds_)[fd]->set_cloexec(arg & FD_CLOEXEC); } return 0; default: exec_point()->disable_shortcutting_bubble_up("Process executed unsupported fcntl ", d(cmd)); return 0; } } int Process::handle_ioctl(const int fd, const int cmd, const int ret, const int error) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, cmd=%d, ret=%d, error=%d", fd, cmd, ret, error); (void) ret; switch (cmd) { case FIOCLEX: if (error == 0) { if (!get_fd(fd)) { exec_point()->disable_shortcutting_bubble_up( "Process successfully ioctl'ed on fd which is known to be closed, " "which means interception missed at least one open()", fd); return -1; } (*fds_)[fd]->set_cloexec(true); } return 0; case FIONCLEX: if (error == 0) { if (!get_fd(fd)) { exec_point()->disable_shortcutting_bubble_up( "Process successfully ioctl'ed on fd which is known to be closed, " "which means interception missed at least one open()", fd); return -1; } (*fds_)[fd]->set_cloexec(false); } return 0; default: exec_point()->disable_shortcutting_bubble_up("Process executed unsupported ioctl", " " + d(cmd)); return 0; } } void Process::handle_read_from_inherited(const int fd, const bool is_pread) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, is_pread=%s", fd, D(is_pread)); (void)is_pread; /* unused */ FileFD *file_fd = get_fd(fd); if (!file_fd) { exec_point()->disable_shortcutting_bubble_up( "Process successfully read from fd which is known to be closed, which means interception" " missed at least one open()", fd); return; } else if (file_fd->type() == FD_PIPE_IN && file_fd->pipe() && file_fd->fd() == exec_point()->jobserver_fd_r()) { // TODO(rbalint) add further check that the process chain did not reopen the fd breaking the // connection to the jobserver /* It is OK to read from the jobserver. It should not affect the build results. */ return; } const ExecedProcess *file_fd_opened_by_exec_point = file_fd->opened_by() ? file_fd->opened_by()->exec_point() : nullptr; if (file_fd_opened_by_exec_point == exec_point()) { exec_point()->disable_shortcutting_bubble_up( "Process sent handle_read_from_inherited for a non-inherited fd", fd); return; } if (file_fd->type() == FD_IGNORED) { return; } Process* opened_by = file_fd->opened_by(); exec_point()->disable_shortcutting_bubble_up_to_excl( opened_by ? opened_by->exec_point() : nullptr, "Process read from inherited fd ", fd); } void Process::handle_write_to_inherited(const int fd, const bool is_pwrite) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, is_pwrite=%s", fd, D(is_pwrite)); FileFD *file_fd = get_fd(fd); if (!file_fd) { exec_point()->disable_shortcutting_bubble_up( "Process successfully wrote to fd which is known to be closed, which means interception" " missed at least one open()", fd); return; } else if (file_fd->type() == FD_PIPE_OUT && file_fd->pipe() && file_fd->fd() == exec_point()->jobserver_fd_w()) { // TODO(rbalint) add further check that the process chain did not reopen the fd breaking the // connection to the jobserver /* It is OK to write to the jobserver, but jobserver communication should not be cached. */ return; } const ExecedProcess *file_fd_opened_by_exec_point = file_fd->opened_by() ? file_fd->opened_by()->exec_point() : nullptr; if (file_fd_opened_by_exec_point == exec_point()) { exec_point()->disable_shortcutting_bubble_up( "Process sent handle_write_to_inherited for a non-inherited fd", fd); return; } if (file_fd->type() == FD_IGNORED) { return; } if (is_pwrite) { Process* opened_by = file_fd->opened_by(); exec_point()->disable_shortcutting_bubble_up_to_excl( opened_by ? opened_by->exec_point() : nullptr, "Process called pwrite() on inherited fd ", fd); } else if (!file_fd->pipe() && !file_fd->filename()) { Process* opened_by = file_fd->opened_by(); exec_point()->disable_shortcutting_bubble_up_to_excl( opened_by ? opened_by->exec_point() : nullptr, "Process wrote to inherited non-pipe and non-file fd ", fd); } } void Process::handle_seek_in_inherited(const int fd, const bool modify_offset) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, modify_offset=%s", fd, D(modify_offset)); FileFD *file_fd = get_fd(fd); if (!file_fd) { exec_point()->disable_shortcutting_bubble_up( "Process successfully seeked in an fd which is known to be closed, which means interception" " missed at least one open()", fd); return; } const ExecedProcess *file_fd_opened_by_exec_point = file_fd->opened_by() ? file_fd->opened_by()->exec_point() : nullptr; if (file_fd_opened_by_exec_point == exec_point()) { exec_point()->disable_shortcutting_bubble_up( "Process sent handle_seek_in_inherited for a non-inherited fd", fd); return; } if (file_fd->type() == FD_IGNORED) { return; } // FIXME Handle the !modify_offset case if (modify_offset && !file_fd->pipe()) { Process* opened_by = file_fd->opened_by(); exec_point()->disable_shortcutting_bubble_up_to_excl( opened_by ? opened_by->exec_point() : nullptr, "Process seeked in inherited non-pipe fd ", fd); } } void Process::handle_inherited_fd_offset(const int fd, const int64_t offset) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d, offset==%" PRId64, fd, offset); FileFD *file_fd = exec_point()->get_fd(fd); if (!file_fd) { exec_point()->disable_shortcutting_bubble_up( "Process reported offset for an intercepted seekable fd which is no known to be open", fd); return; } exec_point()->disable_shortcutting_only_this( "Inherited writable non-append fd not seeked to its end"); for (inherited_file_t& inherited_file : exec_point()->inherited_files()) { if (inherited_file.fds[0] == fd) { inherited_file.start_offset = offset; break; } } } void Process::handle_recvmsg_scm_rights(const bool cloexec, const std::vector fds) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "cloexec=%s fds=%s", D(cloexec), D(fds)); for (int fd : fds) { FileFD *file_fd = get_fd(fd); if (file_fd) { exec_point()->disable_shortcutting_bubble_up( "Process successfully received fd via SCM_RIGHTS which is known to be open, which means" " interception missed at least one close()", fd); } else { add_filefd(fd, std::make_shared(fd, cloexec ? O_CLOEXEC : 0, FD_SCM_RIGHTS, this)); } } exec_point()->disable_shortcutting_bubble_up("Receiving an fd via SCM_RIGHTS is not supported"); } void Process::handle_set_wd(const char * const ar_d, const size_t ar_d_len) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "ar_d=%s", ar_d); wd_ = get_absolute(AT_FDCWD, ar_d, ar_d_len); assert(wd_); add_wd(wd_); } void Process::handle_set_fwd(const int fd) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "fd=%d", fd); const FileFD* ffd = get_fd(fd); if (!ffd) { exec_point()->disable_shortcutting_bubble_up( "Process successfully fchdir()'ed to an fd which is known to be closed, which means " "interception missed at least one open()", fd); return; } wd_ = ffd->filename(); assert(wd_); add_wd(wd_); } void Process::handle_umask(mode_t old_umask, mode_t new_umask) { if (umask() != old_umask) { exec_point()->disable_shortcutting_bubble_up("Old umask mismatches, which means " "interception missed at least one umask()"); } umask_ = new_umask & 0777; } const FileName* Process::get_fd_filename(int fd) const { if (fd == AT_FDCWD) { return wd(); } else { const FileFD* ffd = get_fd(fd); if (ffd) { return ffd->filename(); } else { return nullptr; } } } const FileName* Process::get_absolute(const int dirfd, const char * const name, ssize_t length) const { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "dirfd=%d, name=%s, length=%" PRIssize, dirfd, D(name), length); if (path_is_absolute(name)) { return FileName::Get(name, length); } else { char on_stack_buf[2048], *buf; const FileName* dir = get_fd_filename(dirfd); if (!dir) { return nullptr; } const ssize_t name_length = (length == -1) ? strlen(name) : length; /* Both dir and name are in canonical form thanks to the interceptor, only "." and "" * need handling here. See make_canonical() in the interceptor. */ if (name_length == 0 || (name_length == 1 && name[0] == '.')) { return dir; } const size_t on_stack_buffer_size = sizeof(on_stack_buf); const bool separator_needed = (dir->c_str()[dir->length() - 1] != '/'); /* Only "/" should end with a separator */ assert(separator_needed || dir->length() == 1); const size_t total_buf_len = dir->length() + (separator_needed ? 1 : 0) + name_length + 1; if (on_stack_buffer_size < total_buf_len) { buf = reinterpret_cast(malloc(total_buf_len)); } else { buf = reinterpret_cast(on_stack_buf); } memcpy(buf, dir->c_str(), dir->length()); size_t offset = dir->length(); if (separator_needed) { buf[offset++] = '/'; } memcpy(buf + offset, name, name_length); buf[total_buf_len - 1] = '\0'; const FileName* ret = FileName::Get(buf, total_buf_len - 1); if (on_stack_buffer_size < total_buf_len) { free(buf); } return ret; } } static bool argv_match_from_offset(const std::vector& actual, const int actual_offset, const std::vector& expected, const int expected_offset) { /* For the remaining parameters exact match is needed. */ if (actual.size() - actual_offset != expected.size() - expected_offset) { return false; } for (unsigned int i = expected_offset; i < expected.size(); i++) { if (actual[actual_offset + i - expected_offset] != expected[expected_offset]) { return false; } } return true; } static bool argv_matches_expectation(const std::vector& actual, const std::vector& expected) { /* When launching ["foo", "arg1"], the new process might be something like * ["/bin/bash", "-e", "./foo", "arg1"]. * * If the program name already contained a slash, or if it refers to a binary executable, * it remains unchanged. If it didn't contain a slash and it refers to an interpreted file, * it's replaced by an absolute or relative path containing at least one slash (e.g. "./foo"). * * A single interpreter might only add one additional command line parameter, but there can be * a chain of interpreters, so allow for arbitrarily many prepended parameters. */ if (actual.size() == expected.size()) { /* If the length is the same, exact match is required. */ return actual == expected; } if (actual.size() > expected.size()) { /* If the length grew, check the expected arg0. */ int offset = actual.size() - expected.size(); if (expected[0].find('/') != std::string::npos) { /* If it contained a slash, exact match is needed. */ if (actual[offset] != expected[0]) { return false; } } else { /* If it didn't contain a slash, it needed to get prefixed with some path. */ int expected_arg0_len = expected[0].length(); int actual_arg0_len = actual[offset].length(); /* Needs to be at least 1 longer. */ if (actual_arg0_len <= expected_arg0_len) { return false; } /* See if the preceding character is a '/'. */ if (actual[offset][actual_arg0_len - expected_arg0_len - 1] != '/') { return false; } /* See if the stuff after the '/' is the same. */ if (actual[offset].compare(actual_arg0_len - expected_arg0_len, expected_arg0_len, expected[0]) != 0) { return false; } } return argv_match_from_offset(actual, offset + 1, expected, 1); } else { /* Actual argv is shorter than expected. */ #if !FB_GLIBC_PREREQ (2, 38) /* Older libc-s does not pass "--" between "sh -c" and system() and popen() argument. */ if (expected.size() > 2 && expected[2] == "--" && actual.size() > 2 && actual[2] != "--" && expected[0] == "sh" && expected[1] == "-c" && actual[0] == "sh" && actual[1] == "-c") { return argv_match_from_offset(actual, 3, expected, 4); } #endif return false; } } std::vector>* Process::pop_expected_child_fds(const std::vector& argv, LaunchType *launch_type_p, int *type_flags_p, const bool failed) { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, "failed=%s", D(failed)); if (expected_child_) { if ((expected_child_->launch_type() == LAUNCH_TYPE_SYSTEM && expected_child_->argv().size() == 0) || (argv_matches_expectation(argv, expected_child_->argv()))) { auto fds = expected_child_->pop_fds(); if (launch_type_p) *launch_type_p = expected_child_->launch_type(); if (type_flags_p) *type_flags_p = expected_child_->type_flags(); delete(expected_child_); expected_child_ = nullptr; return fds; } else { exec_point()->disable_shortcutting_bubble_up( "Unexpected system/popen/posix_spawn child appeared", ": " + d(argv) + " " "while waiting for: " + d(expected_child_)); } delete(expected_child_); expected_child_ = nullptr; } else { exec_point()->disable_shortcutting_bubble_up("Unexpected system/popen/posix_spawn child", std::string(failed ? " failed: " : " appeared: ") + d(argv)); } return nullptr; } bool Process::finalized_or_terminated_and_has_orphan_and_finalized_children() const { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (state() == FB_PROC_FINALIZED) { return true; } else if (state() == FB_PROC_RUNNING) { return false; } assert_cmp(state(), ==, FB_PROC_TERMINATED); if (fork_point()->has_orphan_descendant()) { for (ForkedProcess* fork_child : fork_children()) { if (!fork_child->orphan() && !fork_child->finalized_or_terminated_and_has_orphan_and_finalized_children()) { return false; } } /* There can be orphan processes with terminated parents which were not orphan. * Those are not finalized until the last process in the chain terminates, but should be * treated as orphan processes.*/ return exec_child() ? exec_child()->finalized_or_terminated_and_has_orphan_and_finalized_children() : true; } else { return false; } } bool Process::any_child_not_finalized() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (exec_pending_ || has_pending_popen()) { return true; } if (exec_child() && exec_child()->state() != FB_PROC_FINALIZED) { /* The exec child is not yet finalized. We're not ready to finalize either. */ return true; } for (auto fork_child : fork_children_) { if (fork_child->state_ != FB_PROC_FINALIZED) { return true; } } return false; } bool Process::any_child_not_finalized_or_terminated_with_orphan() const { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (exec_pending_ || has_pending_popen()) { return true; } if (exec_child() && !exec_child()->finalized_or_terminated_and_has_orphan_and_finalized_children()) { return true; } for (auto fork_child : fork_children_) { if (!fork_child->finalized_or_terminated_and_has_orphan_and_finalized_children()) { return true; } } return false; } /** * Terminate orphan descendant processes which have only terminated ancestors. * * Those are likely the ones which are not terminated by their parents * thus would be left behind by the build. * * Note: This function should be called on the top (execed) process of the build to * clean up all orphans. */ void Process::terminate_top_orphans() const { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (fork_point()->orphan() && state() == FB_PROC_RUNNING) { /* If the supervisor is a subreaper and there is no subreaper among the supervised processes, * then it is safe to assume that all orphans are still running or are zombies waiting for being * reaped, thus they can be kill()-ed by pid. */ FB_DEBUG(FB_DEBUG_PROC, "Killing top orphan process " + d(this)); kill(pid(), SIGTERM); /* Continue with all fork children of this exec chain. The processes of this exec chain are * not kill()-ed again. */ const Process* curr = this; do { for (const ForkedProcess* child : curr->fork_children()) { child->terminate_top_orphans(); } curr = curr->exec_child(); } while (curr); } if (state() == FB_PROC_TERMINATED) { for (const ForkedProcess* child : fork_children()) { child->terminate_top_orphans(); } if (exec_child()) { exec_child()->terminate_top_orphans(); } } } /** * Finalize the current process. */ void Process::do_finalize() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); assert_cmp(state(), ==, FB_PROC_TERMINATED); set_state(FB_PROC_FINALIZED); } /** * Finalize the current process if possible, and if successful then * bubble it up. */ void Process::maybe_finalize() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (state() != FB_PROC_TERMINATED) { return; } if (any_child_not_finalized()) { if (!fork_point()->parent() && fork_point()->has_orphan_descendant() && finalized_or_terminated_and_has_orphan_and_finalized_children()) { /* Kill all orphan processes when the root exec process can't be finalized because of them. * They may or may not quit on their own, but it is impossible to tell. */ fork_point()->terminate_top_orphans(); /* This top exec process can now be finalized, because no ancestor of the just terminated * orphans would be cached. The supervisor will exit after the descendants of the orphans * terminate, too. Otherwise if we return from this function here the supervisor would quickly * kill all the descendants of the just killed orphans because those became orphans, too. */ } else { /* We're not ready to finalize. */ return; } } /* Only finalize the process after it is clear that if the parent has waited for it. */ // TODO(rbalint) collect/confirm exit status in the parent to make sure that the exit // status saved in the cache will be correct. if (!been_waited_for()) { const Process* fork_parent_ptr = fork_parent(); if (fork_parent_ptr) { const Process* potential_waiter = fork_parent_ptr->last_exec_descendant(); if (potential_waiter->state() != FB_PROC_RUNNING && !potential_waiter->exec_pending()) { /* The process that could wait for this one is not running anymore. */ fork_point()->set_orphan(); exec_point()->disable_shortcutting_bubble_up("Orphan processes can't be shortcut", exec_point()); fork_parent()->fork_point()->set_has_orphan_descendant_bubble_up(); /* Can proceed with finalizing this process, it won't be saved to the cache. */ } else { /* Wait for the fork parent to wait() for this child or to terminate. */ return; } } else { /* The top process will be waited for later. */ return; } } drain_all_pipes(); do_finalize(); if (parent()) { parent()->maybe_finalize(); } } void Process::finish() { TRACKX(FB_DEBUG_PROC, 1, 1, Process, this, ""); if (FB_DEBUGGING(FB_DEBUG_PROC) && expected_child_) { FB_DEBUG(FB_DEBUG_PROC, "Expected system()/popen()/posix_spawn() children that did not appear" " (e.g. posix_spawn() failed in the pre-exec or exec step):"); FB_DEBUG(FB_DEBUG_PROC, " " + d(expected_child_)); } if (!exec_pending_ && !has_pending_popen()) { /* The pending child may show up and it needs to inherit the pipes. */ reset_file_fd_pipe_refs(); } set_state(FB_PROC_TERMINATED); /* Here we check only the fork children. In theory this parent could exec() and the * execed process could wait for its children, but this is rare and costly to detect thus * we disable shortcutting in more cases than it is absolutely needed. */ if (!exec_child() && !exec_pending()) { /* This is the last process in the exec chain. Let's see if orphans were left behind. */ for (Process* curr = fork_point(); curr; curr = curr->exec_child()) { bool orphan_found = false; for (ForkedProcess* fork_child : curr->fork_children()) { if (!fork_child->been_waited_for()) { /* This may also be set in last_exec_descendant->maybe_finalize(), but not when * last_exec_descendant has not finalized children. */ fork_child->set_orphan(); orphan_found = true; /* This disables shortcutting the fork child and maybe finalizes it. Since shortcutting is * disabled up to the top process this process will not be shortcuttable either. */ fork_child->last_exec_descendant()->maybe_finalize(); } } if (orphan_found) { curr->exec_point()->disable_shortcutting_bubble_up("Orphan processes can't be shortcut", exec_point()); curr->fork_point()->set_has_orphan_descendant_bubble_up(); } } } maybe_finalize(); } /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string Process::d_internal(const int level) const { (void)level; /* unused */ return "{Process " + pid_and_exec_count() + "}"; } Process::~Process() { TRACKX(FB_DEBUG_PROC, 1, 0, Process, this, ""); delete(expected_child_); delete(fds_); } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const Process& p, const int level) { return p.d_internal(level); } std::string d(const Process *p, const int level) { if (p) { return d(*p, level); } else { return "{Process NULL}"; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/process.h000066400000000000000000000634111447164520700200700ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PROCESS_H_ #define FIREBUILD_PROCESS_H_ #include #include #include #include #include #include #include "firebuild/debug.h" #include "firebuild/file_fd.h" #include "firebuild/execed_process_env.h" #include "firebuild/cxx_lang_utils.h" #include "firebuild/pipe.h" namespace firebuild { class ExecedProcess; class ForkedProcess; typedef enum { /** * Process is running. */ FB_PROC_RUNNING, /** * Process successfully performed an exit() or alike, exec() or alike, * or crashed on signal. * * In case of exec() it lives on with the same Unix PID, but that's a * different Process in our model. Either exec_pending_ is set (the * execed process haven't appeared yet) or exec_child_ is set * (pointing to the execed process). * * In case of exit() or crash it might still be present as a Unix * zombie process, we don't care about that. Neither exec_pending_ nor * exec_child_ are set. * * When a Process in our model enters the FB_PROC_TERMINATED state it checks if * it had any forked child that it did not wait for. If there were any, * the process can't be shortcut. * * Also a forked child stays in FB_PROC_TERMINATED state until its fork parent * waits for it or terminates. * TODO(rbalint) as a consequence an orphaned child not quitting on its own * hangs firebuild thus such children are not supported yet. */ FB_PROC_TERMINATED, /** * The given process, and all its descendants have terminated. None of * the process's parameters can change anymore. Whatever the process * transitively performed is stored in the cache upon entering this * state. */ FB_PROC_FINALIZED, } process_state; /** * Firebuild's model of a UNIX (TODO: Windows) process' period of life. * Generally it represents the period starting with a successful exec() or fork() * and finished by an other successful exec() or exit(). * * Note the difference from UNIX's process concept. In Unix a process can call * exec() successfully several time preserving process id and some process * attributes, while replacing the process image. Those periods are handled as * different (but related) Processes in Firebuild. */ class Process { public: Process(int pid, int ppid, int exec_count, const FileName *wd, mode_t umask, Process* parent, std::vector>* fds, const bool debug_suppressed); virtual ~Process(); bool operator == (Process const & p) const; void set_parent(Process *p) {parent_ = p;} Process* parent() {return parent_;} const Process* parent() const {return parent_;} /** The nearest ExecedProcess upwards in the tree, including "this". * Guaranteed to be non-NULL. */ virtual ExecedProcess* exec_point() = 0; virtual const ExecedProcess* exec_point() const = 0; virtual ForkedProcess* fork_point() = 0; virtual const ForkedProcess* fork_point() const = 0; Process* fork_parent(); const Process* fork_parent() const; /** The nearest ExecedProcess upwards in the tree, excluding "this". * Same as the parent's exec_point, with safe NULL handling. */ ExecedProcess* parent_exec_point() {return parent() ? parent()->exec_point() : NULL;} const ExecedProcess* parent_exec_point() const {return parent() ? parent()->exec_point() : NULL;} virtual bool exec_started() const {return false;} /* This process has been wait()-ed for by the process that forked it. When the supervisor acts * as a subreaper it does not set the been_waited_for_ flag thus for those processes this function * never returns true. */ virtual bool been_waited_for() const = 0; virtual void set_been_waited_for() = 0; /** * The process is either finalized or is terminated, but in case it is just terminated it has * only finalized, orphan or terminated children, recursively. * In other words all the descendants that are running are orphans or descendants of those * orphans and there is no terminated process that could be finalized. */ bool finalized_or_terminated_and_has_orphan_and_finalized_children() const; /** * There is at least one child, that's not * finalized_or_terminated_and_has_orphan_and_finalized_children(). In other words there is * a running descendant that's not an orphan process or a terminated process that could be * finalized. */ bool any_child_not_finalized_or_terminated_with_orphan() const; void terminate_top_orphans() const; int state() const {return state_;} void set_state(process_state s) {state_ = s;} int fb_pid() const {return fb_pid_;} int pid() const {return pid_;} int ppid() const {return ppid_;} int exec_count() const {return exec_count_;} const FileName* wd() const {return wd_;} void handle_set_wd(const char * const d, const size_t d_len = -1); void handle_set_fwd(const int fd); mode_t umask() const {return umask_;} void set_exec_pending(bool val) {exec_pending_ = val;} bool exec_pending() const {return exec_pending_;} void set_posix_spawn_pending(bool val) {posix_spawn_pending_ = val;} bool posix_spawn_pending() {return posix_spawn_pending_;} void set_exec_child(ExecedProcess *p) {exec_child_ = p;} ExecedProcess* exec_child() const {return exec_child_;} Process* last_exec_descendant(); const Process* last_exec_descendant() const; std::vector& fork_children() {return fork_children_;} const std::vector& fork_children() const {return fork_children_;} void set_system_child(ExecedProcess *proc) {system_child_ = proc;} ExecedProcess *system_child() const {return system_child_;} void set_expected_child(ExecedProcessEnv *ec) { assert_null(expected_child_); expected_child_ = ec; } std::vector>* pop_expected_child_fds(const std::vector&, LaunchType *launch_type_p, int *type_flags_p = nullptr, const bool failed = false); bool has_expected_child () {return expected_child_ ? true : false;} void set_has_pending_popen(bool value) {has_pending_popen_ = value;} bool has_pending_popen() const {return has_pending_popen_;} bool debug_suppressed() const {return debug_suppressed_;} virtual void do_finalize(); virtual void set_on_finalized_ack(int id, int fd) = 0; virtual void maybe_finalize(); void finish(); virtual Process* exec_proc() const = 0; void update_rusage(int64_t utime_u, int64_t stime_u); virtual void resource_usage(int64_t utime_u, int64_t stime_u); FileFD* get_fd(int fd) const { assert(fds_); if (fd < 0 || static_cast(fd) >= fds_->size()) { return nullptr; } else { return (*fds_)[fd].get(); } } std::shared_ptr get_shared_fd(int fd) { assert(fds_); if (fd < 0 || static_cast(fd) >= fds_->size()) { return nullptr; } else { return (*fds_)[fd]; } } void close_fds() { for (int i = fds_->size() - 1; i >= 0; i--) { FileFD* file_fd = get_fd(i); if (file_fd) { handle_close(file_fd); } } } std::vector>* fds() {return fds_;} const std::vector>* fds() const {return fds_;} void set_fds(std::vector>* fds) {fds_ = fds;} /** Add add ffd FileFD* to open fds */ static std::shared_ptr add_filefd(std::vector>* fds, const int fd, std::shared_ptr ffd); std::shared_ptr add_filefd(const int fd, std::shared_ptr ffd); std::vector>* pass_on_fds(const bool execed = true) const; void add_pipe(std::shared_ptr pipe); /** Drain all pipes's associated with open file descriptors of the process reading as much data * as available on each fd1 end of each pipe */ // TODO(rbalint) forward only fd0 pipe ends coming from this process void drain_all_pipes(); void reset_file_fd_pipe_refs() { for (auto& file_fd : *fds_) { if (file_fd && file_fd->pipe()) { file_fd->pipe()->drain_fd1_end(file_fd.get()); file_fd->set_pipe(nullptr); } } } void AddPopenedProcess(int fd, ExecedProcess *proc); ExecedProcess *PopPopenedProcess(int fd) { assert_cmp(fd2popen_child_.count(fd), >, 0); ExecedProcess *ret = fd2popen_child_[fd]; fd2popen_child_.erase(fd); return ret; } /** * Return the resolved and canonicalized absolute pathname, based on the given dirfd directory * (possibly AT_FDCWD for the current directory). Return nullptr if the path is relative and * dirfd is invalid. */ const FileName* get_absolute(const int dirfd, const char * const name, ssize_t length) const; /** * Handle preparation for file opening in the monitored process * @param dirfd the dirfd of openat(), or AT_FDCWD * @param ar_name relative or absolute file name * @param ar_len length of ar_name */ int handle_pre_open(const int dirfd, const char * const ar_name, const size_t ar_len); /** * Handle file opening in the monitored process * @param dirfd the dirfd of openat(), or AT_FDCWD * @param ar_name relative or absolute file name * @param ar_len length of ar_name * @param flags flags of open() * @param mode mode (create permissions) of open() * @param fd the return value, or -1 if file was dlopen()ed successfully * @param error error code of open() * @param fd_conn fd to send ACK on when needed * @param ack_num ACK number to send or 0 if sending ACK is not needed * @param pre_open_sent interceptor already sent pre_open for this open * @param tmp_file the file was opened as a temporary file by mkstemp() or friends */ int handle_open(const int dirfd, const char * const ar_name, const size_t ar_len, const int flags, const mode_t mode, const int fd, const int error, int fd_conn, int ack_num, const bool pre_open_sent, bool tmp_file); /** * Handle file opening in the monitored process * @param ar_name relative or absolute file name * @param ar_len length of ar_name * @param flags flags of open() * @param oldfd the fd of the stream before the operation * @param fd the fd of the stream after the operation * @param error error code of open() * @param fd_conn fd to send ACK on when needed * @param ack_num ACK number to send or 0 if sending ACK is not needed * @param pre_open_sent interceptor already sent pre_open for this open */ int handle_freopen(const char * const ar_name, const size_t ar_len, const int flags, const int oldfd, const int fd, const int error = 0, int fd_conn = -1, int ack_num = 0, bool pre_open_sent = false); /** * Handle dlopen() in the monitored process * @param absolute_filename absolute file name * @param absolute_filename_len length of absolute_filename * @param looked_up_filename file name passed to dlopen() * @param looked_up_filename_len length of looked_up_filename * @param error true when dlopen failed to open the file * @param fd_conn fd to send ACK on when needed * @param ack_num ACK number to send or 0 if sending ACK is not needed */ int handle_dlopen(const char * const absolute_filename, const size_t absolute_filename_len, const char * const looked_up_filename, const size_t looked_up_filename_len, const bool error, int fd_conn, int ack_num); /** * Handle file closure in the monitored process * @param fd file descriptor to close * @param error error code of close() */ int handle_close(const int fd, const int error = 0); /** * Handle file closure in the monitored process * @param file_fd FileFD* to close */ void handle_close(FileFD * file_fd); /** * Handle file closure in the monitored process when we don't know * if the operation succeeded or failed. Required for glibc's way of * ignoring an error from a close() that was registered by * posix_spawn_file_actions_addclose(). Also required as an internal * helper for opening to a particular fd, as done via * posix_spawn_file_actions_addopen(). * @param fd file descriptor to close */ int handle_force_close(const int fd); /** * Handle closefrom() in the monitored process. */ int handle_closefrom(const int lowfd); /** * Handle close_range() in the monitored process.. */ int handle_close_range(const unsigned int first, const unsigned int last, const int flags, const int error = 0); /** * Handle truncate() in the monitored process.. */ int handle_truncate(const char * const ar_name, const size_t ar_len, const off_t length, const int error); /** * Handle unlink in the monitored process * @param dirfd the dirfd of unlinkat(), or AT_FDCWD * @param name relative or absolute file name * @param name_len length of name * @param flags flags passed to unlinkat() * @param error error code of unlink() * @param pre_open_sent interceptor already sent pre_open for this operation */ int handle_unlink(const int dirfd, const char * const name, const size_t name_len, const int flags, const int error, const bool pre_open_sent); /** * Handle stat in the monitored process * @param fd fstat()'s fd, or fstatat()'s dirfd, or AT_FDCWD * @param name relative or absolute file name * @param name_len length of name * @param flags flags passed to fstatat() or AT_SYMLINK_NOFOLLOW in case of lstat() * @param st_mode mode as returned by a successful stat() call * @param st_size size as returned by a successful stat() call * @param error error code of stat() variant */ int handle_fstatat(const int fd, const char * const name, const size_t name_len, const int flags, const mode_t st_mode, const off_t st_size, const int error = 0); /** * Handle statfs in the monitored process * @param name absolute file name * @param name_len length of name * @param error error code of statfs() variant */ int handle_statfs(const char * const name, const size_t name_len, const int error = 0); /** * Handle access, e[uid]access, faccessat in the monitored process */ int handle_faccessat(const int dirfd, const char * const name, const size_t name_len, const int mode, const int flags, const int error = 0); /** * Handle the chmod family in the monitored process * @param fd fchmod()'s fd, or fchmodat()'s dirfd, or AT_FDCWD * @param name relative or absolute file name * @param name_len length of name * @param mode the newly applied mode * @param flags flags passed to fchmodat() or AT_SYMLINK_NOFOLLOW in case of lchmod() * @param error error code of stat() variant */ int handle_fchmodat(const int fd, const char * const name, const size_t name_len, const mode_t mode, const int flags, const int error = 0); #ifdef __linux__ /** * Handle memfd_create in the monitored process */ int handle_memfd_create(const int flags, const int fd); /** * Handle timerfd_create in the monitored process */ int handle_timerfd_create(const int flags, int fd); /** * Handle epoll_create in the monitored process */ int handle_epoll_create(const int flags, const int fd); /** * Handle eventfd in the monitored process */ int handle_eventfd(const int flags, const int fd); /** * Handle signalfd in the monitored process * @param oldfd fd passed to signalfd() to be reused (if not -1) * @param flags flags passed to signalfd() * @param newfd new fd returned by signalfd() */ int handle_signalfd(const int oldfd, const int flags, const int newfd); #endif /** * Handle rmdir in the monitored process * @param name relative or absolute file name * @param name_len length of name * @param error error code of rmdir() * @param pre_open_sent interceptor already sent pre_open for this operation */ int handle_rmdir(const char * const name, const size_t name_len, const int error, const bool pre_open_sent); /** * Handle mkdir in the monitored process * @param dirfd the dirfd of mkdirat(), or AT_FDCWD * @param name relative or absolute file name * @param name_len length of name * @param error error code of mkdir() * @param tmp_dir the call was mkdtemp() actually */ int handle_mkdir(const int dirfd, const char * const name, const size_t name_len, const int error, const bool tmp_dir); /** * Handle pipe creation in the monitored process, steps 1 and 2 out of 3. See #656 for the design. * @param flags The flags passed to pipe2(), or 0 if pipe() was called * @param fd_conn fd to send the pipe_fds message to */ void handle_pipe_request(const int flags, const int fd_conn); /** * Handle pipe creation in the monitored process, step 3 out of 3. See #656 for the design. * @param fd0 fd number of the reading end in the monitored process * @param fd1 fd number of the writing end in the monitored process */ void handle_pipe_fds(const int fd0, const int fd1); /** * Handle socket() in the monitored process */ void handle_socket(const int domain, const int type, const int protocol, const int ret, const int error); /** * Handle socketpair() in the monitored process */ void handle_socketpair(const int domain, const int type, const int protocol, const int fd0, const int fd1, const int error); /** * Handle dup(), dup2() or dup3() in the monitored process * @param oldfd old fd * @param newfd new fd * @param flags extra flags for new fd passed to dup3() * @param error error code * @return 0 on success, -1 on failure */ int handle_dup3(const int oldfd, const int newfd, const int flags, const int error = 0); /** * Handle rename() * @param olddirfd the olddirfd of renameat(), or AT_FDCWD * @param old_ar_name old relative or absolute file name * @param old_ar_len length of old_ar_name * @param newdirfd the newdirfd of renameat(), or AT_FDCWD * @param new_ar_name new relative or absolute file name * @param new_ar_len length of old_ar_name * @param error error code * @return 0 on success, -1 on failure */ int handle_rename(const int olddirfd, const char * const old_ar_name, const size_t old_ar_len, const int newdirfd, const char * const new_ar_name, const size_t new_ar_len, const int error = 0); /** * Handle symlink() * @param target relative or absolute target file name * @param newdirfd the newdirfd of symlinkat(), or AT_FDCWD * @param new_ar_name new relative or absolute file name * @param error error code * @return 0 on success, -1 on failure */ int handle_symlink(const char * const target, const int newdirfd, const char * const new_ar_name, const int error = 0); /** * Handle successfully clearing the cloexec bit, via a * posix_spawn_file_actions_adddup2() handler with oldfd==newfd. * @param fd file descriptor * @return 0 on success, -1 on failure */ int handle_clear_cloexec(const int fd); /** * Handle fcntl() in the monitored process * @param fd file descriptor * @param cmd fcntl's cmd parameter * @param arg fcntl's arg parameter * @param ret fcntl's return value * @param error errno set by fcntl * @return 0 on success, -1 on failure */ int handle_fcntl(const int fd, const int cmd, const int arg, const int ret, const int error = 0); /** * Handle ioctl() in the monitored process * @param fd file descriptor * @param cmd ioctl's cmd parameter * @param ret ioctl's return value * @param error errno set by ioctl * @return 0 on success, -1 on failure */ int handle_ioctl(const int fd, const int cmd, const int ret, const int error = 0); /** * Handle the first read()/recv()-like or the first pread()-like operation from an inherited fd in * the monitored process * @param fd file descriptor * @param is_pread whether the read occurred at a specified position */ void handle_read_from_inherited(const int fd, const bool is_pread); /** * Handle the first write()/send()-like or the first pwrite()-like operation to an inherited fd in * the monitored process * @param fd file descriptor * @param is_pwrite whether the write occurred at a specific position */ void handle_write_to_inherited(const int fd, const bool is_pwrite); /** * Handle first seek()-like operation querying the offset, or the first such operation modifying * the offset of an inherited fd in the monitored process * @param fd file descriptor * @param modify_offset whether the operation requested to change the offset */ void handle_seek_in_inherited(const int fd, const bool modify_offset); /** * Handle interceptorting the start offset of an inherited file that's not seeked to the end. * @param fd file descriptor * @param offset the offset in the file */ void handle_inherited_fd_offset(const int fd, const int64_t offset); /** * Handle the intercepted process receiving some file descriptors via recv[m]msg() and SCM_RIGHTS * @param cloexec the close on exec flag * @param fds file descriptors */ void handle_recvmsg_scm_rights(const bool cloexec, const std::vector fds); /** * Fail to change to a working directory */ virtual void handle_fail_wd(const char * const d) = 0; /** * Record visited working directory */ virtual void add_wd(const FileName *d) = 0; /** * Handle umask() in the monitored process */ void handle_umask(mode_t old_umask, mode_t new_umask); /* For debugging. */ std::string pid_and_exec_count() const {return d(pid()) + "." + d(exec_count());} /* For debugging. */ std::string state_string() const { switch (state_) { case FB_PROC_RUNNING: return "running"; case FB_PROC_TERMINATED: return "terminated"; case FB_PROC_FINALIZED: return "finalized"; default: assert(0 && "unknown state"); return "UNKNOWN"; } } /* Member debugging method. Not to be called directly, call the global d(obj_or_ptr) instead. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ virtual std::string d_internal(const int level = 0) const; private: Process *parent_; process_state state_ :2; int fb_pid_; ///< internal Firebuild id for the process int pid_; ///< UNIX pid int ppid_; ///< UNIX ppid /** For debugging: The "age" of a given PID, i.e. how many execve() hops happened to it. * 0 for a ForkedProcess, 1 for its first ExecedProcess child, 2 for the execed child of * that one, etc. -1 temporarily while constructing a Process object. */ int exec_count_; const FileName* wd_; ///< Current working directory mode_t umask_; ///< Current umask std::vector>* fds_; ///< Active file descriptors std::vector fork_children_; ///< children of the process /** the latest system() child */ ExecedProcess *system_child_ {NULL}; /** Same as `proc_tree->Proc2PendingPopen(this) != nullptr`, redundantly repeated here * for better performance. */ bool has_pending_popen_ {false}; /** for popen()ed children: client fd -> process mapping */ tsl::hopscotch_map fd2popen_child_ {}; /** commands of system(3), popen(3) and posix_spawn[p](3) that are expected to appear */ ExecedProcessEnv *expected_child_; /** Set upon an "exec" message, cleared upon "scproc_query" (i.e. new dynamically linked process * successfully started up) or "exec_failed". Also set in the child upon a "posix_spawn_parent" * in the typical case that the child hasn't appeared yet. Lets us detect statically linked * processes: if a process quits while this flag is true then it was most likely statically * linked (or failed to link, etc.). */ bool exec_pending_ {false}; /** Set upon "posix_spawn", cleared upon "posix_spawn_parent" or "posix_spawn_failed". Lets us * detect the non-typical case when the posix_spawn'ed process appears (does an "scproc_query") * sooner than the parent gets to "posix_spawn_parent". */ bool posix_spawn_pending_ {false}; /** Debugging is suppressed for this process. */ bool debug_suppressed_; ExecedProcess * exec_child_; const FileName* get_fd_filename(int fd) const; bool any_child_not_finalized(); DISALLOW_COPY_AND_ASSIGN(Process); }; inline bool Process::operator == (Process const & p) const { return (p.fb_pid_ == fb_pid_); } /* Global debugging methods. * level is the nesting level of objects calling each other's d(), bigger means less info to print. * See #431 for design and rationale. */ std::string d(const Process& p, const int level = 0); std::string d(const Process *p, const int level = 0); } /* namespace firebuild */ #endif // FIREBUILD_PROCESS_H_ firebuild-0.8.2/src/firebuild/process_debug_suppressor.h000066400000000000000000000031251447164520700235370ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PROCESS_DEBUG_SUPPRESSOR_H_ #define FIREBUILD_PROCESS_DEBUG_SUPPRESSOR_H_ #include "firebuild/debug.h" #include "firebuild/process.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { class ProcessDebugSuppressor { public: explicit ProcessDebugSuppressor(const Process* const proc) : debug_suppressed_changed_(proc), debug_suppressed_orig_(debug_suppressed) { if (proc && firebuild::debug_filter) { debug_suppressed = proc->debug_suppressed(); } } ~ProcessDebugSuppressor() { if (debug_suppressed_changed_) { debug_suppressed = debug_suppressed_orig_; } } private: bool debug_suppressed_changed_; bool debug_suppressed_orig_; }; } /* namespace firebuild */ #endif // FIREBUILD_PROCESS_DEBUG_SUPPRESSOR_H_ firebuild-0.8.2/src/firebuild/process_factory.cc000066400000000000000000000113741447164520700217560ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/process_factory.h" #include #include #include "firebuild/exe_matcher.h" #include "firebuild/file_name.h" namespace firebuild { ForkedProcess* ProcessFactory::getForkedProcess(const int pid, Process * const parent) { TRACK(FB_DEBUG_PROC, "pid=%d, parent=%s", pid, D(parent)); return new ForkedProcess(pid, parent->pid(), parent, parent->pass_on_fds(false)); } ExecedProcess* ProcessFactory::getExecedProcess(const FBBCOMM_Serialized_scproc_query *const msg, Process * const parent, std::vector>* fds) { TRACK(FB_DEBUG_PROC, "parent=%s", D(parent)); const FileName* executable = FileName::Get(msg->get_executable()); const FileName* executed_path = msg->has_executed_path() ? FileName::Get(msg->get_executed_path()) : executable; char* original_executed_path = msg->has_original_executed_path() ? strndup(msg->get_original_executed_path(), msg->get_original_executed_path_len()) : const_cast(executed_path->c_str()); const size_t libs_count = msg->get_libs_count(); std::vector libs(libs_count); for (size_t i = 0; i < libs_count; i++) { libs[i] = (FileName::Get(msg->get_libs_at(i), msg->get_libs_len_at(i))); } auto e = new ExecedProcess(msg->get_pid(), msg->get_ppid(), FileName::Get(msg->get_cwd()), executable, executed_path, original_executed_path, msg->get_arg_as_vector(), msg->get_env_var_as_vector(), std::move(libs), msg->get_umask(), parent, /* When processing this msg the suppression is already set globally, * or for this thread. */ debug_suppressed, fds); /* Debug the full command line, env vars etc. */ FB_DEBUG(FB_DEBUG_PROC, "Created ExecedProcess " + d(e, 1) + " with:"); FB_DEBUG(FB_DEBUG_PROC, "- exe = " + d(e->executable())); FB_DEBUG(FB_DEBUG_PROC, "- arg = " + d(e->args())); FB_DEBUG(FB_DEBUG_PROC, "- cwd = " + d(e->initial_wd())); FB_DEBUG(FB_DEBUG_PROC, "- env = " + d(e->env_vars())); FB_DEBUG(FB_DEBUG_PROC, "- lib = " + d(msg->get_libs_as_vector())); FB_DEBUG(FB_DEBUG_PROC, "- umask = " + d(e->umask())); if (fbbcomm_serialized_scproc_query_get_jobserver_fds_count(msg) == 2) { const int* jobserver_fds = fbbcomm_serialized_scproc_query_get_jobserver_fds(msg); e->maybe_set_jobserver_fds(jobserver_fds[0], jobserver_fds[1]); FB_DEBUG(FB_DEBUG_PROC, "- jobserver_fd_r = " + d(e->jobserver_fd_r())); FB_DEBUG(FB_DEBUG_PROC, "- jobserver_fd_w = " + d(e->jobserver_fd_w())); } return e; } bool ProcessFactory::peekProcessDebuggingSuppressed(const FBBCOMM_Serialized *fbbcomm_buf) { if (!debug_filter) { return false; } const int tag = fbbcomm_buf->get_tag(); if (tag == FBBCOMM_TAG_scproc_query) { const FBBCOMM_Serialized_scproc_query *msg = reinterpret_cast(fbbcomm_buf); const FileName* executable = FileName::Get(msg->get_executable()); const FileName* executed_path = msg->has_executed_path() ? FileName::Get(msg->get_executed_path()) : executable; const auto args = msg->get_arg_as_vector(); return !debug_filter->match(executable, executed_path, args.size() > 0 ? args[0] : ""); } else if (tag == FBBCOMM_TAG_fork_child) { const FBBCOMM_Serialized_fork_child *msg = reinterpret_cast(fbbcomm_buf); const auto ppid = msg->get_ppid(); auto pproc = proc_tree->pid2proc(ppid); assert(pproc); return !debug_filter->match(pproc->exec_point()); } else { assert(0 && "unexpected tag"); return false; } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/process_factory.h000066400000000000000000000037351447164520700216220ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PROCESS_FACTORY_H_ #define FIREBUILD_PROCESS_FACTORY_H_ #include #include #include "./fbbcomm.h" #include "firebuild/execed_process.h" #include "firebuild/forked_process.h" #include "firebuild/process_tree.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { /** * Converts FBB messages from monitored processes to new Process * instances. It is an implementation of the GoF Factory pattern. * The class itself is never instantiated, but groups a set of * static functions which accept a ProcessTree reference and an incoming FBB * message to the process from. */ class ProcessFactory { public: static ForkedProcess* getForkedProcess(int pid, Process * const parent); static ExecedProcess* getExecedProcess(const FBBCOMM_Serialized_scproc_query *const msg, Process * const parent, std::vector>* fds); static bool peekProcessDebuggingSuppressed(const FBBCOMM_Serialized *fbbcomm_buf); private: DISALLOW_COPY_AND_ASSIGN(ProcessFactory); }; } /* namespace firebuild */ #endif // FIREBUILD_PROCESS_FACTORY_H_ firebuild-0.8.2/src/firebuild/process_fbb_adaptor.h000066400000000000000000000311721447164520700224120ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PROCESS_FBB_ADAPTOR_H_ #define FIREBUILD_PROCESS_FBB_ADAPTOR_H_ #include #include #include "./fbbcomm.h" #include "firebuild/process.h" #include "firebuild/cxx_lang_utils.h" namespace firebuild { /** * Converts messages from monitored processes to calls to Process instances. * It is not a clean implementation of the GoF Adaptor pattern, but something * like that. The class itself is never instantiated, but groups a set of * static functions which accept a Process reference and an incoming FBB * message for the process. */ class ProcessFBBAdaptor { public: static int handle(Process *proc, const FBBCOMM_Serialized_pre_open *msg) { return proc->handle_pre_open(msg->get_dirfd_with_fallback(AT_FDCWD), msg->get_pathname(), msg->get_pathname_len()); } static int handle(Process *proc, const FBBCOMM_Serialized_open *msg, int fd_conn, int ack_num) { const int dirfd = msg->get_dirfd_with_fallback(AT_FDCWD); int error = msg->get_error_no_with_fallback(0); int ret = msg->get_ret_with_fallback(-1); return proc->handle_open(dirfd, msg->get_pathname(), msg->get_pathname_len(), msg->get_flags(), msg->get_mode_with_fallback(0), ret, error, fd_conn, ack_num, msg->get_pre_open_sent(), msg->get_tmp_file_with_fallback(false)); } static int handle(Process *proc, const FBBCOMM_Serialized_freopen *msg, int fd_conn, int ack_num) { int error = msg->get_error_no_with_fallback(0); int oldfd = msg->get_ret_with_fallback(-1); int ret = msg->get_ret_with_fallback(-1); return proc->handle_freopen(msg->get_pathname(), msg->get_pathname_len(), msg->get_flags(), oldfd, ret, error, fd_conn, ack_num, msg->get_pre_open_sent()); } static int handle(Process *proc, const FBBCOMM_Serialized_dlopen *msg, int fd_conn, int ack_num) { return proc->handle_dlopen( msg->has_absolute_filename() ? msg->get_absolute_filename() : nullptr, msg->has_absolute_filename() ? msg->get_absolute_filename_len() : 0, msg->has_filename() ? msg->get_filename() : nullptr, msg->has_filename() ? msg->get_filename_len() : 0, msg->get_error(), fd_conn, ack_num); } static int handle(Process *proc, const FBBCOMM_Serialized_close *msg) { const int error = msg->get_error_no_with_fallback(0); return proc->handle_close(msg->get_fd(), error); } static int handle(Process *proc, const FBBCOMM_Serialized_closefrom *msg) { return proc->handle_closefrom(msg->get_lowfd()); } static int handle(Process *proc, const FBBCOMM_Serialized_close_range *msg) { const int error = msg->get_error_no_with_fallback(0); return proc->handle_close_range(msg->get_first(), msg->get_last(), msg->get_flags(), error); } static int handle(Process *proc, const FBBCOMM_Serialized_truncate *msg) { const int error = msg->get_error_no_with_fallback(0); return proc->handle_truncate(msg->get_pathname(), msg->get_pathname_len(), msg->get_length(), error); } static int handle(Process *proc, const FBBCOMM_Serialized_unlink *msg) { const int dirfd = msg->get_dirfd_with_fallback(AT_FDCWD); const int flags = msg->get_flags_with_fallback(0); const int error = msg->get_error_no_with_fallback(0); return proc->handle_unlink(dirfd, msg->get_pathname(), msg->get_pathname_len(), flags, error, msg->get_pre_open_sent()); } static int handle(Process *proc, const FBBCOMM_Serialized_rmdir *msg) { const int error = msg->get_error_no_with_fallback(0); return proc->handle_rmdir(msg->get_pathname(), msg->get_pathname_len(), error, msg->get_pre_open_sent()); } static int handle(Process *proc, const FBBCOMM_Serialized_mkdir *msg) { const int dirfd = msg->get_dirfd_with_fallback(AT_FDCWD); const int error = msg->get_error_no_with_fallback(0); return proc->handle_mkdir(dirfd, msg->get_pathname(), msg->get_pathname_len(), error, msg->get_tmp_dir_with_fallback(false)); } static int handle(Process *proc, const FBBCOMM_Serialized_fstatat *msg) { const int fd = msg->get_fd_with_fallback(AT_FDCWD); const mode_t st_mode = msg->get_st_mode_with_fallback(0); const off_t st_size = msg->get_st_size_with_fallback(0); const int flags = msg->get_flags_with_fallback(0); const int error = msg->get_error_no_with_fallback(0); return proc->handle_fstatat(fd, msg->get_pathname(), msg->get_pathname_len(), flags, st_mode, st_size, error); } static int handle(Process *proc, const FBBCOMM_Serialized_faccessat *msg) { const int dirfd = msg->get_dirfd_with_fallback(AT_FDCWD); const int mode = msg->get_mode(); const int flags = msg->get_flags_with_fallback(0); const int error = msg->get_error_no_with_fallback(0); return proc->handle_faccessat(dirfd, msg->get_pathname(), msg->get_pathname_len(), mode, flags, error); } static int handle(Process *proc, const FBBCOMM_Serialized_fchmodat *msg) { const int fd = msg->get_fd_with_fallback(AT_FDCWD); const mode_t mode = msg->get_mode(); const int flags = msg->get_flags_with_fallback(0); const int error = msg->get_error_no_with_fallback(0); return proc->handle_fchmodat(fd, msg->get_pathname(), msg->get_pathname_len(), mode, flags, error); } #ifdef __linux__ static int handle(Process *proc, const FBBCOMM_Serialized_memfd_create *msg) { return proc->handle_memfd_create(msg->get_flags(), msg->get_ret()); } static int handle(Process *proc, const FBBCOMM_Serialized_timerfd_create *msg) { return proc->handle_timerfd_create(msg->get_flags(), msg->get_ret()); } static int handle(Process *proc, const FBBCOMM_Serialized_epoll_create *msg) { return proc->handle_epoll_create(msg->get_flags_with_fallback(0), msg->get_ret()); } static int handle(Process *proc, const FBBCOMM_Serialized_eventfd *msg) { return proc->handle_eventfd(msg->get_flags(), msg->get_ret()); } static int handle(Process *proc, const FBBCOMM_Serialized_signalfd *msg) { return proc->handle_signalfd(msg->get_fd(), msg->get_flags(), msg->get_ret()); } #endif static int handle(Process *proc, const FBBCOMM_Serialized_dup *msg) { const int error = msg->get_error_no_with_fallback(0); return proc->handle_dup3(msg->get_oldfd(), msg->get_ret(), 0, error); } static int handle(Process *proc, const FBBCOMM_Serialized_dup3 *msg) { const int error = msg->get_error_no_with_fallback(0); const int flags = msg->get_flags_with_fallback(0); return proc->handle_dup3(msg->get_oldfd(), msg->get_newfd(), flags, error); } static int handle(Process *proc, const FBBCOMM_Serialized_rename *msg) { const int olddirfd = msg->get_olddirfd_with_fallback(AT_FDCWD); const int newdirfd = msg->get_newdirfd_with_fallback(AT_FDCWD); const int error = msg->get_error_no_with_fallback(0); return proc->handle_rename(olddirfd, msg->get_oldpath(), msg->get_oldpath_len(), newdirfd, msg->get_newpath(), msg->get_newpath_len(), error); } static int handle(Process *proc, const FBBCOMM_Serialized_symlink *msg) { const int newdirfd = msg->get_newdirfd_with_fallback(AT_FDCWD); const int error = msg->get_error_no_with_fallback(0); return proc->handle_symlink(msg->get_target(), newdirfd, msg->get_newpath(), error); } static int handle(Process *proc, const FBBCOMM_Serialized_fcntl *msg) { const int error = msg->get_error_no_with_fallback(0); int arg = msg->get_arg_with_fallback(0); int ret = msg->get_ret_with_fallback(-1); return proc->handle_fcntl(msg->get_fd(), msg->get_cmd(), arg, ret, error); } static int handle(Process *proc, const FBBCOMM_Serialized_ioctl *msg) { const int error = msg->get_error_no_with_fallback(0); int ret = msg->get_ret_with_fallback(-1); return proc->handle_ioctl(msg->get_fd(), msg->get_cmd(), ret, error); } static int handle(Process *proc, const FBBCOMM_Serialized_read_from_inherited *msg) { proc->handle_read_from_inherited(msg->get_fd(), msg->get_is_pread()); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_write_to_inherited *msg) { proc->handle_write_to_inherited(msg->get_fd(), msg->get_is_pwrite()); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_seek_in_inherited *msg) { proc->handle_seek_in_inherited(msg->get_fd(), msg->get_modify_offset()); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_inherited_fd_offset *msg) { proc->handle_inherited_fd_offset(msg->get_fd(), msg->get_offset()); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_recvmsg_scm_rights *msg) { bool cloexec = msg->get_cloexec(); std::vector fds = msg->get_fds_as_vector(); proc->handle_recvmsg_scm_rights(cloexec, fds); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_umask *msg) { const mode_t old_umask = msg->get_ret(); const mode_t new_umask = msg->get_mask(); proc->handle_umask(old_umask, new_umask); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_chdir *msg) { const int error = msg->get_error_no_with_fallback(0); if (error == 0) { proc->handle_set_wd(msg->get_pathname(), msg->get_pathname_len()); } else { proc->handle_fail_wd(msg->get_pathname()); } return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_fchdir *msg) { const int error = msg->get_error_no_with_fallback(0); if (error == 0) { proc->handle_set_fwd(msg->get_fd()); } return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_pipe_request *msg, int fd_conn) { const int flags = msg->get_flags_with_fallback(0); proc->handle_pipe_request(flags, fd_conn); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_pipe_fds *msg) { const int fd0 = msg->get_fd0(); const int fd1 = msg->get_fd1(); proc->handle_pipe_fds(fd0, fd1); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_socket *msg) { proc->handle_socket(msg->get_domain(), msg->get_type(), msg->get_protocol(), msg->get_ret_with_fallback(-1), msg->get_error_no_with_fallback(0)); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_socketpair *msg) { proc->handle_socketpair(msg->get_domain(), msg->get_type(), msg->get_protocol(), msg->get_fd0_with_fallback(-1), msg->get_fd1_with_fallback(-1), msg->get_error_no_with_fallback(0)); return 0; } static int handle(Process *proc, const FBBCOMM_Serialized_statfs *msg) { if (msg->has_pathname()) { return proc->handle_statfs(msg->get_pathname(), msg->get_pathname_len(), msg->get_error_no_with_fallback(0)); } else { return proc->handle_statfs(nullptr, 0, msg->get_error_no_with_fallback(0)); } } private: DISALLOW_COPY_AND_ASSIGN(ProcessFBBAdaptor); }; #define PFBBA_HANDLE(process, tag, buffer) \ ProcessFBBAdaptor::handle( \ process, \ reinterpret_cast(buffer)) #define PFBBA_HANDLE_ACKED(process, tag, buffer, fd_conn, ack_num) \ ProcessFBBAdaptor::handle( \ process, \ reinterpret_cast(buffer), \ fd_conn, ack_num) } /* namespace firebuild */ #endif // FIREBUILD_PROCESS_FBB_ADAPTOR_H_ firebuild-0.8.2/src/firebuild/process_tree.cc000066400000000000000000000125551447164520700212500ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/process_tree.h" #include #include #include "common/platform.h" #include "firebuild/debug.h" extern bool generate_report; namespace firebuild { /* singleton */ ProcessTree *proc_tree; ProcessTree::~ProcessTree() { TRACK(FB_DEBUG_PROCTREE, ""); /* clean up all processes, from the leaves towards the root */ delete_process_subtree(root()); /* clean up pending exec() children */ for (auto& pair : pid2exec_child_sock_) { delete(pair.second.incomplete_child); } /* clean up pending posix_spawn() children */ for (auto& pair : pid2posix_spawn_child_sock_) { delete(pair.second.incomplete_child); } } void ProcessTree::delete_process_subtree(Process *p) { if (!p) { return; } delete_process_subtree(p->exec_child()); for (ForkedProcess *fork_child : p->fork_children()) { delete_process_subtree(fork_child); } delete p; } void ProcessTree::insert_process(Process *p) { TRACK(FB_DEBUG_PROCTREE, "p=%s", D(p)); fb_pid2proc_[p->fb_pid()] = p; pid2proc_[p->pid()] = p; } void ProcessTree::insert(Process *p) { TRACK(FB_DEBUG_PROCTREE, "p=%s", D(p)); assert(p->fork_point()); assert(p->exec_point() || p == root_); if (p->parent() == NULL && p != root_) { /* root's parent is firebuild which is not in the tree. * If any other parent is missing, Firebuild missed process * that can happen due to the missing process(es) being statically built */ fb_error("TODO(rbalint) handle: Process without known exec parent\n"); } insert_process(p); } void ProcessTree::insert_root(pid_t root_pid, int stdin_fd, int stdout_fd, int stderr_fd) { TRACK(FB_DEBUG_PROCTREE, "root_pid=%d", root_pid); root_ = new firebuild::ForkedProcess(root_pid, getpid(), nullptr, new std::vector>()); root_->set_state(firebuild::FB_PROC_TERMINATED); // TODO(rbalint) support other inherited fds /* Create the FileFD representing stdin of the top process. Pipe will be NULL, that's fine. */ root_->add_filefd(stdin_fd, std::make_shared(stdin_fd, O_RDONLY, nullptr, nullptr)); /* Create the Pipes and FileFDs representing stdout and stderr of the top process. */ // FIXME Make this more generic, for all the received pipes / terminal outputs. bool stdout_stderr_match = fdcmp(stdout_fd, stderr_fd) == 0; FB_DEBUG(FB_DEBUG_PROCTREE, stdout_stderr_match ? "Top level stdout and stderr are the same" : "Top level stdout and stderr are distinct"); std::shared_ptr pipe; for (auto fd : {stdout_fd, stderr_fd}) { if (fd == stderr_fd && stdout_stderr_match) { /* stdout and stderr point to the same location (changing one's flags did change the * other's). Reuse the Pipe object that we created in the loop's first iteration. */ root_->add_filefd(fd, std::make_shared(fd, (*root_->fds())[stdout_fd], false)); } else { /* Create a new Pipe for this file descriptor. * The fd keeps blocking/non-blocking behaviour, it seems to be ok with epoll. * The fd is dup()-ed first to let it be closed without closing the original fd. */ int fd_dup = fcntl(fd, F_DUPFD_CLOEXEC, stderr_fd + 1); assert(fd_dup != -1); #ifdef __clang_analyzer__ /* Scan-build reports a false leak for the correct code. This is used only in static * analysis. It is broken because all shared pointers to the Pipe must be copies of * the shared self pointer stored in it. */ pipe = std::make_shared(fd_dup, nullptr); #else pipe = (new Pipe(fd_dup, nullptr))->shared_ptr(); #endif FB_DEBUG(FB_DEBUG_PIPE, "created pipe with fd0: " + d(fd) + ", dup()-ed as: " + d(fd_dup)); /* Top level inherited fds are special, they should not be closed. */ inherited_fd_pipes_.insert(pipe); root_->add_filefd(fd, std::make_shared(fd, O_WRONLY, pipe, nullptr)); } } insert_process(root_); } void ProcessTree::GcProcesses() { bool have_orphan = root()->has_orphan_descendant(); while (!proc_gc_queue_.empty()) { ExecedProcess* proc = proc_gc_queue_.front(); if (!generate_report) { // TODO(rbalint) delete subtrees not affected by orphans if (proc->parent() && !have_orphan) { proc->parent()->set_exec_child(nullptr); delete_process_subtree(proc); } else { proc->file_usages().clear(); proc->args().clear(); proc->env_vars().clear(); proc->libs().clear(); } } else { proc->inherited_files().clear(); } proc_gc_queue_.pop(); } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/process_tree.h000066400000000000000000000242431447164520700211070ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_PROCESS_TREE_H_ #define FIREBUILD_PROCESS_TREE_H_ #include #include #include #include #include #include #include #include #include #include #include #include "firebuild/debug.h" #include "firebuild/process.h" #include "firebuild/execed_process.h" #include "firebuild/forked_process.h" #include "firebuild/cxx_lang_utils.h" #include "firebuild/utils.h" namespace firebuild { struct subcmd_prof { int64_t sum_aggr_time_u = 0; int64_t count = 0; bool recursed = false; }; struct cmd_prof { int64_t aggr_time_u = 0; int64_t cmd_time_u = 0; /** {time_u, count} */ tsl::hopscotch_map subcmds = {}; }; /** Connection of a waiting fork() child process*/ struct fork_child_sock { /** Connection fork child is waiting on */ int sock; /** PID of fork child */ int child_pid; /** ACK number the process is waiting for */ int ack_num; /** Location to save child's pointer to after it is created */ Process** fork_child_ref; }; /** Connection of a waiting exec() child process*/ struct exec_child_sock { /** Connection exec() child is waiting on */ int sock; /** Child data without fds filled */ ExecedProcess* incomplete_child; }; /** ACK a parent process is waiting for when the child appears */ struct pending_parent_ack { /** ACK number the parent is waiting for */ int ack_num; /** Connection system/popen/posix_spawn parent is waiting on */ int sock; }; /** Details about a pending pipe() operation. */ struct pending_pipe_t { /** The flags parameter of pipe2() */ int flags; /** The supervisor-side end of fd0, i.e. where the intercepted process reads from * and the supervisor writes to */ int fd0; /* The supervisor-side end of fd1, i.e. where the intercepted process writes to * and the supervisor reads from */ int fd1; }; /** Details about a pending popen() operation. */ struct pending_popen_t { /** popen()'s "type", converted to O_* flags. [Set at the opening "popen" message.] */ int type_flags {0}; /** The child "sh -c". [Set at the "scproc_query" message.] */ ExecedProcess *child {nullptr}; /** Connection fd of the child. [Set at the "scproc_query" message.] */ int child_conn {-1}; /** Connection fd of the parent that's performing the popen(). [Set at the "popen_parent" * message.] */ int parent_conn {-1}; /** ACK ID to send to the parent. [Set at the "popen_parent" message.] */ uint16_t ack_num {0}; /** The client fd of the pipe in the parent. [Set at the "popen_parent" message.] */ int fd {-1}; }; class ProcessTree { public: ProcessTree() : inherited_fd_pipes_(), fb_pid2proc_(), pid2proc_(), ppid2fork_child_sock_(), pid2exec_child_sock_(), pid2posix_spawn_child_sock_(), top_dir_(FileName::Get(std::filesystem::current_path())) { } ~ProcessTree(); void insert(Process *p); void insert_root(pid_t root_pid, int stdin_fd, int stdout_fd, int stderr_fd); ForkedProcess* root() {return root_;} const FileName* top_dir() const {return top_dir_;} Process* pid2proc(int pid) { auto it = pid2proc_.find(pid); if (it != pid2proc_.end()) { return it->second; } else { return NULL; } } int64_t shortcut_cpu_time_ms() { return (root_ && root_->exec_child()) ? root_->exec_child()->shortcut_cpu_time_ms() : 0; } void QueueForkChild(int pid, int sock, int ppid, int ack_num, Process **fork_child_ref) { assert(!Pid2ForkChildSock(ppid)); ppid2fork_child_sock_[ppid] = {sock, pid, ack_num, fork_child_ref}; } void QueueExecChild(int pid, int sock, ExecedProcess* incomplete_child) { pid2exec_child_sock_[pid] = {sock, incomplete_child}; } void QueuePosixSpawnChild(int pid, int sock, ExecedProcess* incomplete_child) { pid2posix_spawn_child_sock_[pid] = {sock, incomplete_child}; } void QueueParentAck(int ppid, int ack, int sock) { assert(!PPid2ParentAck(ppid)); ppid2pending_parent_ack_[ppid] = {ack, sock}; } void QueuePendingPipe(Process *proc, pending_pipe_t pending_pipe) { assert(!Proc2PendingPipe(proc)); proc2pending_pipe_[proc] = pending_pipe; } void QueuePendingPopen(Process *proc, pending_popen_t pending_popen) { assert(!Proc2PendingPopen(proc)); proc2pending_popen_[proc] = pending_popen; } void QueueExecProcForGC(ExecedProcess* proc) { proc_gc_queue_.push(proc); } const fork_child_sock* Pid2ForkChildSock(const int pid) { auto it = ppid2fork_child_sock_.find(pid); if (it != ppid2fork_child_sock_.end()) { return &it->second; } else { return nullptr; } } const exec_child_sock* Pid2ExecChildSock(const int pid) { auto it = pid2exec_child_sock_.find(pid); if (it != pid2exec_child_sock_.end()) { return &it->second; } else { return nullptr; } } const exec_child_sock* Pid2PosixSpawnChildSock(const int pid) { auto it = pid2posix_spawn_child_sock_.find(pid); if (it != pid2posix_spawn_child_sock_.end()) { return &it->second; } else { return nullptr; } } const pending_parent_ack* PPid2ParentAck(const int ppid) { auto it = ppid2pending_parent_ack_.find(ppid); if (it != ppid2pending_parent_ack_.end()) { return &it->second; } else { return nullptr; } } pending_pipe_t* Proc2PendingPipe(Process *proc) { auto it = proc2pending_pipe_.find(proc); if (it != proc2pending_pipe_.end()) { return &it.value(); /* tsl::hopscotch_map'ism */ } else { return nullptr; } } pending_popen_t* Proc2PendingPopen(Process *proc) { auto it = proc2pending_popen_.find(proc); if (it != proc2pending_popen_.end()) { return &it.value(); /* tsl::hopscotch_map'ism */ } else { return nullptr; } } void DropQueuedForkChild(const int pid) { ppid2fork_child_sock_.erase(pid); } void DropQueuedExecChild(const int pid) { pid2exec_child_sock_.erase(pid); } void DropQueuedPosixSpawnChild(const int pid) { pid2posix_spawn_child_sock_.erase(pid); } void DropParentAck(const int ppid) { ppid2pending_parent_ack_.erase(ppid); } void DropPendingPipe(Process *proc) { proc2pending_pipe_.erase(proc); } void DropPendingPopen(Process *proc) { proc2pending_popen_.erase(proc); } void GcProcesses(); void AckParent(const int ppid) { const pending_parent_ack *ack = PPid2ParentAck(ppid); if (ack) { ack_msg(ack->sock, ack->ack_num); DropParentAck(ppid); } } void FinishInheritedFdPipes() { for (auto& pipe : inherited_fd_pipes_) { pipe->finish(); } /* Destruct these Pipe objects, and in turn their recorders, by dropping the last reference. */ inherited_fd_pipes_.clear(); } private: ForkedProcess *root_ = NULL; /** The pipes (or terminal lines) inherited from the external world, * each represented by a Pipe object created by this ProcessTree. */ tsl::hopscotch_set> inherited_fd_pipes_; tsl::hopscotch_map fb_pid2proc_; tsl::hopscotch_map pid2proc_; tsl::hopscotch_map ppid2fork_child_sock_; /** Whenever an exec*() child appears, but we haven't yet fully processed its exec parent, * we need to put aside the new process until we finish processing its ancestor. */ tsl::hopscotch_map pid2exec_child_sock_; /** Whenever a posix_spawn*() child process appears, but we haven't yet processed the * posix_spawn_parent message from the parent, we have to put aside the new process until * we get to this point in the parent. The key is the parent's pid. */ tsl::hopscotch_map pid2posix_spawn_child_sock_; tsl::hopscotch_map ppid2pending_parent_ack_ = {}; /** A process can only have one pending pipe() or pipe2() operation * because the interceptor holds the global mutex for its duration. * Store this rarely used data here to decrease the size of Process objects. * The key is the process that performs the pipe() call. */ tsl::hopscotch_map proc2pending_pipe_ = {}; /** Although a process can have multiple popen()ed children running in parallel, * it can only have at most one pending popen() operation at a given time. * This is because the parent process holds the global interceptor mutex and waits for an ACK, * and the supervisor only sends that ACK when the child "sh -c" has already appeared. * Store this rarely used data here to decrease the size of Process objects. * The key is the process that performs the popen() call. */ tsl::hopscotch_map proc2pending_popen_ = {}; /** Finalized ExecedProcesses to be garbage collected. */ std::queue proc_gc_queue_ = {}; /** * Directory the first executed process starts in. This is presumably the top directory * of the project to be built. */ const FileName* top_dir_ {nullptr}; void insert_process(Process *p); void delete_process_subtree(Process *p); void profile_collect_cmds(const Process &p, tsl::hopscotch_map *cmds, std::set *ancestors); DISALLOW_COPY_AND_ASSIGN(ProcessTree); }; /* singleton */ extern ProcessTree *proc_tree; } /* namespace firebuild */ #endif // FIREBUILD_PROCESS_TREE_H_ firebuild-0.8.2/src/firebuild/report.cc000066400000000000000000000474431447164520700200720ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/report.h" #include #include #include #include #include #include #include #include #include #include "firebuild/debug.h" #include "firebuild/process_tree.h" namespace firebuild { /** * Profile is aggregated by command name (argv[0]). * For each command (C) we store the cumulated CPU time in microseconds * (system + user time), and count the invocations of each other command * by C. */ tsl::hopscotch_map cmd_profs {}; /** * Index of each used file in the JavaScript files[] array. */ tsl::hopscotch_map used_files_index_map {}; struct string_vector_ptr_hash { size_t operator()(const std::vector* v) const { std::hash hasher; std::size_t seed = 0; for (auto const & s : *v) { seed ^= hasher(s) + 0x9e3779b9 + (seed << 6) + (seed >> 2); } return seed; } }; struct string_vector_ptr_eq { bool operator() (const std::vector* a, const std::vector* b) const { return *a == *b; } }; /** * Index of each used environment in the JavaScript envs[] array. */ tsl::hopscotch_map*, int, struct string_vector_ptr_hash, struct string_vector_ptr_eq> used_envs_index_map {}; /** * Escape std::string for JavaScript * from http://stackoverflow.com/questions/7724448/simple-json-string-escape-for-c * TODO: use JSONCpp instead to handle all cases */ static std::string escapeJsonString(const std::string& input) { std::ostringstream ss; for (auto iter = input.cbegin(); iter != input.cend(); iter++) { switch (*iter) { case '\\': ss << "\\\\"; break; case '"': ss << "\\\""; break; case '\b': ss << "\\b"; break; case '\f': ss << "\\f"; break; case '\n': ss << "\\n"; break; case '\r': ss << "\\r"; break; case '\t': ss << "\\t"; break; default: ss << *iter; break; } } return ss.str(); } static const char* full_relative_path_or_basename(const char *name) { const char* name_last_slash = strrchr(name, '/'); return name_last_slash && path_is_absolute(name) ? name_last_slash + 1 : name; } static void fprintf_ffu_file(FILE* stream, const file_file_usage& ffu) { fprintf(stream, "files[%d],", used_files_index_map[ffu.file]); } static void export2js(const ExecedProcess* proc, const unsigned int level, FILE* stream, unsigned int * nodeid) { // TODO(rbalint): escape all strings properly auto indent_str = std::string(2 * level, ' '); const char* indent = indent_str.c_str(); fprintf(stream, "name:\"%s\",\n", full_relative_path_or_basename(proc->args()[0].c_str())); fprintf(stream, "%s id: %u,\n", indent, (*nodeid)++); fprintf(stream, "%s pid: %u,\n", indent, proc->pid()); fprintf(stream, "%s ppid: %u,\n", indent, proc->ppid()); fprintf(stream, "%s fb_pid: %u,\n", indent, proc->fb_pid()); fprintf(stream, "%s initial_wd:\"%s\",\n", indent, proc->initial_wd()->c_str()); fprintf(stream, "%s exe:\"%s\",\n", indent, proc->executable()->c_str()); fprintf(stream, "%s state: %u,\n", indent, proc->state()); if (proc->was_shortcut()) { fprintf(stream, "%s was_shortcut: true,\n", indent); } if (proc->shortcut_result()) { fprintf(stream, "%s sc_result: \"%s\",\n", indent, escapeJsonString(proc->shortcut_result()).c_str()); } if (!proc->can_shortcut()) { fprintf(stream, "%s cant_sc_reason: \"%s\",\n", indent, escapeJsonString(proc->cant_shortcut_reason()).c_str()); if (proc->cant_shortcut_proc()->exec_proc()->fb_pid() != proc->fb_pid()) { fprintf(stream, "%s cant_sc_fb_pid: \"%u\",\n", indent, proc->cant_shortcut_proc()->exec_proc()->fb_pid()); } } fprintf(stream, "%s args: [", indent); for (auto& arg : proc->args()) { fprintf(stream, "\"%s\",", escapeJsonString(arg).c_str()); } fprintf(stream, "],\n"); fprintf(stream, "%s env: envs[%d],\n", indent, used_envs_index_map[&proc->env_vars()]); fprintf(stream, "%s libs: [", indent); for (auto& lib : proc->libs()) { fprintf(stream, "\"%s\",", lib->c_str()); } fprintf(stream, "],\n"); fprintf(stream, "%s wds: [", indent); for (auto& wd : proc->wds()) { fprintf(stream, "\"%s\",", wd->c_str()); } fprintf(stream, "],\n"); fprintf(stream, "%s failed_wds: [", indent); for (auto& f_wd : proc->failed_wds()) { fprintf(stream, "\"%s\",", f_wd->c_str()); } fprintf(stream, "],\n"); /* sort files before printing */ std::vector ordered_file_usages; for (auto& pair : proc->file_usages()) { if (!pair.second->propagated()) { ordered_file_usages.push_back({pair.first, pair.second}); } } std::sort(ordered_file_usages.begin(), ordered_file_usages.end(), file_file_usage_cmp); fprintf(stream, "%s fcreated: [", indent); for (auto& ffu : ordered_file_usages) { bool isreg_with_hash = ffu.usage->initial_type() == ISREG && ffu.usage->initial_hash_known(); if (!isreg_with_hash && ffu.usage->written()) { fprintf_ffu_file(stream, ffu); } } fprintf(stream, "],\n"); fprintf(stream, "%s fmodified: [", indent); for (auto& ffu : ordered_file_usages) { bool isreg_with_hash = ffu.usage->initial_type() == ISREG && ffu.usage->initial_hash_known(); if (isreg_with_hash && ffu.usage->written()) { fprintf_ffu_file(stream, ffu); } } fprintf(stream, "],\n"); fprintf(stream, "%s fread: [", indent); for (auto& ffu : ordered_file_usages) { bool isreg_with_hash = ffu.usage->initial_type() == ISREG && ffu.usage->initial_hash_known(); if (isreg_with_hash && !ffu.usage->written()) { fprintf_ffu_file(stream, ffu); } } fprintf(stream, "],\n"); fprintf(stream, "%s fnotf: [", indent); for (auto& ffu : ordered_file_usages) { if (ffu.usage->initial_type() == NOTEXIST) { fprintf_ffu_file(stream, ffu); } } fprintf(stream, "],\n"); if (proc->state() != FB_PROC_FINALIZED) { // TODO(rbalint) something went wrong } if (proc->fork_point()->exit_status() != -1) { fprintf(stream, "%s exit_status: %u,\n", indent, proc->fork_point()->exit_status()); } fprintf(stream, "%s utime_u: %" PRId64 ",\n", indent, proc->utime_u()); fprintf(stream, "%s stime_u: %" PRId64 ",\n", indent, proc->stime_u()); fprintf(stream, "%s aggr_time_u: %" PRId64 ",\n", indent, proc->aggr_cpu_time_u()); } static void export2js_recurse_ep(const ExecedProcess* proc, const unsigned int level, FILE* stream, unsigned int *nodeid); static void export2js_recurse_p(const Process* proc, const unsigned int level, FILE* stream, unsigned int *nodeid) { if (proc->exec_child() != NULL) { export2js_recurse_ep(proc->exec_child(), level + 1, stream, nodeid); } for (const Process* fork_child : proc->fork_children()) { export2js_recurse_p(fork_child, level, stream, nodeid); } } static void export2js_recurse_ep(const ExecedProcess* proc, const unsigned int level, FILE* stream, unsigned int *nodeid) { if (level > 0) { fprintf(stream, "\n"); } fprintf(stream, "%s{", std::string(2 * level, ' ').c_str()); export2js(proc, level, stream, nodeid); fprintf(stream, "%s children: [", std::string(2 * level, ' ').c_str()); export2js_recurse_p(proc, level, stream, nodeid); if (level == 0) { fprintf(stream, "]};\n"); } else { fprintf(stream, "]},\n"); } } static void export2js(ProcessTree* proc_tree, FILE * stream) { fprintf(stream, "data = "); unsigned int nodeid = 0; if (proc_tree->root()->exec_child()) { export2js_recurse_ep(proc_tree->root()->exec_child(), 0, stream, &nodeid); } else { // TODO(rbalint) provide nicer report on this error fprintf(stream, "{name: \"\", id: 0, aggr_time_u: 0, children: []};"); } } static void collect_used_files_and_envs(const Process &p, tsl::hopscotch_set *used_files, tsl::hopscotch_set*, string_vector_ptr_hash, string_vector_ptr_eq> *envs) { if (p.exec_child() != NULL) { ExecedProcess *exec_child = static_cast(p.exec_child()); for (const auto& pair : exec_child->file_usages()) { if (!pair.second->propagated()) { /* Save time by not processing propagated ones. */ used_files->insert(pair.first); } } envs->insert(&exec_child->env_vars()); collect_used_files_and_envs(*exec_child, used_files, envs); } for (auto& fork_child : p.fork_children()) { collect_used_files_and_envs(*fork_child, used_files, envs); } } void fprint_collected_files(FILE* stream, const tsl::hopscotch_set& used_files_set) { int index = 0; fprintf(stream, "files = [\n"); for (const FileName* filename : used_files_set) { fprintf(stream, " \"%s\", // files[%d]\n", escapeJsonString(filename->to_string()).c_str(), index); used_files_index_map.insert({filename, index++}); } fprintf(stream, "];\n"); } void fprint_collected_envs( FILE* stream, const tsl::hopscotch_set*, string_vector_ptr_hash, string_vector_ptr_eq>& used_envs_set) { int index = 0; fprintf(stream, "envs = [\n"); for (const std::vector* env : used_envs_set) { fprintf(stream, " ["); for (const std::string& env_var : *env) { fprintf(stream, "\"%s\",", escapeJsonString(env_var).c_str()); } fprintf(stream, "], // envs[%d]\n", index); used_envs_index_map.insert({env, index++}); } fprintf(stream, "];\n"); } static void profile_collect_cmds(const Process &p, tsl::hopscotch_map *cmds, std::set *ancestors) { if (p.exec_child() != NULL) { ExecedProcess *ec = static_cast(p.exec_child()); if (ancestors->count(ec->args()[0]) == 0) { (*cmds)[ec->args()[0]].sum_aggr_time_u += ec->aggr_cpu_time_u(); } else { if (!(*cmds)[ec->args()[0]].recursed) { (*cmds)[ec->args()[0]].recursed = true; } } (*cmds)[ec->args()[0]].count += 1; } for (auto& fork_child : p.fork_children()) { profile_collect_cmds(*fork_child, cmds, ancestors); } } static void build_profile(const Process &p, std::set *ancestors) { bool first_visited = false; if (p.exec_started()) { auto *e = static_cast(&p); auto &cmd_prof = cmd_profs[e->args()[0]]; if (ancestors->count(e->args()[0]) == 0) { cmd_prof.aggr_time_u += e->aggr_cpu_time_u(); ancestors->insert(e->args()[0]); first_visited = true; } cmd_prof.cmd_time_u += e->utime_u() + e->stime_u(); profile_collect_cmds(p, &cmd_prof.subcmds, ancestors); } if (p.exec_child() != NULL) { build_profile(*p.exec_child(), ancestors); } for (auto& fork_child : p.fork_children()) { build_profile(*fork_child, ancestors); } if (first_visited) { ancestors->erase(static_cast(&p)->args()[0]); } } /** * Convert HSL color to HSV color * * From http://ariya.blogspot.hu/2008/07/converting-between-hsl-and-hsv.html */ static void hsl_to_hsv(const double hh, const double ss, const double ll, double *const h, double * const s, double * const v) { double ss_tmp; *h = hh; ss_tmp = ss * ((ll <= 0.5) ? ll : 1 - ll); *v = ll + ss_tmp; *s = (2 * ss_tmp) / (ll + ss_tmp); } /** * Ratio to HSL color std::string * @param r 0.0 .. 1.0 */ static std::string pct_to_hsv_str(const double p) { const double hsl_min[] = {2.0/3.0, 0.80, 0.25}; /* blue */ const double hsl_max[] = {0.0, 1.0, 0.5}; /* red */ const double r = p / 100; double hsl[3]; double hsv[3]; hsl[0] = hsl_min[0] + r * (hsl_max[0] - hsl_min[0]); hsl[1] = hsl_min[1] + r * (hsl_max[1] - hsl_min[1]); hsl[2] = hsl_min[2] + r * (hsl_max[2] - hsl_min[2]); hsl_to_hsv(hsl[0], hsl[1], hsl[2], &(hsv[0]), &(hsv[1]), &(hsv[2])); return (std::to_string(hsv[0]) + ", " + std::to_string(hsv[1]) + ", " + std::to_string(hsv[2])); } static double percent_of(const double val, const double of) { return (((of < std::numeric_limits::epsilon()) && (of > -std::numeric_limits::epsilon())) ? 0.0 : val * 100 / of); } static void export_profile2dot(FILE* stream) { std::set cmd_chain; double min_penwidth = 1, max_penwidth = 8; int64_t build_time; /* build profile */ build_profile(*proc_tree->root(), &cmd_chain); build_time = (proc_tree->root() && proc_tree->root()->exec_child()) ? proc_tree->root()->exec_child()->aggr_cpu_time_u() : 0; /* print it */ fprintf(stream, "digraph {\n"); fprintf(stream, "graph [dpi=63, ranksep=0.25, rankdir=LR, " "bgcolor=transparent, fontname=Helvetica, fontsize=12, " "nodesep=0.125];\n" "node [fontname=Helvetica, fontsize=12, style=filled, height=0," " width=0, shape=box, fontcolor=white];\n" "edge [fontname=Helvetica, fontsize=12]\n"); for (auto& pair : cmd_profs) { fprintf(stream, " \"%s\" [label=<%s
", pair.first.c_str(), full_relative_path_or_basename(pair.first.c_str())); fprintf(stream, "%.2lf%%
(%.2lf%%)>, color=\"%s\"]\n", percent_of(pair.second.aggr_time_u, build_time), percent_of(pair.second.cmd_time_u, build_time), pct_to_hsv_str(percent_of(pair.second.aggr_time_u, build_time)).c_str()); for (auto& pair2 : pair.second.subcmds) { fprintf(stream, " \"%s\" -> \"%s\" [label=\"", pair.first.c_str(), pair2.first.c_str()); if (!pair2.second.recursed) { fprintf(stream, "%.2lf%%\\n", percent_of(pair2.second.sum_aggr_time_u, build_time)); } fprintf(stream, "×%" PRId64 "\", color=\"%s\"," " penwidth=\"%lf\"];", pair2.second.count, pct_to_hsv_str(percent_of(pair2.second.sum_aggr_time_u, build_time)).c_str(), (min_penwidth + ((percent_of(pair2.second.sum_aggr_time_u, build_time) / 100) * (max_penwidth - min_penwidth)))); } } fprintf(stream, "}\n"); } /** * Copy whole file content from in_fd to out_fd retrying on temporary problems. * @param out_fd file desctiptor to write content to * @param in_fd file desctiptor to read content from * @return bytes written, -1 on error */ static ssize_t sendfile_full(int out_fd, int in_fd) { char buf[4096]; ssize_t nread, ret = 0; while (nread = read(in_fd, buf, sizeof buf), nread > 0) { char *out_ptr = buf; ssize_t nwritten; do { nwritten = write(out_fd, out_ptr, nread); if (nwritten >= 0) { nread -= nwritten; out_ptr += nwritten; ret += nwritten; } else if (errno != EINTR) { return -1; } } while (nread > 0); } return ret; } /* * TODO(rbalint) error handling */ void Report::write(const std::string &html_filename, const std::string &datadir) { const char dot_filename[] = "firebuild-profile.dot"; const char svg_filename[] = "firebuild-profile.svg"; // FIXME Use a search path, according to the locations in various popular distributions const std::string d3_datadir = "/usr/share/nodejs/d3/dist"; const char d3_filename[] = "d3.min.js"; const char tree_filename[] = "firebuild-process-tree.js"; const char html_orig_filename[] = "build-report.html"; const std::string dot_cmd = "dot"; FILE* src_file = fopen((datadir + "/" + html_orig_filename).c_str(), "r"); if (src_file == NULL) { fb_perror("fopen"); fb_error("Opening file " + (datadir + "/" + html_orig_filename) + " failed."); fb_error("Can not write build report."); return; } // dirname may modify its parameter thus we provide a writable char string char *html_filename_tmp = new char[html_filename.size() + 1]; strncpy(html_filename_tmp, html_filename.c_str(), html_filename.size() + 1); std::string dir = dirname(html_filename_tmp); delete[] html_filename_tmp; /* export profile */ { FILE* dot = fopen((dir + "/" + dot_filename).c_str(), "w"); if (dot == NULL) { fb_perror("fopen"); fb_error("Failed to open dot file for writing profile graph."); } export_profile2dot(dot); fclose(dot); } auto system_cmd = dot_cmd + " -Tsvg " + dir + "/" + dot_filename + " | sed 's/viewBox=\\\"[^\\\"]*\\\" //' > " + dir + "/" + svg_filename; if (system(system_cmd.c_str()) != 0) { fb_perror("system"); fb_error("Failed to generate profile graph with the following command: " + system_cmd); } FILE* dst_file = fopen(html_filename.c_str(), "w"); int ret = dst_file == NULL ? -1 : 0; while ((ret != -1)) { char* line = NULL; size_t zero = 0; if (getline(&line, &zero, src_file) == -1) { /* finished reading file */ if (!feof(src_file)) { fb_perror("getline"); fb_error("Reading from report template failed."); } free(line); break; } if (strstr(line, d3_filename) != NULL) { int d3 = open((d3_datadir + "/" + d3_filename).c_str(), O_RDONLY); if (d3 == -1) { /* File is not available locally, use the online version. */ fprintf(dst_file, "\n"); fflush(dst_file); } else { fprintf(dst_file, "\n"); close(d3); } } else if (strstr(line, tree_filename) != NULL) { fprintf(dst_file, "\n"); } else if (strstr(line, svg_filename) != NULL) { int svg = open((dir + "/" + svg_filename).c_str(), O_RDONLY); fflush(dst_file); ret = sendfile_full(fileno(dst_file), svg); fsync(fileno(dst_file)); close(svg); } else { fprintf(dst_file, "%s", line); } free(line); } fclose(src_file); fclose(dst_file); std::cout << "FIREBUILD: Generated report: " + html_filename << std::endl; } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/report.h000066400000000000000000000024111447164520700177160ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_REPORT_H_ #define FIREBUILD_REPORT_H_ #include namespace firebuild { class Report { public: /** * Write report to specified file * * @param html_filename report file to be written * @param datadir report template's location * TODO(rbalint) error handling */ static void write(const std::string &html_filename, const std::string &datadir); }; } /* namespace firebuild */ #endif // FIREBUILD_REPORT_H_ firebuild-0.8.2/src/firebuild/sigchild_callback.cc000066400000000000000000000067421447164520700221560ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/sigchild_callback.h" #include #include #include #include "firebuild/debug.h" #include "firebuild/firebuild.h" #include "firebuild/process_debug_suppressor.h" #include "firebuild/process_tree.h" namespace firebuild { static void save_child_status(pid_t pid, int status, int * ret, bool orphan) { TRACK(FB_DEBUG_PROC, "pid=%d, status=%d, orphan=%s", pid, status, D(orphan)); if (WIFEXITED(status)) { *ret = WEXITSTATUS(status); Process* proc = proc_tree->pid2proc(pid); if (proc && proc->fork_point()) { proc->fork_point()->set_exit_status(*ret); } FB_DEBUG(FB_DEBUG_COMM, std::string(orphan ? "orphan" : "child") + " process exited with status " + std::to_string(*ret) + ". (" + d(proc_tree->pid2proc(pid)) + ")"); } else if (WIFSIGNALED(status)) { fprintf(stderr, "%s process has been killed by signal %d\n", orphan ? "Orphan" : "Child", WTERMSIG(status)); } } /* This is the actual business logic for SIGCHLD, called synchronously when processing the events * returned by epoll_wait(). */ void sigchild_cb(const struct epoll_event* event, void *arg) { TRACK(FB_DEBUG_PROC, ""); (void)event; /* unused */ (void)arg; /* unused */ char dummy; int read_ret = read(sigchild_selfpipe[0], &dummy, 1); (void)read_ret; /* unused */ int status = 0; pid_t waitpid_ret; /* Collect exiting children. */ do { waitpid_ret = waitpid(-1, &status, WNOHANG); if (waitpid_ret == child_pid) { /* This is the top process the supervisor started. */ Process* proc = proc_tree->pid2proc(child_pid); assert(proc); ProcessDebugSuppressor debug_suppressor(proc); save_child_status(waitpid_ret, status, &child_ret, false); proc->set_been_waited_for(); } else if (waitpid_ret > 0) { /* This is an orphan process. Its fork parent quit without wait()-ing for it * and as a subreaper the supervisor received the SIGCHLD for it. */ Process* proc = proc_tree->pid2proc(waitpid_ret); if (proc) { /* Since the parent of this orphan process did not wait() for it, it will not be stored in * the cache even when finalizing it. */ assert(!proc->been_waited_for()); } int ret = -1; save_child_status(waitpid_ret, status, &ret, true); } } while (waitpid_ret > 0); if (waitpid_ret < 0) { /* All children exited. Stop listening on the socket, and set listener to -1 to tell the main * epoll loop to quit. */ if (listener > 0) { epoll->del_fd(listener); close(listener); listener = -1; } } } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/sigchild_callback.h000066400000000000000000000021121447164520700220030ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_SIGCHILD_CALLBACK_H_ #define FIREBUILD_SIGCHILD_CALLBACK_H_ #include "firebuild/epoll.h" namespace firebuild { void sigchild_cb(const struct epoll_event* event, void *arg); } /* namespace firebuild */ #endif // FIREBUILD_SIGCHILD_CALLBACK_H_ firebuild-0.8.2/src/firebuild/subkey.h000066400000000000000000000047471447164520700177230ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_SUBKEY_H_ #define FIREBUILD_SUBKEY_H_ #ifdef __APPLE__ #include #include #define htobe64(x) OSSwapHostToBigInt64(x) #else #include #endif #include #include #include #include #include "firebuild/base64.h" #include "firebuild/hash.h" namespace firebuild { class Subkey { public: Subkey() = default; explicit Subkey(uint64_t key) { uint64_t be_key = htobe64(key); Base64::encode(reinterpret_cast(&be_key), str_, sizeof(uint64_t)); } explicit Subkey(const unsigned char digest[8]) { Base64::encode(digest, str_, sizeof(uint64_t)); } explicit Subkey(const char * const str) { #ifdef FB_EXTRA_DEBUG assert(valid_ascii(str)); #endif for (size_t i = 0; i < kAsciiLength + 1; i++) { str_[i] = str[i]; } str_[kAsciiLength] = '\0'; } bool operator<(const Subkey& other) const { return memcmp(str_, other.str_, kAsciiLength) < 0; } const char * c_str() const { return str_; } /** ASCII representation length without the trailing '\0' */ static const size_t kAsciiLength {11}; static bool valid_ascii(const char* const str) { return Base64::valid_ascii(str, kAsciiLength); } private: char str_[kAsciiLength + 1]; }; inline std::string d(const Subkey& ascii_hash, const int level = 0) { return d(ascii_hash.c_str(), level); } } /* namespace firebuild */ namespace std { template <> class hash { public: size_t operator()(const firebuild::Subkey &a) const { return XXH3_64bits(a.c_str(), firebuild::Subkey::kAsciiLength); } }; } #endif // FIREBUILD_SUBKEY_H_ firebuild-0.8.2/src/firebuild/utils.cc000066400000000000000000000237661447164520700177210ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "firebuild/utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "./fbbcomm.h" #include "common/firebuild_common.h" #include "common/platform.h" #include "firebuild/debug.h" tsl::hopscotch_set* deduplicated_strings = nullptr; ssize_t fb_write(int fd, const void *buf, size_t count) { FB_READ_WRITE(write, fd, buf, count); } ssize_t fb_writev(int fd, struct iovec *iov, int iovcnt) { FB_READV_WRITEV(writev, fd, iov, iovcnt); } ssize_t fb_read(int fd, void *buf, size_t count) { FB_READ_WRITE(read, fd, buf, count); } ssize_t fb_copy_file_range(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, const size_t len, unsigned int flags) { ssize_t ret; size_t remaining = len; do { #ifdef __APPLE__ ret = -1; errno = ENOSYS; (void)flags; #else ret = copy_file_range(fd_in, off_in, fd_out, off_out, remaining, flags); #endif if (ret == -1) { if (errno == EXDEV || errno == ENOSYS) { /* Fall back to read and write. */ const bool do_malloc = remaining > 64 * 1024; void* buf = do_malloc ? malloc(remaining) : alloca(remaining); ssize_t bytes_read, bytes_written; if (off_in) { const off_t start_pos = lseek(fd_in, 0, SEEK_CUR); ret = lseek(fd_in, *off_in, SEEK_SET); assert_cmp(ret, ==, *off_in); bytes_read = fb_read(fd_in, buf, remaining); *off_in += bytes_read; ret = lseek(fd_in, start_pos, SEEK_SET); assert_cmp(ret, ==, start_pos); } else { bytes_read = fb_read(fd_in, buf, remaining); } if (off_out) { const off_t start_pos = lseek(fd_out, 0, SEEK_CUR); ret = lseek(fd_out, *off_out, SEEK_SET); assert_cmp(ret, ==, *off_out); bytes_written = fb_write(fd_out, buf, bytes_read); *off_out += bytes_written; ret = lseek(fd_out, start_pos, SEEK_SET); assert_cmp(ret, ==, start_pos); } else { bytes_written = fb_write(fd_out, buf, bytes_read); } if (do_malloc) { free(buf); } return bytes_written; } else { return ret; } } else if (ret == 0) { return len - remaining; } else { remaining -= ret; } } while (remaining > 0); return len; } unsigned char fixed_dirent_type(const struct dirent* dirent, DIR* dir, const std::string& dir_path) { if (dirent->d_type == DT_UNKNOWN) { struct stat st; if (fstatat(dirfd(dir), dirent->d_name, &st, AT_SYMLINK_NOFOLLOW) == -1) { firebuild::fb_error("Failed checking stat()-ing file: " + dir_path + "/" + firebuild::d(dirent->d_name)); perror("fstatat"); return dirent->d_type; } else { switch (st.st_mode & S_IFMT) { case S_IFREG: return DT_REG; case S_IFDIR: return DT_DIR; default: /* Leaving d_type as it was. */ return DT_UNKNOWN; } } } else { return dirent->d_type; } } off_t file_size(DIR* dir, const char* name) { struct stat st; int dir_fd = dir ? dirfd(dir) : AT_FDCWD; if (fstatat(dir_fd, name, &st, 0) == 0) { return S_ISREG(st.st_mode) ? st.st_size : 0; } else { firebuild::fb_perror("fstatat"); return 0; } } off_t recursive_total_file_size(const std::string& path) { DIR * dir = opendir(path.c_str()); if (dir == NULL) { return 0; } /* Visit dirs recursively and collect all the file sizes. */ off_t total = 0; struct dirent *dirent; while ((dirent = readdir(dir)) != NULL) { const char* name = dirent->d_name; if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) { continue; } switch (fixed_dirent_type(dirent, dir, path)) { case DT_DIR: { total += recursive_total_file_size(path + "/" + name); break; } case DT_REG: { struct stat st; if (fstatat(dirfd(dir), name, &st, 0) == 0) { total += st.st_size; } break; } default: /* Just ignore the non-regulat file. */ break; } } closedir(dir); return total; } int file_overwrite_printf(const std::string& path, const char* format, ...) { int ret; FILE* f; const std::string tmp_path = path + "." + std::to_string(getpid()); if (!(f = fopen(tmp_path.c_str(), "w"))) { perror("fopen"); exit(EXIT_FAILURE); } va_list ap; va_start(ap, format); ret = vfprintf(f, format, ap); if (ret == -1) { perror("vfprintf"); unlink(tmp_path.c_str()); return ret; } va_end(ap); fclose(f); ret = rename(tmp_path.c_str(), path.c_str()); if (ret < 0) { unlink(tmp_path.c_str()); } return ret; } void bump_limits() { struct rlimit rlim; getrlimit(RLIMIT_NOFILE, &rlim); /* 8K is expected to be enough for up more than 2K parallel intercepted processes, thus try to * bump the limit above that. */ rlim_t preferred_limit = (rlim.rlim_max == RLIM_INFINITY) ? 8192 : rlim.rlim_max; if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < preferred_limit) { FB_DEBUG(firebuild::FB_DEBUG_COMM, "Increasing limit of open files from " + std::to_string(rlim.rlim_cur) + " to " + std::to_string(preferred_limit) + ""); rlim.rlim_cur = preferred_limit; setrlimit(RLIMIT_NOFILE, &rlim); } } namespace firebuild { void ack_msg(const int conn, const uint16_t ack_num) { TRACK(FB_DEBUG_COMM, "conn=%s, ack_num=%d", D_FD(conn), ack_num); FB_DEBUG(firebuild::FB_DEBUG_COMM, "sending ACK no. " + d(ack_num)); msg_header msg = {}; msg.ack_id = ack_num; fb_write(conn, &msg, sizeof(msg)); FB_DEBUG(firebuild::FB_DEBUG_COMM, "ACK sent"); } void send_fbb(int conn, int ack_num, const FBBCOMM_Builder *msg, int *fds, int fd_count) { TRACK(FB_DEBUG_COMM, "conn=%s, ack_num=%d fd_count=%d", D_FD(conn), ack_num, fd_count); if (FB_DEBUGGING(firebuild::FB_DEBUG_COMM)) { std::vector fds_vec(fds, fds + fd_count); fprintf(stderr, "Sending message with ancillary fds %s:\n", D(fds_vec)); msg->debug(stderr); } int len = msg->measure(); char *buf = reinterpret_cast(alloca(sizeof(msg_header) + len)); memset(buf, 0, sizeof(msg_header)); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" reinterpret_cast(buf)->ack_id = ack_num; reinterpret_cast(buf)->msg_size = len; reinterpret_cast(buf)->fd_count = fd_count; #pragma GCC diagnostic pop msg->serialize(buf + sizeof(msg_header)); if (fd_count == 0) { /* No fds to attach. Send the header and the payload in a single step. */ fb_write(conn, buf, sizeof(msg_header) + len); } else { /* We have some fds to attach. Send the header and the payload separately. This means that the * file descriptors (ancillary data) are attached to the first byte of the payload. */ /* Send the header. */ fb_write(conn, buf, sizeof(msg_header)); /* Prepare to send the payload, with the fds attached as ancillary data. */ struct iovec iov = {}; iov.iov_base = buf + sizeof(msg_header); iov.iov_len = len; void *anc_buf; size_t anc_buf_size = CMSG_SPACE(fd_count * sizeof(int)); anc_buf = alloca(anc_buf_size); memset(anc_buf, 0, anc_buf_size); struct msghdr msgh = {}; msgh.msg_iov = &iov; msgh.msg_iovlen = 1; msgh.msg_control = anc_buf; msgh.msg_controllen = anc_buf_size; struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msgh); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; cmsg->cmsg_len = CMSG_LEN(fd_count * sizeof(int)); memcpy(CMSG_DATA(cmsg), fds, fd_count * sizeof(int)); /* Send the payload. The socket is almost empty (it can only contain the header), so we can * safely expect sendmsg() to fully succeed, no short write, if the message is reasonably sized. * FIXME implement fb_sendmsg() which retries, just to be even safer. */ sendmsg(conn, &msgh, 0); } } void fb_perror(const char *s) { perror((std::string("FIREBUILD: ") + s).c_str()); } # int fb_renameat2(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags) { int ret; #if FB_GLIBC_PREREQ(2, 28) ret = renameat2(olddirfd, oldpath, newdirfd, newpath, flags); #else #ifdef SYS_renameat2 ret = syscall(SYS_renameat2, olddirfd, oldpath, newdirfd, newpath, flags); #else ret = -1; errno = ENOSYS; (void)flags; #endif #endif if (ret == -1 && (errno == ENOSYS || errno == EINVAL)) { if (flags & RENAME_NOREPLACE && faccessat(newdirfd, newpath, F_OK, 0) == 0) { errno = EEXIST; return -1; } else { return renameat(olddirfd, oldpath, newdirfd, newpath); } } else { return ret; } } const std::string& deduplicated_string(std::string str) { if (!deduplicated_strings) { deduplicated_strings = new tsl::hopscotch_set(); } return *deduplicated_strings->insert(str).first; } } /* namespace firebuild */ firebuild-0.8.2/src/firebuild/utils.h000066400000000000000000000111131447164520700175420ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_UTILS_H_ #define FIREBUILD_UTILS_H_ #include #include #include #include #include "common/platform.h" #include "./fbbcomm.h" /** Wrapper retrying on recoverable errors */ ssize_t fb_copy_file_range(int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags); /** Add two struct timespecs. Equivalent to the same method of BSD. */ #define timespecadd(a, b, res) do { \ (res)->tv_sec = (a)->tv_sec + (b)->tv_sec; \ (res)->tv_nsec = (a)->tv_nsec + (b)->tv_nsec; \ if ((res)->tv_nsec >= 1000 * 1000 * 1000) { \ (res)->tv_sec++; \ (res)->tv_nsec -= 1000 * 1000 * 1000; \ } \ } while (0) /** Subtract two struct timespecs. Equivalent to the same method of BSD. */ #define timespecsub(a, b, res) do { \ (res)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ (res)->tv_nsec = (a)->tv_nsec - (b)->tv_nsec; \ if ((res)->tv_nsec < 0) { \ (res)->tv_sec--; \ (res)->tv_nsec += 1000 * 1000 * 1000; \ } \ } while (0) /** Compare two struct timespecs. Equivalent to the same method of BSD. */ #define timespeccmp(a, b, OP) \ (((a)->tv_sec == (b)->tv_sec) ? \ (((a)->tv_nsec)OP((b)->tv_nsec)) : \ (((a)->tv_sec)OP((b)->tv_sec))) /** Get fixed up dirent type even if the underlying filesystem did not support d_type. */ unsigned char fixed_dirent_type(const struct dirent* dirent, DIR* dir, const std::string& dir_path); /** * Return file size of dir / name in bytes, or 0 on error (printing error) * Directory sizes are reported as 0 bytes. */ off_t file_size(DIR* dir, const char* name); /** Returns total size of all regular files in the directory recursively. */ off_t recursive_total_file_size(const std::string& path); /** Overwrite file with the passed printf formatted string */ int file_overwrite_printf(const std::string& path, const char* format, ...); /** * Bump RLIMIT_NOFILE to hard limit to allow more parallel interceptor connections. */ void bump_limits(); namespace firebuild { /** * ACK a message from the supervised process * @param conn connection file descriptor to send the ACK on * @param ack_num the ACK id */ void ack_msg(const int conn, const uint16_t ack_num); /** * Send an FBB message along with its header, potentially attaching two fds as ancillary data. * * These fds will appear in the intercepted process as opened file descriptors, possibly at * different numeric values (the numbers are automatically rewritten by the kernel). * This is sort of a cross-process dup(), see SCM_RIGHTS in cmsg(3) and unix(7). * Also see #656 for the overall design why we're doing this. * * If there are fds to attach, the message header and the message payload are sent in separate * steps, the message payload carrying the attached fds. * * @param conn connection file descriptor * @param ack_num the ack_num to send * @param msg the FBB message's builder object * @param fds pointer to the file descriptor array * @param fd_count number of fds to send */ void send_fbb(int conn, int ack_num, const FBBCOMM_Builder *msg, int *fds = NULL, int fd_count = 0); void fb_perror(const char *s); #ifndef RENAME_NOREPLACE #define RENAME_NOREPLACE (1 << 0) #endif int fb_renameat2(int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags); /** * Deduplicated strings allocated for the lifetime of the firebuild process. */ const std::string& deduplicated_string(std::string); } /* namespace firebuild */ #endif // FIREBUILD_UTILS_H_ firebuild-0.8.2/src/interceptor/000077500000000000000000000000001447164520700166255ustar00rootroot00000000000000firebuild-0.8.2/src/interceptor/.gitignore000066400000000000000000000000301447164520700206060ustar00rootroot00000000000000gen_* libfirebuild.so.* firebuild-0.8.2/src/interceptor/CMakeLists.txt000066400000000000000000000047331447164520700213740ustar00rootroot00000000000000include_directories("${CMAKE_CURRENT_BINARY_DIR}") # gnu89 required to avoid the __isoc99_scanf hell # ignored warnings most likely don't exist in every compiler version set_source_files_properties(interceptors.c PROPERTIES COMPILE_FLAGS "-std=gnu89 -Wno-pragmas -Wno-unknown-warning-option -Wno-deprecated-declarations ${FAT_ARCH_C_FLAGS}") # the trace markers may truncate the marker messages set_source_files_properties(intercept.c PROPERTIES COMPILE_FLAGS "-Wno-unknown-warning-option -Wno-stringop-truncation -DFIREBUILD_VERSION='\"${FIREBUILD_VERSION}\"' ${FAT_ARCH_C_FLAGS}") if (SANITIZE) set(SANITIZE_INTERCEPTOR_LINK_OPTIONS -fsanitize=undefined) endif() add_custom_command ( OUTPUT gen_decl.h gen_def.c gen_impl.c gen_impl_syscalls.c.inc gen_list.txt gen_reset.c DEPENDS generate_interceptors tpl.c tpl_clone.c tpl_closefrom.c tpl_copy_file_range.c tpl_close_range.c tpl_dlopen.c tpl_dup2.c tpl_error.c tpl_exec.c tpl__exit.c tpl_exit.c tpl_fcntl.c tpl_fork.c tpl__fork.c tpl_ioctl.c tpl_marker_only.c tpl_once.c tpl_open.c tpl_pclose.c tpl_pipe.c tpl_popen.c tpl_posix_spawn.c tpl_posix_spawn_file_actions.c tpl_pthread_create.c tpl_read.c tpl_readlink.c tpl_recvmsg.c tpl_seek.c tpl_shm_open.c tpl_signal.c tpl_skip.c tpl_syscall.c tpl_system.c tpl_write.c WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ./generate_interceptors ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating interceptor files from templates") add_custom_target(gen_files ALL DEPENDS gen_decl.h gen_def.c gen_impl.c gen_impl_syscalls.c.inc gen_list.txt gen_reset.c) set_source_files_properties( env.c ic_file_ops.c PROPERTIES COMPILE_FLAGS "${FAT_ARCH_C_FLAGS}") add_library(firebuild SHARED env.c ic_file_ops.c intercept.c interceptors.c $ $) add_dependencies(firebuild gen_files fbbcomm_gen_files) set_target_properties(firebuild PROPERTIES VERSION "0.0.0" SOVERSION 0) if (NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") set_target_properties(firebuild PROPERTIES INTERPROCEDURAL_OPTIMIZATION True) endif() target_link_libraries(firebuild dl) #if(APPLE) #target_link_options(firebuild PUBLIC -Wno-strict-overflow ${SANITIZE_INTERCEPTOR_LINK_OPTIONS} -arch arm64 -arch arm64e -arch x86_64) #else() target_link_options(firebuild PUBLIC -Wno-strict-overflow ${SANITIZE_INTERCEPTOR_LINK_OPTIONS}) #endif() install(TARGETS firebuild DESTINATION ${CMAKE_INSTALL_LIBDIR}) firebuild-0.8.2/src/interceptor/CPPLINT.cfg000066400000000000000000000001561447164520700204210ustar00rootroot00000000000000exclude_files=(gen_|tpl).*\.(c|h) filter=-build/include,-runtime/int,-build/header_guard,-readability/casting firebuild-0.8.2/src/interceptor/env.c000066400000000000000000000246551447164520700175750ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "interceptor/env.h" #include #include #include #include #include #include "interceptor/intercept.h" /* Avoid typos in repetitive names */ #define FB_INSERT_TRACE_MARKERS "FB_INSERT_TRACE_MARKERS" #define FB_SOCKET "FB_SOCKET" #define LIBFIREBUILD_SO_LEN ((int) strlen(LIBFIREBUILD_SO)) /* Like getenv(), but from a custom environment array */ static char *getenv_from(char **env, const char *name) { int name_len = strlen(name); for (int i = 0; env[i] != NULL; i++) { if (memcmp(name, env[i], name_len) == 0 && env[i][name_len] == '=') { return env[i] + name_len + 1; } } return NULL; } static bool fb_insert_trace_markers_needs_fixup(char **env) { #ifdef FB_EXTRA_DEBUG char *current_value = getenv_from(env, FB_INSERT_TRACE_MARKERS); if (current_value == NULL && !insert_trace_markers) { return false; } if (current_value == NULL || !insert_trace_markers) { return true; } return strcmp(current_value, "1") != 0; #else (void)env; return false; #endif } static bool fb_socket_needs_fixup(char **env) { char *current_value = getenv_from(env, FB_SOCKET); return current_value == NULL || strcmp(current_value, fb_conn_string) != 0; } static bool ld_library_path_needs_fixup(char **env) { if (env_ld_library_path[0] == '\0') { return false; } char *current_value = getenv_from(env, LD_LIBRARY_PATH); // FIXME use more precise search (split both values at ':' or ';', and compare the components) return current_value == NULL || strstr(current_value, env_ld_library_path) == NULL; } static bool ld_preload_needs_fixup(char **env) { char *current_value = getenv_from(env, LD_PRELOAD); if (current_value == NULL) { return true; } const char *loc = strstr(current_value, LIBFIREBUILD_SO); if (loc) { const char *loc_end = loc + LIBFIREBUILD_SO_LEN; if ((loc == current_value || *(loc - 1) == ':' || *(loc - 1) == ' ') && (*loc_end == '\0' || *loc_end == ':' || *loc_end == ' ')) { return false; } } return true; } bool env_needs_fixup(char **env) { /* FB_READ_ONLY_LOCATIONS and FB_IGNORE_LOCATIONS are not fixed up because they are not needed * for correctness, just for improving performance a bit. */ return (fb_insert_trace_markers_needs_fixup(env) || fb_socket_needs_fixup(env) || ld_library_path_needs_fixup(env) || ld_preload_needs_fixup(env)); } int get_env_fixup_size(char **env) { size_t ret; /* Count the number of items. */ int i; for (i = 0; env[i] != NULL; i++) {} /* At most 4 env vars might need to be freshly created, plus make room for the trailing NULL. */ ret = (i + 5) * sizeof(char *); /* Room required, depending on the variable: - variable name + equals sign + restored value + trailing NUL, or - variable name + equals sign + current value + separator + restored value appended + trailing NUL. We might be counting a slightly upper estimate. */ #ifdef FB_EXTRA_DEBUG ret += strlen(FB_INSERT_TRACE_MARKERS "=1") + 1; #endif ret += strlen(FB_SOCKET "=") + strlen(fb_conn_string) + 1; char *e = getenv_from(env, LD_PRELOAD); ret += strlen(LD_PRELOAD "=") + (e ? strlen(e) : 0) + 1 + LIBFIREBUILD_SO_LEN + 1; e = getenv_from(env, LD_LIBRARY_PATH); ret += strlen(LD_LIBRARY_PATH "=") + (e ? strlen(e) : 0) + 1 + (env_ld_library_path[0] != '\0' ? strlen(env_ld_library_path) : 0) + 1; return ret; } #ifdef FB_EXTRA_DEBUG /* Places the desired value of the FB_INSERT_TRACE_MARKERS env var * (including the "FB_INSERT_TRACE_MARKERS=" prefix) at @p. * * Returns the number of bytes placed (including the trailing NUL), * or 0 if this variable doesn't need to be set. */ static int fixup_fb_insert_trace_markers(char *p) { insert_debug_msg("Fixing up FB_INSERT_TRACE_MARKERS in the environment"); if (!insert_trace_markers) { return 0; } int offset; sprintf(p, "%s=1%n", FB_INSERT_TRACE_MARKERS, &offset); /* NOLINT */ return offset + 1; } #endif /* Places the desired value of the FB_SOCKET env var * (including the "FB_SOCKET=" prefix) at @p. * * Returns the number of bytes placed (including the trailing NUL). */ static int fixup_fb_socket(char *p) { insert_debug_msg("Fixing up FB_SOCKET in the environment"); int offset; sprintf(p, "%s=%s%n", FB_SOCKET, fb_conn_string, &offset); /* NOLINT */ return offset + 1; } /* Places the desired value of the LD_LIBRARY_PATH env var * (including the "LD_LIBRARY_PATH=" prefix) at @p. * The desired value depends on the @current_value. * * Returns the number of bytes placed (including the trailing NUL). */ static int fixup_ld_library_path(const char *current_value, char *p) { insert_debug_msg("Fixing up LD_LIBRARY_PATH in the environment"); int offset; if (current_value == NULL) { sprintf(p, "%s=%s%n", LD_LIBRARY_PATH, env_ld_library_path, &offset); /* NOLINT */ } else { sprintf(p, "%s=%s:%s%n", LD_LIBRARY_PATH, current_value, env_ld_library_path, &offset); /* NOLINT */ } return offset + 1; } /* Places the desired value of the LD_PRELOAD env var * (including the "LD_PRELOAD=" prefix) at @p. * The desired value depends on the @current_value. * * Appends libfirebuild.so to the end, if needed. * (The intercepted program removed libfirebuild.so from LD_PRELOAD and added something, * presumably its own library instead of _prepending_ its own library. The fix is thus _appending_ * libfirebuild.so to pretend that the program did the proper prepending.) * * Returns the number of bytes placed (including the trailing NUL). */ static int fixup_ld_preload(const char *current_value, char *p) { insert_debug_msg("Fixing up LD_PRELOAD in the environment"); int offset; if (current_value == NULL) { sprintf(p, "%s=%s%n", LD_PRELOAD, LIBFIREBUILD_SO, &offset); /* NOLINT */ } else { /* Append the library. */ sprintf(p, "%s=%s:%s%n", LD_PRELOAD, current_value, LIBFIREBUILD_SO, &offset); /* NOLINT */ } return offset + 1; } static inline bool begins_with(const char *str, const char *prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } void env_fixup(char **env, void *buf) { /* The first part of buf, accessed via buf1, up to buf2, will contain the new env pointers. * The second part, from buf2, will contain the env strings that needed to be copied and fixed. */ char **buf1 = (char **) buf; assert(buf1 != NULL); /* Make scan-build happy */ /* Count the number of items. */ int i; for (i = 0; env[i] != NULL; i++) {} /* At most 4 env vars might need to be freshly created, plus make room for the trailing NULL. */ char *buf2 = (char *) buf + (i + 5) * sizeof(char *); bool fb_insert_trace_markers_fixed_up = false; bool fb_socket_fixed_up = false; bool ld_library_path_fixed_up = false; bool ld_preload_fixed_up = false; #ifdef FB_EXTRA_DEBUG /* Fix up FB_INSERT_TRACE_MARKERS if needed */ if (fb_insert_trace_markers_needs_fixup(env)) { int size = fixup_fb_insert_trace_markers(buf2); if (size > 0) { *buf1++ = buf2; buf2 += size; } fb_insert_trace_markers_fixed_up = true; } #endif /* Fix up FB_SOCKET if needed */ if (fb_socket_needs_fixup(env)) { int size = fixup_fb_socket(buf2); assert(size > 0); *buf1++ = buf2; buf2 += size; fb_socket_fixed_up = true; } /* Fix up LD_LIBRARY_PATH if needed */ if (ld_library_path_needs_fixup(env)) { char *current_value = getenv_from(env, LD_LIBRARY_PATH); int size = fixup_ld_library_path(current_value, buf2); assert(size > 0); *buf1++ = buf2; buf2 += size; ld_library_path_fixed_up = true; } /* Fix up LD_PRELOAD if needed */ if (ld_preload_needs_fixup(env)) { char *current_value = getenv_from(env, LD_PRELOAD); #ifndef NDEBUG int size = #endif fixup_ld_preload(current_value, buf2); assert(size > 0); *buf1++ = buf2; /* buf2 += size; */ ld_preload_fixed_up = true; } /* Copy the environment, skipping the ones that we already fixed up. */ for (i = 0; env[i] != NULL; i++) { if ((fb_insert_trace_markers_fixed_up && begins_with(env[i], FB_INSERT_TRACE_MARKERS "=")) || (fb_socket_fixed_up && begins_with(env[i], FB_SOCKET "=")) || (ld_library_path_fixed_up && begins_with(env[i], LD_LIBRARY_PATH "=")) || (ld_preload_fixed_up && begins_with(env[i], LD_PRELOAD "="))) { continue; } *buf1++ = env[i]; } *buf1 = NULL; } void env_purge(char **env) { char **cur = env; assert(cur != NULL); /* Make scan-build happy */ /* Copy the environment, skipping the ones that need to be removed. */ for (int i = 0; env[i] != NULL; i++) { if ((begins_with(env[i], FB_INSERT_TRACE_MARKERS "=")) || (begins_with(env[i], FB_SOCKET "="))) { continue; } if (begins_with(env[i], LD_PRELOAD "=")) { /* Clear libfirebuild.so */ if (strcmp(env[i], LD_PRELOAD "=" LIBFIREBUILD_SO) == 0) { /* Just skip LD_PRELOAD. */ continue; } else { char * start = strstr(env[i], LIBFIREBUILD_SO); size_t move_len = LIBFIREBUILD_SO_LEN; if (start) { if (*(start - 1) == ':' || *(start - 1) == ' ') { /* Clear separator before. */ start--; move_len++; } else if (*(start + move_len) == ':' || *(start + move_len) == ' ') { /* Clear separator after. */ move_len++; } size_t remaining_len = strlen(start); /* Move LD_PRELOAD's contents including '\0' earlier to overwrite libfirebuild.so. */ memmove(start, start + move_len, remaining_len - move_len + 1); } } } *cur++ = env[i]; } *cur = NULL; } firebuild-0.8.2/src/interceptor/env.h000066400000000000000000000063501447164520700175720ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_ENV_H_ #define FIREBUILD_ENV_H_ #ifdef __APPLE__ #include #define environ (*_NSGetEnviron()) #endif #include /** * Whether the environment needs any fixup at all. */ bool env_needs_fixup(char **env); /** * Returns a size that's large enough to hold the fixed up environment, * including the array of pointers, and the strings that needed to be modified. * * This method was designed to be usable if the caller wants to fix the environment * on the stack, because exec*() need this. */ int get_env_fixup_size(char **env); /** * Fixes up the environment to hold the essential stuff required for Firebuild. * * Wherever possible, only the pointers are copied. Wherever necessary, a copy and * fix of the string is created. * * buf has to be large enough to contain the data, as returned by get_env_fixup_size(). * * The fixed up environment will begin at buf. * * This method was designed to be usable if the caller wants to fix the environment * on the stack, because exec*() need this. */ void env_fixup(char **env, void *buf); /** * Fix up the environment * * This is racy because it operates on the global "environ", but is probably good enough. * A proper solution would require to prefix "cmd" with a wrapper that fixes it up, but that could * be slow. */ #define ENVIRON_SAVE_AND_FIXUP(did_env_fixup, environ_saved) \ bool did_env_fixup = false; \ char **environ_saved = environ; \ if (i_am_intercepting && env_needs_fixup(environ)) { \ did_env_fixup = true; \ int env_fixup_size = get_env_fixup_size(environ); \ environ = alloca(env_fixup_size); \ env_fixup(environ_saved, environ); \ } \ do { \ } while (0) #define ENVIRON_RESTORE(did_env_fixup, environ_saved) \ if (did_env_fixup) { \ environ = environ_saved; \ } \ do { \ } while (0) /** * Remove environment variables injected by firebuild, to disable interception of children. */ void env_purge(char **env); #endif // FIREBUILD_ENV_H_ firebuild-0.8.2/src/interceptor/generate_interceptors000077500000000000000000003443761447164520700231670ustar00rootroot00000000000000#!/usr/bin/env python3 # Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import os import sys from jinja2 import Environment, FileSystemLoader if len(sys.argv) != 2: print("Usage: ./generate_interceptors outputdir", file=sys.stderr) exit(1) outdir = sys.argv[1] supported_platforms = ['linux', 'darwin'] if sys.platform.startswith('linux'): target = "linux" elif sys.platform.startswith('darwin'): target = "darwin" else: print("Unsupported platform: {}".format(platform.system())) exit(1) libc_outputs = ['decl.h', 'def.c', 'impl.c', 'list.txt', 'reset.c'] syscall_outputs = ['decl.h', 'def.c', 'impl_syscalls.c.inc', 'reset.c'] env = Environment(loader=FileSystemLoader('.'), line_statement_prefix='###', trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=True, extensions=['jinja2.ext.do']) # Use a practically unique temporary filename, see #314 tmpsuffix = ".tmp." + str(os.getpid()) # Time related functions defined on 32 bit architectures time64_functions = set({ "__adjtime64", "__clock_adjtime64", "__clock_gettime64", "__clock_settime64", "__futimens64", "__futimes64", "__futimesat64", "__gettimeofday64", "__lutimes64", "__ntp_gettime64", "__ntp_gettimex64", "__recvmmsg64", "__recvmsg64", "__sendmmsg64", "__sendmsg64", "__settimeofday64", "__time64", "__utime64", "__utimensat64", "__utimes64" }) # Best effort to try to debug as many parameters as easily doable. def add_debug_to_dict(type, name, dict): debug_type_to_when_and_how = { "int": ("before", "%d"), "unsigned int": ("before", "%u"), "mode_t": ("before", "%\" PRImode \""), "uid_t": ("before", "%u"), "gid_t": ("before", "%u"), "long": ("before", "%ld"), "ssize_t": ("before", "%\" PRIssize \""), "off_t": ("before", "%\" PRIoff \""), "off64_t": ("before", "%\" PRIoff64 \""), "unsigned long": ("before", "%lu"), "size_t": ("before", "%\" PRIsize \""), "const char *": ("before", "\\\"%s\\\""), "char *": ("after", "\\\"%s\\\""), # assume the call modifies, so log at the end "FILE *": ("before", "%p"), "DIR *": ("before", "%p"), } if type in debug_type_to_when_and_how: (when, fmt) = debug_type_to_when_and_how[type] if name == 'ret': when = "after" dict['debug_' + when + '_fmt'] += ", " + name + "=" + fmt dict['debug_' + when + '_args'] += ", " + name # Split the C parameter specification of a function, at spaces that are outside of parentheses. # E.g. "int (*compar)(void *, void *), void *arg" -> ["int (*compar)(void *, void *)", "void *arg"] def split_param_spec(sig): sig = sig.strip() if not sig: return [] ret = [] start_pos = 0 pos = 0 nest = 0 while pos <= len(sig): if pos == len(sig) or (sig[pos] == ',' and nest == 0): ret.append(sig[start_pos:pos].strip()) start_pos = pos + 1 elif sig[pos] == '(': nest += 1 elif sig[pos] == ')': nest -= 1 pos += 1 return ret # For one parameter specification of a C function, extract the type and the name into a dictionary. # # E.g. "int pipefd[2]" -> name: "pipefd" # type: "int[2]" # type_and_name: "int pipefd[2]" # vatype: "int *" # vatype_and_name: "int *pipefd" # # E.g. "int (*compar)(void *, void *)" -> name: "compar" # type, vatype: "int (*)(void *, void *)" # type_and_name, vatype_and_name: "int (*compar)(void *, void *)" # def get_arg(sig): arg = {} sig = sig.strip() name_end = len(sig) dimension = 0 # Handle array declaration, e.g. "[2]" in "int pipefd[2]". while name_end > 0 and sig[name_end - 1] == ']': name_end = sig.rindex('[', 0, name_end) dimension += 1 subscript_begin = name_end # Handle function pointer parameter, e.g. "int (*compar)(void *, void *)". # Walk backwards from the end, locate the beginning of the parameter list. if name_end > 0 and sig[name_end - 1] == ')': pos = name_end - 1 nest = 1 while pos > 0 and nest > 0: if sig[pos - 1] == ')': nest += 1 elif sig[pos - 1] == '(': nest -= 1 pos -= 1 # pos now points to the parameter list, e.g. "(void *, void *)". Still need to walk back through a ')'. name_end = sig.rindex(')', 0, pos) # name_end is now the end of the variable name. Let's find the sequence of alphanumeric + '_' characters. name_start = name_end while name_start > 0 and (sig[name_start - 1].isalnum() or sig[name_start - 1] == '_'): name_start -= 1 # The variable name is between name_start and name_end, the rest is the signature. arg['name'] = sig[name_start:name_end].strip() arg['type'] = sig[0:name_start].strip() + sig[name_end:].strip() arg['type_and_name'] = sig[0:name_start] + arg['name'] + sig[name_end:].strip() # it's essentially sig arg['vatype'] = sig[0:name_start].strip() + dimension * '*' + sig[name_end:subscript_begin].strip() arg['vatype_and_name'] = sig[0:name_start] + ('*' * dimension) + arg['name'] + sig[name_end:subscript_begin].strip() return arg # Given a method signature (return type, types and names of parameters) # as strings, generates various values for the convenience of templates. # Supports arrays, but does not support function pointers. # # For the debug info to work, requires a space before the start, # e.g. "char *" and not "char*". # # E.g. # rettype = "int" # sig = "const char *pathname, int flags, int pipefd[2], ..." # -> # args = [{ # 'name': "pathname", # 'type': "const char *", # 'type_and_name': "const char *pathname", # 'vatype': "const char *", # 'vatype_and_name': "const char *pathname", # }, { # 'name': "flags" # 'type': "int", # 'type_and_name': "int flags", # 'vatype': "int", # 'vatype_and_name': "int flags", # }, { # 'name': "pipefd", # 'type': "int[2]", # 'type_and_name': "int pipefd[2]", # 'vatype': "int *", # 'vatype_and_name': "int *pipefd", # }] # vararg = True # types_str = "const char *, int, int[2], ..." # sig_str = "const char *pathname, int flags, int pipefd[2], ..." # names_str = "pathname, flags, pipefd" # # debug_before_fmt ≈ ", pathname = \"%s\", flags = %d" # debug_before_args ≈ ", pathname, flags" # debug_after_fmt ≈ ", ret = %d" # debug_after_args ≈ ", ret" def add_signature_to_dict(rettype, sig, dict): dict['args'] = [] dict['vararg'] = False dict['sig_str'] = sig dict['debug_before_fmt'] = '' dict['debug_before_args'] = '' dict['debug_after_fmt'] = '' dict['debug_after_args'] = '' params = split_param_spec(sig) for type_and_name in params: type_and_name = type_and_name.strip() if type_and_name == '...': dict['vararg'] = True else: arg = get_arg(type_and_name) dict['args'].append(arg) add_debug_to_dict(arg['type'], arg['name'], dict) dict['types_str'] = ', '.join(arg['type'] for arg in dict['args']) if dict['vararg']: dict['types_str'] += ', ...' dict['names_str'] = ', '.join(arg['name'] for arg in dict['args']) add_debug_to_dict(rettype, 'ret', dict) return dict # Keep track of the functions we've already generated. generated={} # Generate stuff for the given libc / kernel method. # # Makes sure to exit with an error if there's a skip() or another # generate() call somewhere else for the same method. # # rettype: method's return type, as string # funcs: one or more function names, or syscall names if start with "SYS_", # as (list of) string # sig: the function signature, as string # tpl: the template file to use, default is 'tpl.c' # msg: the message type, default is the first func # success: the success condition, default depends on rettype # dict: further parameters to pass to the template, # see the template for their documentation def generate(rettype, funcs, sig, **dict): if type(funcs) == str: funcs = [funcs] if 'platforms' in dict: for platform in dict['platforms']: if platform not in supported_platforms: print("Unsupported platform: {}".format(platform)) exit(1) if target not in dict['platforms']: return if 'tpl' not in dict: dict['tpl'] = 'tpl.c' else: dict['tpl'] = 'tpl_' + dict['tpl'] + '.c' template = env.get_template(dict['tpl']) dict['target'] = target if 'msg' not in dict: dict['msg'] = funcs[0] if dict['tpl'] != 'tpl_once.c' else 'gen_call' if 'success' not in dict: if rettype == 'void': dict['success'] = "true /* default success condition for void rettype */" elif '*' in rettype: dict['success'] = "ret != NULL /* default success condition for pointer rettype */" else: dict['success'] = "ret >= 0 /* default success condition for scalar rettype */" add_signature_to_dict(rettype, sig, dict) if 'msg_skip_fields' in dict: for msg_skip in dict['msg_skip_fields']: if msg_skip != "error_no" and msg_skip not in [arg['name'] for arg in dict['args']]: print("generate_interceptors: invalid msg_skip_fields value '" + msg_skip + "' for func '" + funcs[0] + "'") exit(1) for func in funcs: if func in generated and func not in ["gettimeofday", "__gettimeofday", "vfork", "SYS_vfork", "__vfork"]: print("generate_interceptors: Error: Already generated '" + func + "'", file=sys.stderr) exit(1) generated[func] = True if func[:4] == 'SYS_': syscall = True outputs = syscall_outputs call_ic_orig_func = "ic_orig_" + func else: syscall = False outputs = libc_outputs call_ic_orig_func = "get_ic_orig_" + func + "()" if func.endswith("_time64") or func in time64_functions: if 'ifdef_guard' not in dict or dict['ifdef_guard'] is None: dict['ifdef_guard'] = glibc_ge(2, 34) + " && (defined(__TIMESIZE) && __TIMESIZE == 32)" else: dict['ifdef_guard'] = dict['ifdef_guard'] + " && (defined(__TIMESIZE) && __TIMESIZE == 32)" if target == 'darwin' and (func.endswith("_unlocked") or func.endswith("_chk") \ or func.endswith("64") or func.endswith("64v2") \ or func.startswith("__")): return for gen in outputs: rendered = template.render(gen=gen, rettype=rettype, func=func, syscall=syscall, call_ic_orig_func=call_ic_orig_func, **dict) if rendered: with open(outdir + "/gen_" + gen + tmpsuffix, "a") as f: f.write(rendered) # Do not intercept the given libc / kernel method. # # This does two things: # # Creates a "#define get_ic_orig_func() func" redirect so that you can always # call get_ic_orig_foo()(args) in the interceptor code, without worrying whether # that method is overridden or not. # # Also makes sure to exit with an error if there's a # generate() call somewhere else for the same method. # # You can pass multiple function names at once. def skip(*list): for func in list: generate('', func, '', tpl="skip") def glibc_ge(major, minor): return "#if FB_GLIBC_PREREQ({}, {})".format(major, minor) def not_glibc_ge(major, minor): return "#if !FB_GLIBC_PREREQ({}, {})".format(major, minor) def apple_ge(major, minor): return '#if (defined __APPLE__) && (__MAC_OS_X_VERSION_MIN_REQUIRED >= {}{}00)'.format(str(major).zfill(2), str(minor).zfill(2)) # Initialize the output files with their headers. # Also truncate them before we'll reopen them plenty of times for appending. for gen in set(libc_outputs + syscall_outputs): with open(outdir + "/gen_" + gen + tmpsuffix, "w") as f: f.write("/* Auto-generated by generate_interceptors, do not edit */\n\n") # Intercept vararg open() and friends generate("int", ["open", "open64", "__open64", "SYS_open", "__open"], "const char *pathname, int flags, ...", tpl="open", msg_skip_fields=["pathname"]) generate("int", ["openat", "openat64", "SYS_openat"], "int dirfd, const char *pathname, int flags, ...", tpl="open", msg_skip_fields=["pathname"], msg="open") # FIXME Intercept SYS_openat2 (#889). It's not the same as the glibc symbols __openat[64]_2. # Intercept open_2 variants generate("int", ["__open_2", "__open64_2"], "const char *pathname, int flags", tpl="open", msg_skip_fields=["pathname"], msg="open") generate("int", ["__openat_2", "__openat64_2"], "int dirfd, const char *pathname, int flags", tpl="open", msg_skip_fields=["pathname"], msg="open") # Intercept creat() generate("int", ["creat", "creat64", "SYS_creat"], "const char *pathname, mode_t mode", tpl="open", before_lines=["const int flags = O_CREAT | O_WRONLY | O_TRUNC;"], msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_flags(&ic_msg, flags);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, pre_open_sent);"], msg="open") # Intercept fopen def open_ack_condition(msg): return "success " \ "&& !is_path_at_locations(fbbcomm_builder_" + msg + "_get_pathname(&ic_msg), fbbcomm_builder_" + msg + "_get_pathname_len(&ic_msg), &read_only_locations) " \ "&& !is_path_at_locations(fbbcomm_builder_" + msg + "_get_pathname(&ic_msg), fbbcomm_builder_" + msg + "_get_pathname_len(&ic_msg), &ignore_locations)" # Note: confusingly open()'s and fopen()'s manual uses the word "mode" for something completely different. generate("FILE *", ["fopen", "fopen64"], "const char *pathname, const char *mode", before_lines=["int open_flags = intercept_fopen_mode_to_open_flags_helper(mode);", "bool pre_open_sent = i_am_intercepting && maybe_send_pre_open(AT_FDCWD, pathname, open_flags);"], after_lines=["int fd = safe_fileno(ret);", "if (i_am_intercepting && success) clear_notify_on_read_write_state(fd);", "assert(!voidp_set_contains(&popened_streams, ret));"], msg="open", msg_skip_fields=["pathname", "mode"], msg_add_fields=["fbbcomm_builder_open_set_flags(&ic_msg, open_flags);", "if (open_flags & O_CREAT) fbbcomm_builder_open_set_mode(&ic_msg, 0666);", "if (success) fbbcomm_builder_open_set_ret(&ic_msg, fd);", "BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, pre_open_sent);"], ack_condition=open_ack_condition("open")) # Intercept freopen generate("FILE *", ["freopen", "freopen64"], "const char *pathname, const char *mode, FILE *stream", before_lines=["int open_flags = intercept_fopen_mode_to_open_flags_helper(mode);", "bool pre_open_sent = i_am_intercepting && maybe_send_pre_open(AT_FDCWD, pathname, open_flags);", "int oldfd = safe_fileno(stream);", "if (i_am_intercepting) clear_notify_on_read_write_state(oldfd);"], after_lines=["int newfd = safe_fileno(ret);", "if (i_am_intercepting && success) clear_notify_on_read_write_state(newfd);"], msg_skip_fields=["pathname", "mode", "stream"], msg_add_fields=["fbbcomm_builder_freopen_set_flags(&ic_msg, intercept_fopen_mode_to_open_flags_helper(mode));", "if (oldfd >= 0) fbbcomm_builder_freopen_set_oldfd(&ic_msg, oldfd);", "if (success) fbbcomm_builder_freopen_set_ret(&ic_msg, newfd);", "BUILDER_SET_ABSOLUTE_CANONICAL(freopen, pathname);", "fbbcomm_builder_freopen_set_pre_open_sent(&ic_msg, pre_open_sent);"], ack_condition=open_ack_condition("freopen")) # High level stream operation only generate("FILE *", "fdopen", "int fd, const char *mode", msg=None, after_lines=["assert(!voidp_set_contains(&popened_streams, ret));"]) generate("int", "shm_open", "const char *name, int oflag, " + ("mode_t mode" if target == 'linux' else "..."), msg="gen_call", tpl="shm_open") generate("int", "shm_unlink", "const char *name", tpl="once") generate("int", "shmget", "key_t key, size_t size, int shmflg", tpl="once") generate("int", ["memfd_create", "SYS_memfd_create"], "const char *name, unsigned int flags", platforms=['linux'], after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], send_ret_on_success=True, send_msg_on_error=False) generate("int", ["timerfd_create", "SYS_timerfd_create"], "int clockid, int flags", platforms=['linux'], after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["clockid"], send_ret_on_success=True, send_msg_on_error=False) generate("int", ["epoll_create", "SYS_epoll_create"], "int size", platforms=['linux'], after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["size"], send_ret_on_success=True, send_msg_on_error=False) generate("int", ["epoll_create1", "SYS_epoll_create1"], "int flags", platforms=['linux'], after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg="epoll_create", send_ret_on_success=True, send_msg_on_error=False) # SYS_eventfd only takes the initial value as parameter. # glibc's eventfd() corresponds to SYS_eventfd2 which takes an additional flags parameter. generate("int", "SYS_eventfd", "unsigned int initval", platforms=['linux'], msg="eventfd", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["initval"], msg_add_fields=["fbbcomm_builder_eventfd_set_flags(&ic_msg, 0);"], send_ret_on_success=True, send_msg_on_error=False) generate("int", ["eventfd", "SYS_eventfd2"], "unsigned int initval, int flags", platforms=['linux'], after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["initval"], send_ret_on_success=True, send_msg_on_error=False) generate("int", "signalfd", "int fd, const sigset_t *mask, int flags", platforms=['linux'], after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["mask"], send_ret_on_success=True, send_msg_on_error=False) generate("int", "SYS_signalfd", "int fd, const sigset_t *mask, size_t sizemask", platforms=['linux'], msg="signalfd", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["mask", "sizemask"], msg_add_fields=["fbbcomm_builder_signalfd_set_flags(&ic_msg, 0);"], send_ret_on_success=True, send_msg_on_error=False) generate("int", "SYS_signalfd4", "int fd, const sigset_t *mask, size_t sizemask, int flags", platforms=['linux'], msg="signalfd", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], msg_skip_fields=["mask", "sizemask"], send_ret_on_success=True, send_msg_on_error=False) # Intercept close and fclose # Don't call the actual method on the supervisor connection fd. generate("int", ["close", "SYS_close", "__close"], "int fd", before_lines=["if (i_am_intercepting) set_notify_on_read_write_state(fd);"]) generate("int", "fclose", "FILE *stream", before_lines=["int fd = safe_fileno(stream); /* save it here, we can't do fileno() after the fclose() */", "if (i_am_intercepting) set_notify_on_read_write_state(fd);", "voidp_set_erase(&popened_streams, stream);"], send_msg_condition="fd != -1", msg="close", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_close_set_fd(&ic_msg, fd);"]) # Intercept closefrom and close_range generate("void", ["closefrom"], "int lowfd", platforms=['linux'], tpl="closefrom", send_msg_on_error=False) # The manpage says "unsigned int flags" but in unistd.h it's "int flags". generate("int", ["close_range", "SYS_close_range"], "unsigned int first, unsigned int last, int flags", platforms=['linux'], tpl="close_range") # Unlike fclose(), fcloseall() only closes the high level streams and not the underlying fds. # So we don't need to notify the supervisor. We need to clear the popened_streams set, though. generate("int", "fcloseall", "", platforms=['linux'], msg=None, after_lines=["voidp_set_clear(&popened_streams);"]) # Intercept opendir, closedir (no need to intercept fdopendir) generate("DIR *", "opendir", "const char *pathname", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(dirfd(ret));"], msg="open", msg_skip_fields=["pathname"], msg_add_fields=["fbbcomm_builder_open_set_flags(&ic_msg, O_RDONLY | O_CLOEXEC | O_DIRECTORY);", "if (success) fbbcomm_builder_open_set_ret(&ic_msg, get_ic_orig_dirfd()(ret));", "BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, false);"], ack_condition=open_ack_condition("open")) generate("int", "closedir", "DIR *dirp", before_lines=["int fd = safe_dirfd(dirp); /* save it here, we can't do dirfd() after the closedir() */"], msg="close", msg_skip_fields=["dirp"], msg_add_fields=["fbbcomm_builder_close_set_fd(&ic_msg, fd);"]) # FIXME implement scandir(at)(64) generate("int", ["mkdir", "SYS_mkdir"], "const char *pathname, mode_t mode", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(mkdir, pathname);"]) generate("int", ["mkdirat", "SYS_mkdirat"], "int dirfd, const char *pathname, mode_t mode", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(mkdir, dirfd, pathname);"], msg="mkdir") # Intercept the remove, rmdir, unlink families generate("int", ["unlink", "remove", "SYS_unlink"], "const char *pathname", before_lines=["bool pre_open_sent = i_am_intercepting && maybe_send_pre_open(AT_FDCWD, pathname, O_WRONLY | O_TRUNC);"], msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(unlink, pathname);", "fbbcomm_builder_unlink_set_pre_open_sent(&ic_msg, pre_open_sent);"]) generate("int", ["unlinkat", "SYS_unlinkat"], "int dirfd, const char *pathname, int flags", before_lines=["bool pre_open_sent = i_am_intercepting && maybe_send_pre_open(dirfd, pathname, O_WRONLY | O_TRUNC);"], msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(unlink, dirfd, pathname);", "fbbcomm_builder_unlink_set_pre_open_sent(&ic_msg, pre_open_sent);"], msg="unlink") generate("int", ["rmdir", "SYS_rmdir"], "const char *pathname", before_lines=["bool pre_open_sent = i_am_intercepting && maybe_send_pre_open(AT_FDCWD, pathname, O_WRONLY | O_TRUNC);"], msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(rmdir, pathname);", "fbbcomm_builder_rmdir_set_pre_open_sent(&ic_msg, pre_open_sent);"]) # Intercept the rename family generate("int", ["rename", "SYS_rename"], "const char *oldpath, const char *newpath", before_lines=["if (i_am_intercepting) {", " send_pre_open_without_ack_request(AT_FDCWD, oldpath);", " send_pre_open(AT_FDCWD, newpath);", "}"], msg_skip_fields=["oldpath", "newpath"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(rename, oldpath);", "BUILDER_SET_ABSOLUTE_CANONICAL(rename, newpath);"], ack_condition="true") generate("int", ["renameat", "SYS_renameat"], "int olddirfd, const char *oldpath, int newdirfd, const char *newpath", before_lines=["if (i_am_intercepting) {", " send_pre_open_without_ack_request(olddirfd, oldpath);", " send_pre_open(newdirfd, newpath);", "}"], msg_skip_fields=["oldpath", "newpath"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(rename, olddirfd, oldpath);", "BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(rename, newdirfd, newpath);"], msg="rename", ack_condition="true") for func_guard in [("SYS_renameat2", None), ("renameat2", glibc_ge(2, 28))]: (func, ifdef_guard) = func_guard generate("int", func, "int olddirfd, const char *oldpath, int newdirfd, const char *newpath, unsigned int flags", platforms=['linux'], ifdef_guard=ifdef_guard, before_lines=["if (i_am_intercepting) {", " send_pre_open_without_ack_request(olddirfd, oldpath);", " send_pre_open(newdirfd, newpath);", "}"], msg_skip_fields=["oldpath", "newpath"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(rename, olddirfd, oldpath);", "BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(rename, newdirfd, newpath);"], msg="rename", ack_condition="true") # Intercept operations that read from a file descriptor. # FIXME also intercept mmap with PROT_READ generate("ssize_t", ["read", "SYS_read", "__read"], "int fd, void *buf, size_t count", tpl="read", msg_skip_fields=["buf", "count"]) generate("ssize_t", "__read_chk", "int fd, void *buf, size_t count, size_t fortify_size", tpl="read", msg_skip_fields=["buf", "count", "fortify_size"]) generate("ssize_t", "readv", "int fd, const struct iovec *iov, int iovcnt", tpl="read", msg_skip_fields=["iov", "iovcnt"]) # Note: SYS_readv is like readv() but takes longs rather than ints. generate("ssize_t", "SYS_readv", "long fd, const struct iovec *iov, unsigned long iovcnt", tpl="read", msg_skip_fields=["iov", "iovcnt"]) generate("ssize_t", "pread", "int fd, void *buf, size_t count, off_t offset", tpl="read", is_pread="true", msg_skip_fields=["buf", "count", "offset"]) generate("ssize_t", "__pread_chk", "int fd, void *buf, size_t count, off_t offset, size_t fortify_size", tpl="read", is_pread="true", msg_skip_fields=["buf", "count", "offset", "fortify_size"]) generate("ssize_t", ["pread64", "__pread64", "SYS_pread64"], "int fd, void *buf, size_t count, off64_t offset", platforms=['linux'], tpl="read", is_pread="true", msg_skip_fields=["buf", "count", "offset"]) generate("ssize_t", "__pread64_chk", "int fd, void *buf, size_t count, off64_t offset, size_t fortify_size", platforms=['linux'], tpl="read", is_pread="true", msg_skip_fields=["buf", "count", "offset", "fortify_size"]) generate("ssize_t", "preadv", "int fd, const struct iovec *iov, int iovcnt, off_t offset", tpl="read", is_pread="true", msg_skip_fields=["iov", "iovcnt", "offset"]) # Note: SYS_preadv is like preadv() but takes longs rather than ints and splits the offset in two. # The manpage says "These arguments contain, respectively, the low order and high order 32 bits of offset." # This seems to be true on 32-bit architectures only. On 64-bit they take 64 bits each, making offset_h unused. generate("ssize_t", "SYS_preadv", "long fd, const struct iovec *iov, unsigned long iovcnt, unsigned long offset_l, unsigned long offset_h", tpl="read", is_pread="true", msg_skip_fields=["iov", "iovcnt", "offset_l", "offset_h"]) generate("ssize_t", "preadv64", "int fd, const struct iovec *iov, int iovcnt, off64_t offset", platforms=['linux'], tpl="read", is_pread="true", msg_skip_fields=["iov", "iovcnt", "offset"]) generate("ssize_t", "preadv2", "int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags", platforms=['linux'], tpl="read", is_pread="offset != -1", msg_skip_fields=["iov", "iovcnt", "offset", "flags"]) # Note: SYS_preadv2 is like preadv2() but takes longs rather than ints and splits the offset in two. # The manpage says "These arguments contain, respectively, the low order and high order 32 bits of offset." # This seems to be true on 32-bit architectures only. On 64-bit they take 64 bits each, making offset_h unused. generate("ssize_t", "SYS_preadv2", "long fd, const struct iovec *iov, unsigned long iovcnt, unsigned long offset_l, unsigned long offset_h, int flags", platforms=['linux'], tpl="read", before_lines=["off64_t offset = sizeof(long) >= 8 ? offset_l : (off64_t)offset_h << 32 | offset_l;"], is_pread="offset != -1", msg_skip_fields=["iov", "iovcnt", "offset_l", "offset_h", "flags"]) generate("ssize_t", "preadv64v2", "int fd, const struct iovec *iov, int iovcnt, off64_t offset, int flags", platforms=['linux'], tpl="read", is_pread="offset != -1", msg_skip_fields=["iov", "iovcnt", "offset", "flags"]) generate("size_t", ["fread", "fread_unlocked"], "void *ptr, size_t size, size_t nmemb, FILE *stream", tpl="read", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="ret > 0 || !ferror(stream)", msg_skip_fields=["ptr", "size", "nmemb", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("size_t", ["__fread_chk", "__fread_unlocked_chk"], "void *ptr, size_t fortify_size, size_t size, size_t nmemb, FILE *stream", tpl="read", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="ret > 0 || !ferror(stream)", msg_skip_fields=["ptr", "fortify_size", "size", "nmemb", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("int", ["fgetc", "fgetc_unlocked", "getc", "getc_unlocked", "getw"], "FILE *stream", tpl="read", success="ret != EOF || !ferror(stream)", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("wint_t", ["fgetwc", "fgetwc_unlocked", "getwc", "getwc_unlocked"], "FILE *stream", tpl="read", success="ret != WEOF || !ferror(stream)", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("char *", ["fgets", "fgets_unlocked"], "char *s, int size, FILE *stream", tpl="read", success="ret != NULL || !ferror(stream)", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["s", "size", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("char *", ["__fgets_chk", "__fgets_unlocked_chk"], "char *s, size_t fortify_size, int size, FILE *stream", success="ret != NULL || !ferror(stream)", tpl="read", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["s", "fortify_size", "size", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("wchar_t *", ["fgetws", "fgetws_unlocked"], "wchar_t *s, int size, FILE *stream", tpl="read", success="ret != NULL || !ferror(stream)", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["s", "size", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("wchar_t *", ["__fgetws_chk", "__fgetws_unlocked_chk"], "wchar_t *s, size_t fortify_size, int size, FILE *stream", success="ret != NULL || !ferror(stream)", tpl="read", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["s", "fortify_size", "size", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("int", ["getchar", "getchar_unlocked"], "", tpl="read", success="ret != EOF || !ferror(stdin)", before_lines=["int fd = safe_fileno(stdin);"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("wint_t", ["getwchar", "getwchar_unlocked"], "", tpl="read", success="ret != WEOF || !ferror(stdin)", before_lines=["int fd = safe_fileno(stdin);"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("char *", "gets", "char *s", # should be never used, see man gets tpl="read", success="ret != NULL || !ferror(stdin)", before_lines=["int fd = safe_fileno(stdin);"], msg_skip_fields=["s"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("char *", "__gets_chk", "char *s, size_t fortify_size", # should be never used, see man gets tpl="read", success="ret != NULL || !ferror(stdin)", before_lines=["int fd = safe_fileno(stdin);"], msg_skip_fields=["s", "fortify_size"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("FB_SSIZE_T", "getline", "char **lineptr, size_t *n, FILE *stream", tpl="read", success="ret != EOF || !ferror(stream)", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["lineptr", "n", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("FB_SSIZE_T", ["getdelim", "__getdelim"], "char **lineptr, size_t *n, int delim, FILE *stream", tpl="read", success="ret != EOF || !ferror(stream)", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["lineptr", "n", "delim", "stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) # glibc magic, see #343 generate("int", ["__uflow", "__underflow"], "FILE *stream", tpl="read", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="true", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) generate("wint_t", ["__wuflow", "__wunderflow"], "FILE *stream", tpl="read", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="true", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"]) # Handle (__isoc99_)?v?f?w?scanf for i in ['', '__isoc99_']: for v in ['', 'v']: for f in ['', 'f']: for w in ['', 'w']: func = i + v + f + w + "scanf" if f == '': sig_str = '' names_str = '' stream="stdin" before_lines = ["int fd = safe_fileno(stdin);"] msg_skip_fields = [] msg_add_fields = ["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"] elif f == 'f': sig_str = "FILE *stream, " names_str = "stream, " stream="stream" before_lines = ["int fd = safe_fileno(stream);"] send_msg_condition="fd != -1", msg_skip_fields = ["stream"] msg_add_fields = ["fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd);"] if w: sig_str += "const wchar_t *format, " else: sig_str += "const char *format, " success="ret != EOF || !ferror(" + stream + ")" names_str += "format, " msg_skip_fields += ["format"] if v: sig_str += "FB_VA_LIST ap" call_orig_lines = None msg_skip_fields += ["ap"] else: sig_str += "..." call_orig_lines = ["ret = get_ic_orig_" + i + "v" + f + w + "scanf()(" + names_str + "ap);"] generate("int", func, sig_str, tpl="read", success=success, before_lines=before_lines, call_orig_lines=call_orig_lines, msg_skip_fields=msg_skip_fields, msg_add_fields=msg_add_fields) if func == "vfscanf": generate("int", "__" + func, sig_str, tpl="read", success=success, before_lines=before_lines, call_orig_lines=call_orig_lines, msg_skip_fields=msg_skip_fields, msg_add_fields=msg_add_fields) for i in ['', '__isoc99_']: for v in ['', 'v']: for w in ['', 'w']: skip(i + v + "s" + w + "scanf") generate("ssize_t", ["recv", "SYS_recv", "__recv"], "int fd, void *buf, size_t len, int flags", tpl="read", msg_skip_fields=["buf", "len", "flags"]) generate("ssize_t", "__recv_chk", "int fd, void *buf, size_t len, size_t fortify_size, int flags", tpl="read", msg_skip_fields=["buf", "len", "fortify_size", "flags"]) generate("ssize_t", ["recvfrom", "SYS_recvfrom"], "int fd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen", tpl="read", msg_skip_fields=["buf", "len", "flags", "src_addr", "addrlen"]) generate("ssize_t", "__recvfrom_chk", "int fd, void *buf, size_t len, size_t fortify_size, int flags, struct sockaddr *src_addr, socklen_t *addrlen", tpl="read", msg_skip_fields=["buf", "len", "fortify_size", "flags", "src_addr", "addrlen"]) generate("ssize_t", ["recvmsg", "SYS_recvmsg", "__recvmsg64"], "int fd, struct msghdr *msg, int flags", tpl="recvmsg", msg_skip_fields=["msg", "flags"]) # FIXME add SYS_recvmmsg_time64 generate("int", ["recvmmsg", "SYS_recvmmsg", "__recvmmsg64"], "int fd, struct mmsghdr *msgvec, unsigned int vlen, int flags, struct timespec *timeout", platforms=['linux'], tpl="recvmsg", msg_skip_fields=["msgvec", "vlen", "flags", "timeout"]) # Intercept operations that write to a file descriptor. # FIXME also intercept mmap with PROT_WRITE generate("ssize_t", ["write", "SYS_write", "__write"], "int fd, const void *buf, size_t count", tpl="write", msg_skip_fields=["buf", "count"]) generate("ssize_t", "writev", "int fd, const struct iovec *iov, int iovcnt", tpl="write", msg_skip_fields=["iov", "iovcnt"]) # Note: SYS_writev is like writev() but takes longs rather than ints. generate("ssize_t", "SYS_writev", "long fd, const struct iovec *iov, unsigned long iovcnt", tpl="write", msg_skip_fields=["iov", "iovcnt"]) generate("ssize_t", "pwrite", "int fd, const void *buf, size_t count, off_t offset", tpl="write", is_pwrite="true", msg_skip_fields=["buf", "count", "offset"]) generate("ssize_t", ["pwrite64", "__pwrite64", "SYS_pwrite64"], "int fd, const void *buf, size_t count, off64_t offset", platforms=['linux'], tpl="write", is_pwrite="true", msg_skip_fields=["buf", "count", "offset"]) generate("ssize_t", "pwritev", "int fd, const struct iovec *iov, int iovcnt, off_t offset", tpl="write", is_pwrite="true", msg_skip_fields=["iov", "iovcnt", "offset"]) # Note: SYS_pwritev is like pwritev() but takes longs rather than ints and splits the offset in two. # The manpage says "These arguments contain, respectively, the low order and high order 32 bits of offset." # This seems to be true on 32-bit architectures only. On 64-bit they take 64 bits each, making offset_h unused. generate("ssize_t", "SYS_pwritev", "long fd, const struct iovec *iov, unsigned long iovcnt, unsigned long offset_l, unsigned long offset_h", tpl="write", is_pwrite="true", msg_skip_fields=["iov", "iovcnt", "offset_l", "offset_h"]) generate("ssize_t", "pwritev64", "int fd, const struct iovec *iov, int iovcnt, off64_t offset", platforms=['linux'], tpl="write", is_pwrite="true", msg_skip_fields=["iov", "iovcnt", "offset"]) generate("ssize_t", "pwritev2", "int fd, const struct iovec *iov, int iovcnt, off_t offset, int flags", platforms=['linux'], tpl="write", is_pwrite="offset != -1", msg_skip_fields=["iov", "iovcnt", "offset", "flags"]) # Note: SYS_pwritev2 is like pwritev2() but takes longs rather than ints and splits the offset in two # The manpage says "These arguments contain, respectively, the low order and high order 32 bits of offset." # This seems to be true on 32-bit architectures only. On 64-bit they take 64 bits each, making offset_h unused. generate("ssize_t", "SYS_pwritev2", "long fd, const struct iovec *iov, unsigned long iovcnt, unsigned long offset_l, unsigned long offset_h, int flags", tpl="write", before_lines=["off64_t offset = sizeof(long) >= 8 ? offset_l : (off64_t)offset_h << 32 | offset_l;"], is_pwrite="offset != -1", msg_skip_fields=["iov", "iovcnt", "offset_l", "offset_h", "flags"]) generate("ssize_t", "pwritev64v2", "int fd, const struct iovec *iov, int iovcnt, off64_t offset, int flags", platforms=['linux'], tpl="write", is_pwrite="offset != -1", msg_skip_fields=["iov", "iovcnt", "offset", "flags"]) generate("size_t", ["fwrite", "fwrite_unlocked"], "const void *ptr, size_t size, size_t nmemb, FILE *stream", tpl="write", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="ret > 0 || !ferror(stream)", msg_skip_fields=["ptr", "size", "nmemb", "stream"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("int", ["fputc", "fputc_unlocked", "putc", "putc_unlocked", "putw"], "int c, FILE *stream", tpl="write", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["c", "stream"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("wint_t", ["fputwc", "fputwc_unlocked", "putwc", "putwc_unlocked"], "wchar_t wc, FILE *stream", tpl="write", success="ret != WEOF", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["wc", "stream"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("int", ["fputs", "fputs_unlocked"], "const char *s, FILE *stream", tpl="write", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["s", "stream"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("int", ["fputws", "fputws_unlocked"], "const wchar_t *s, FILE *stream", tpl="write", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", msg_skip_fields=["s", "stream"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("int", ["putchar", "putchar_unlocked"], "int c", tpl="write", before_lines=["int fd = safe_fileno(stdout);"], msg_skip_fields=["c"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("wint_t", ["putwchar", "putwchar_unlocked"], "wchar_t wc", tpl="write", success="ret != WEOF", before_lines=["int fd = safe_fileno(stdout);"], msg_skip_fields=["wc"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("int", "puts", "const char *s", tpl="write", before_lines=["int fd = safe_fileno(stdout);"], msg_skip_fields=["s"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) # glibc magic, see #343 generate("int", "__overflow", "FILE *stream, int ch", tpl="write", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="true", msg_skip_fields=["stream", "ch"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("wint_t", "__woverflow", "FILE *stream, wint_t ch", tpl="write", before_lines=["int fd = safe_fileno(stream);"], send_msg_condition="fd != -1", success="true", msg_skip_fields=["stream", "ch"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) # There's no public verror() and verror_at_line(). # Construct the message ourselves so that we don't have to rely on # gcc's __builtin_apply() with its arbitrarily guessed size parameter. generate("void", "error", "int status, int errnum, const char *format, ...", platforms=['linux'], tpl="error", before_lines=["int fd = safe_fileno(stderr);"], call_orig_lines=["int msglen = vsnprintf(NULL, 0, format, ap);", "va_end(ap);", "char msgbuf[msglen + 1];", "va_start(ap, format);", "vsnprintf(msgbuf, msglen + 1, format, ap);", "get_ic_orig_error()(status, errnum, \"%s\", msgbuf);"], msg_skip_fields=["status", "errnum", "format"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("void", "error_at_line", "int status, int errnum, const char *filename, unsigned int linenum, const char *format, ...", platforms=['linux'], tpl="error", before_lines=["int fd = safe_fileno(stderr);"], call_orig_lines=["int msglen = vsnprintf(NULL, 0, format, ap);", "va_end(ap);", "char msgbuf[msglen + 1];", "va_start(ap, format);", "vsnprintf(msgbuf, msglen + 1, format, ap);", "get_ic_orig_error_at_line()(status, errnum, filename, linenum, \"%s\", msgbuf);"], msg_skip_fields=["status", "errnum", "filename", "linenum", "format"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) generate("void", ["herror", "perror"], "const char *s", tpl="write", before_lines=["int fd = safe_fileno(stderr);"], msg_skip_fields=["s"], msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) # Handle v?(err|warn)x? for v in ['', 'v']: for e in ['err', 'warn']: for x in ['', 'x']: func = v + e + x if e == 'err': sig_str = "int status, " names_str = "status, " tpl = "error" msg_skip_fields = ["status"] else: sig_str = '' names_str = '' tpl = "write" msg_skip_fields = [] sig_str += "const char *format, " names_str += "format, " msg_skip_fields += ["format"] if v: sig_str += "va_list ap" call_orig_lines = None msg_skip_fields += ["ap"] else: sig_str += "..." call_orig_lines = ["get_ic_orig_v" + e + x + "()(" + names_str + "ap);"] generate("void", func, sig_str, tpl=tpl, before_lines=["int fd = safe_fileno(stderr);"], call_orig_lines=call_orig_lines, msg_skip_fields=msg_skip_fields, msg_add_fields=["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"]) # Handle (__)?v?[fd]?w?printf(_chk)? for v in ['', 'v']: for f in ['', 'f', 'd']: for w in ['', 'w']: for (c1, c2) in [('', ''), ('__', '_chk')]: if f == 'd' and w == 'w': # No [v]dwprintf() in glibc continue func = c1 + v + f + w + "printf" + c2 if f == '': sig_str = '' names_str = '' before_lines = ["int fd = safe_fileno(stdout);"] msg_skip_fields = [] msg_add_fields = ["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"] elif f == 'f': sig_str = "FILE *stream, " names_str = "stream, " before_lines = ["int fd = safe_fileno(stream);"] send_msg_condition="fd != -1", msg_skip_fields = ["stream"] msg_add_fields = ["fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd);"] elif f == 'd': sig_str = "int fd, " names_str = "fd, " before_lines = [] msg_skip_fields = [] msg_add_fields = [] if c1: sig_str += "int fortify_flag, " names_str += "fortify_flag, " msg_skip_fields += ["fortify_flag"] if w: sig_str += "const wchar_t *format, " else: sig_str += "const char *format, " names_str += "format, " msg_skip_fields += ["format"] if v: sig_str += "FB_VA_LIST ap" call_orig_lines = None msg_skip_fields += ["ap"] else: sig_str += "..." call_orig_lines = ["ret = get_ic_orig_" + c1 + "v" + f + w + "printf" + c2 + "()(" + names_str + "ap);"] generate("int", func, sig_str, tpl="write", before_lines=before_lines, call_orig_lines=call_orig_lines, msg_skip_fields=msg_skip_fields, msg_add_fields=msg_add_fields) for v in ['', 'v']: for w in ['', 'w', 'n']: for (c1, c2) in [('', ''), ('__', '_chk')]: skip(c1 + v + "s" + w + "printf" + c2) for v in ['', 'v']: for (c1, c2) in [('', ''), ('__', '_chk')]: skip(c1 + v + "asprintf" + c2) for func_guard in [(["send", "SYS_send"], None), ("__send", "#if !defined __aarch64__")]: (func, ifdef_guard) = func_guard generate("ssize_t", func, "int fd, const void *buf, size_t len, int flags", ifdef_guard=ifdef_guard, tpl="write", msg_skip_fields=["buf", "len", "flags"]) generate("ssize_t", ["sendto", "SYS_sendto"], "int fd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen", tpl="write", msg_skip_fields=["buf", "len", "flags", "dest_addr", "addrlen"]) generate("ssize_t", ["sendmsg", "SYS_sendmsg", "__sendmsg64"], "int fd, const struct msghdr *msg, int flags", tpl="write", msg_skip_fields=["msg", "flags"]) generate("int", ["sendmmsg", "SYS_sendmmsg", "__sendmmsg", "__sendmmsg64"], "int fd, struct mmsghdr *msgvec, unsigned int vlen, int flags", platforms=['linux'], tpl="write", msg_skip_fields=["msgvec", "vlen", "flags"]) # TODO(rbalint) report as read and write without disabling shortcutting generate("ssize_t", ["sendfile64", "SYS_sendfile64"], "int out_fd, int in_fd, off64_t *offset, size_t count", tpl="copy_file_range") generate("ssize_t", ["sendfile", "SYS_sendfile"], "int out_fd, int in_fd, off_t *offset, size_t count", platforms=['linux'], tpl="copy_file_range") generate("int", ["sendfile", "SYS_sendfile"], "int fd, int s, off_t offset, off_t *len, struct sf_hdtr *hdtr, int flags", platforms=['darwin'], tpl="copy_file_range") generate("ssize_t", ["copy_file_range", "SYS_copy_file_range"], "int fd_in, loff_t *off_in, int fd_out, loff_t *off_out, size_t len, unsigned int flags", platforms=['linux'], tpl="copy_file_range") # Intercept querying or modifying the file offset # FIXME What's up with llseek, SYS_llseek, _llseek, __lseek, _IO_*seek* etc.? generate("off_t", ["lseek", "SYS_lseek", "__lseek"], "int fd, off_t offset, int whence", tpl="seek", modify_offset="offset != 0 || whence != SEEK_CUR", msg_skip_fields=["offset", "whence"]) generate("off64_t", "lseek64", "int fd, off64_t offset, int whence", tpl="seek", modify_offset="offset != 0 || whence != SEEK_CUR", msg_skip_fields=["offset", "whence"]) generate("int", "fseek", "FILE *stream, long offset, int whence", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="offset != 0 || whence != SEEK_CUR", msg_skip_fields=["stream", "offset", "whence"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("int", "fseeko", "FILE *stream, off_t offset, int whence", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="offset != 0 || whence != SEEK_CUR", msg_skip_fields=["stream", "offset", "whence"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) for func_guard in [("fseeko64", None), ("__fseeko64", glibc_ge(2, 28))]: (func, ifdef_guard) = func_guard generate("int", func, "FILE *stream, off64_t offset, int whence", ifdef_guard=ifdef_guard, tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="offset != 0 || whence != SEEK_CUR", msg_skip_fields=["stream", "offset", "whence"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("long", "ftell", "FILE *stream", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="false", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("off_t", "ftello", "FILE *stream", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="false", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) for func_guard in [("ftello64", None), ("__ftello64", glibc_ge(2, 28))]: (func, ifdef_guard) = func_guard generate("off64_t", func, "FILE *stream", ifdef_guard=ifdef_guard, tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="false", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("int", "fgetpos", "FILE *stream, fpos_t *pos", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="false", msg_skip_fields=["stream", "pos"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("int", "fgetpos64", "FILE *stream, fpos64_t *pos", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="false", msg_skip_fields=["stream", "pos"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("int", "fsetpos", "FILE *stream, const fpos_t *pos", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="true", msg_skip_fields=["stream", "pos"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("int", "fsetpos64", "FILE *stream, const fpos64_t *pos", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="true", msg_skip_fields=["stream", "pos"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) generate("void", "rewind", "FILE *stream", tpl="seek", before_lines=["int fd = safe_fileno(stream);"], modify_offset="true", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_seek_in_inherited_set_fd(&ic_msg, fd);"]) # Intercept chdir and fchdir generate("int", ["chdir", "SYS_chdir"], "const char *pathname", msg_skip_fields=["pathname"], # Don't make pathname absolute, since the macro would calculate it after calling chdir msg_add_fields=["BUILDER_SET_CANONICAL(chdir, pathname);", "if (success) {", " char* getcwd_ret = get_ic_orig_getcwd()(ic_cwd, FB_PATH_BUFSIZE);", " (void) getcwd_ret;", " ic_cwd_len = strlen(ic_cwd);", "}"]) generate("int", ["fchdir", "SYS_fchdir"], "int fd", after_lines=["if (success) {", " char* getcwd_ret = get_ic_orig_getcwd()(ic_cwd, FB_PATH_BUFSIZE);", " (void) getcwd_ret;", " ic_cwd_len = strlen(ic_cwd);", "}"]) # Intercept the chmod family generate("int", ["chmod", "SYS_chmod"], "const char *pathname, mode_t mode", msg="fchmodat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fchmodat, pathname);"]) generate("int", "lchmod", "const char *pathname, mode_t mode", msg="fchmodat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fchmodat, pathname);", "fbbcomm_builder_fchmodat_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);"]) generate("int", "fchmodat", "int fd, const char *pathname, mode_t mode, int flags", msg="fchmodat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fchmodat, fd, pathname);"]) # Note: SYS_fchmodat doesn't have a flags parameter generate("int", "SYS_fchmodat", "int fd, const char *pathname, mode_t mode", msg="fchmodat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fchmodat, fd, pathname);", "fbbcomm_builder_fchmodat_set_flags(&ic_msg, 0);"]) generate("int", ["fchmod", "SYS_fchmod"], "int fd, mode_t mode", msg="fchmodat") # Intercept the chown family # FIXME SYS_chown32, SYS_lchown32, SYS_fchown32 generate("int", ["chown", "SYS_chown"], "const char *pathname, uid_t owner, gid_t group", msg="fchownat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fchownat, pathname);"]) generate("int", ["lchown", "SYS_lchown"], "const char *pathname, uid_t owner, gid_t group", msg="fchownat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fchownat, pathname);", "fbbcomm_builder_fchownat_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);"]) generate("int", ["fchownat", "SYS_fchownat"], "int fd, const char *pathname, uid_t owner, gid_t group, int flags", msg="fchownat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fchownat, fd, pathname);"]) generate("int", ["fchown", "SYS_fchown"], "int fd, uid_t owner, gid_t group", msg="fchownat") # Intercept the link, symlink and readlink families generate("int", ["link", "SYS_link"], "const char *oldpath, const char *newpath", msg_skip_fields = ["oldpath", "newpath"], msg_add_fields = ["BUILDER_SET_ABSOLUTE_CANONICAL(link, oldpath);", "BUILDER_SET_ABSOLUTE_CANONICAL(link, newpath);"]) generate("int", ["linkat", "SYS_linkat"], "int olddirfd, const char *oldpath, int newdirfd, const char *newpath, int flags", msg_skip_fields = ["oldpath", "newpath"], msg_add_fields = ["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(link, olddirfd, oldpath);", "BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(link, newdirfd, newpath);"], msg="link") generate("int", ["symlink", "SYS_symlink"], "const char *target, const char *newpath", msg_skip_fields = ["newpath"], msg_add_fields = ["BUILDER_SET_ABSOLUTE_CANONICAL(symlink, newpath);"]) generate("int", ["symlinkat", "SYS_symlinkat"], "const char *target, int newdirfd, const char *newpath", msg_skip_fields = ["newpath"], msg_add_fields = ["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(symlink, newdirfd, newpath);"], msg="symlink") generate("ssize_t", ["readlink", "SYS_readlink"], "const char *pathname, char *buf, size_t bufsiz", tpl="readlink", msg_skip_fields=["pathname", "buf"], msg_add_fields = ["BUILDER_SET_ABSOLUTE_CANONICAL(readlink, pathname);"]) generate("ssize_t", "__readlink_chk", "const char *pathname, char *buf, size_t bufsiz, size_t fortify_size", tpl="readlink", msg="readlink", msg_skip_fields=["pathname", "buf", "fortify_size"], msg_add_fields = ["BUILDER_SET_ABSOLUTE_CANONICAL(readlink, pathname);"]) generate("ssize_t", ["readlinkat", "SYS_readlinkat"], "int dirfd, const char *pathname, char *buf, size_t bufsiz", tpl="readlink", msg="readlink", msg_skip_fields=["pathname", "buf"], msg_add_fields = ["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(readlink, dirfd, pathname);"]) generate("ssize_t", "__readlinkat_chk", "int dirfd, const char *pathname, char *buf, size_t bufsiz, size_t fortify_size", tpl="readlink", msg="readlink", msg_skip_fields=["pathname", "buf", "fortify_size"], msg_add_fields = ["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(readlink, dirfd, pathname);"]) # FIXME realpath # Intercept the stat family # FIXME SYS_oldstat, SYS_oldlstat, SYS_oldfstat, SYS_osf_*stat* # mysterious hacks prior to glibc 2.33 generate("int", "__xstat", "int ver, const char *pathname, struct stat *stat_buf", msg="fstatat", msg_skip_fields=["ver", "pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__xstat64", "int ver, const char *pathname, struct stat64 *stat_buf", platforms=['linux'], msg="fstatat", msg_skip_fields=["ver", "pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__lxstat", "int ver, const char *pathname, struct stat *stat_buf", msg="fstatat", msg_skip_fields=["ver", "pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "fbbcomm_builder_fstatat_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__lxstat64", "int ver, const char *pathname, struct stat64 *stat_buf", platforms=['linux'], msg="fstatat", msg_skip_fields=["ver", "pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "fbbcomm_builder_fstatat_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__fxstat", "int ver, int fd, struct stat *stat_buf", msg="fstatat", msg_skip_fields=["ver", "stat_buf"], msg_add_fields=["if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__fxstat64", "int ver, int fd, struct stat64 *stat_buf", platforms=['linux'], msg="fstatat", msg_skip_fields=["ver", "stat_buf"], msg_add_fields=["if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__fxstatat", "int ver, int fd, const char *pathname, struct stat *stat_buf, int flags", msg="fstatat", msg_skip_fields=["ver", "pathname", "stat_buf"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fstatat, fd, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "__fxstatat64", "int ver, int fd, const char *pathname, struct stat64 *stat_buf, int flags", platforms=['linux'], msg="fstatat", msg_skip_fields=["ver", "pathname", "stat_buf"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fstatat, fd, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) # cleaned up beginning with glibc 2.33 generate("int", "stat", "const char *pathname, struct stat *stat_buf", msg="fstatat", msg_skip_fields=["pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", ["stat64", "SYS_stat64", "__stat64_time64"], "const char *pathname, struct stat64 *stat_buf", msg="fstatat", msg_skip_fields=["pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", "lstat", "const char *pathname, struct stat *stat_buf", msg="fstatat", msg_skip_fields=["pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "fbbcomm_builder_fstatat_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", ["lstat64", "SYS_lstat64", "__lstat64_time64"], "const char *pathname, struct stat64 *stat_buf", msg="fstatat", msg_skip_fields=["pathname", "stat_buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(fstatat, pathname);", "fbbcomm_builder_fstatat_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", ["fstat", "SYS_newfstat"], "int fd, struct stat *stat_buf", msg="fstatat", msg_skip_fields=["stat_buf"], msg_add_fields=["if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) for func_guard in [(["fstat64", "SYS_fstat64", "__fstat64_time64"], None), ("__fstat64", glibc_ge(2, 33))]: (func, ifdef_guard) = func_guard generate("int", func, "int fd, struct stat64 *stat_buf", ifdef_guard=ifdef_guard, msg="fstatat", msg_skip_fields=["stat_buf"], msg_add_fields=["if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", ["fstatat", "SYS_newfstatat"], "int fd, const char *pathname, struct stat *stat_buf, int flags", msg="fstatat", msg_skip_fields=["pathname", "stat_buf"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fstatat, fd, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) generate("int", ["fstatat64", "SYS_fstatat64", "__fstatat64_time64"], "int fd, const char *pathname, struct stat64 *stat_buf, int flags", msg="fstatat", msg_skip_fields=["pathname", "stat_buf"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fstatat, fd, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, stat_buf->st_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, stat_buf->st_size);", "}"]) # since glibc 2.28 for func_guard in [("SYS_statx", None), ("statx", glibc_ge(2, 28))]: (func, ifdef_guard) = func_guard generate("int", func, "int fd, const char *pathname, int flags, unsigned int mask, struct statx *statx_buf", ifdef_guard=ifdef_guard, # TODO(rbalint) have separate statx() message with mask also sent msg="fstatat", # Always query type and mode to let the supervisor have valid stx_mode before_lines=["mask |= STATX_TYPE | STATX_MODE | STATX_SIZE;"], msg_skip_fields=["pathname", "mask", "statx_buf"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(fstatat, fd, pathname);", "if (success) {", " fbbcomm_builder_fstatat_set_st_mode(&ic_msg, statx_buf->stx_mode);", " fbbcomm_builder_fstatat_set_st_size(&ic_msg, statx_buf->stx_size);", # TODO(rbalint) handle when type and mode are not returned (e.g. because they are not requested by the intercepted application, just by the interceptor modifying the mask) "}"]) # Treat isfdtype() as a fstat() generate("int", "isfdtype", "int fd, int fdtype", platforms=['linux'], msg="fstatat", msg_skip_fields=["fdtype"]) # Disable shortcutting on the statfs family # FIXME SYS_osf_*statfs* generate("int", ["ustat", "SYS_ustat"], "dev_t dev, struct ustat *ubuf", platforms=['linux'], msg="statfs", msg_skip_fields=["dev", "ubuf"]) # Note: SYS_statfs64 is like statfs64() but also takes a size parameter. generate("int", "SYS_statfs64", "const char *pathname, size_t sz, struct statfs64 *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["pathname", "sz", "buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(statfs, pathname);"]) generate("int", ["statfs", "SYS_statfs", "__statfs"], "const char *pathname, struct statfs *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["pathname", "buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(statfs, pathname);"]) generate("int", "statfs64", "const char *pathname, struct statfs64 *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["pathname", "buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(statfs, pathname);"]) # Note: SYS_fstatfs64 is like fstatfs64() but also takes a size parameter. generate("int", "SYS_fstatfs64", "int fd, size_t sz, struct statfs64 *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["fd", "sz", "buf"]) generate("int", ["fstatfs", "SYS_fstatfs"], "int fd, struct statfs *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["fd", "buf"]) generate("int", "fstatfs64", "int fd, struct statfs64 *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["fd", "buf"]) generate("int", "statvfs", "const char *pathname, struct statvfs *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["pathname", "buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(statfs, pathname);"]) generate("int", "statvfs64", "const char *pathname, struct statvfs64 *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["pathname", "buf"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(statfs, pathname);"]) generate("int", "fstatvfs", "int fd, struct statvfs *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["fd", "buf"]) generate("int", "fstatvfs64", "int fd, struct statvfs64 *buf", platforms=['linux'], msg="statfs", msg_skip_fields=["fd", "buf"]) # Intercept lockf */ generate("int", "lockf", "int fd, int cmd, off_t len") generate("int", "lockf64", "int fd, int cmd, off64_t len", platforms=['linux'], msg="lockf") # Intercept fcntl and fcntl64 # The return value is selectively set from tpl_fcntl.c for certain commands. for func_guard in [(["fcntl", "SYS_fcntl64", "SYS_fcntl", "__fcntl", "__fcntl_time64"], None), ("fcntl64", glibc_ge(2, 28))]: (func, ifdef_guard) = func_guard generate("int", func, "int fd, int cmd, ...", ifdef_guard=ifdef_guard, msg="fcntl", tpl="fcntl") # Intercept ioctl - cmd is long in glibc, int in the Linux kernel generate("int", ["ioctl", "__ioctl_time64"], "int fd, unsigned long cmd, ...", tpl="ioctl", send_ret_on_success=True) generate("int", "SYS_ioctl", "int fd, unsigned int cmd, ...", tpl="ioctl", msg="ioctl", send_ret_on_success=True) # Intercept the dup family generate("int", ["dup", "SYS_dup"], "int oldfd", after_lines=["if (i_am_intercepting && success) copy_notify_on_read_write_state(ret, oldfd);"], send_ret_on_success=True) generate("int", ["dup2", "SYS_dup2", "__dup2"], "int oldfd, int newfd", tpl="dup2", msg="dup3") generate("int", ["dup3", "SYS_dup3"], "int oldfd, int newfd, int flags", platforms=['linux'], tpl="dup2", msg="dup3") # Intercept access variants generate("int", ["access", "SYS_access"], "const char *pathname, int mode", msg="faccessat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(faccessat, pathname);"]) generate("int", ["euidaccess", "eaccess"], "const char *pathname, int mode", platforms=['linux'], msg="faccessat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(faccessat, pathname);", "fbbcomm_builder_faccessat_set_flags(&ic_msg, AT_EACCESS);"]) # SYS_faccessat doesn't take a flags parameter. # glibc's faccessat() corresponds to SYS_faccessat2 which takes an additional flags parameter. generate("int", "SYS_faccessat", "int dirfd, const char *pathname, int mode", msg="faccessat", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(faccessat, dirfd, pathname);", "fbbcomm_builder_faccessat_set_flags(&ic_msg, 0);"]) generate("int", ["faccessat", "SYS_faccessat2"], "int dirfd, const char *pathname, int mode, int flags", msg_skip_fields=["pathname"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(faccessat, dirfd, pathname);"]) # Intercept the (l)utime family # FIXME SYS_osf_utimes, SYS_utimensat_time{32,64}, SYS_futimesat_time32 generate("int", ["utime", "SYS_utime", "__utime64"], "const char *pathname, const struct utimbuf *times", msg_skip_fields=["pathname", "times"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(utime, pathname);", "fbbcomm_builder_utime_set_all_utime_now(&ic_msg, times == NULL);"]) generate("int", ["utimes", "SYS_utimes", "__utimes64"], "const char *pathname, const struct timeval " + ("times[2]" if target == 'linux' else "*times"), msg="utime", msg_skip_fields=["pathname", "times"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(utime, pathname);", "fbbcomm_builder_utime_set_all_utime_now(&ic_msg, times == NULL);"]) generate("int", ["lutimes", "__lutimes64"], "const char *pathname, const struct timeval " + ("times[2]" if target == 'linux' else "*times"), msg="utime", msg_skip_fields=["pathname", "times"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(utime, pathname);", "fbbcomm_builder_utime_set_all_utime_now(&ic_msg, times == NULL);", "fbbcomm_builder_utime_set_flags(&ic_msg, AT_SYMLINK_NOFOLLOW);"]) generate("int", ["utimensat", "__utimensat64"], "int dirfd, const char *pathname, const struct timespec times[2], int flags", msg="utime", msg_skip_fields=["pathname", "times"], msg_add_fields=["BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(utime, dirfd, pathname);", "fbbcomm_builder_utime_set_all_utime_now(&ic_msg, times == NULL || (times[0].tv_nsec == UTIME_NOW && times[1].tv_nsec == UTIME_NOW));"]) generate("int", "SYS_utimensat", "int dirfd, const char *pathname, const struct timespec times[2], int flags", msg="utime", msg_skip_fields=["pathname", "times"], msg_add_fields=["if (pathname) BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(utime, dirfd, pathname);", "fbbcomm_builder_utime_set_all_utime_now(&ic_msg, times == NULL || (times[0].tv_nsec == UTIME_NOW && times[1].tv_nsec == UTIME_NOW));"]) generate("int", ["futimesat", "SYS_futimesat", "__futimesat64"], "int dirfd, const char *pathname, const struct timeval times[2]", platforms=['linux'], msg="utime", msg_skip_fields=["pathname", "times"], msg_add_fields=["if (pathname) BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(utime, dirfd, pathname);", "fbbcomm_builder_utime_set_all_utime_now(&ic_msg, times == NULL);"]) # Intercept the futime family generate("int", ["futimes", "__futimes64"], "int fd, const struct timeval " + ("times[2]" if target == 'linux' else "*times"), msg="futime", msg_skip_fields=["times"], msg_add_fields=["fbbcomm_builder_futime_set_all_utime_now(&ic_msg, times == NULL);"]) generate("int", ["futimens", "__futimens64"], "int fd, const struct timespec times[2]", msg="futime", msg_skip_fields=["times"], msg_add_fields=["fbbcomm_builder_futime_set_all_utime_now(&ic_msg, times == NULL || (times[0].tv_nsec == UTIME_NOW && times[1].tv_nsec == UTIME_NOW));"]) # Intercept dlopen # The new library's constructor may generate intercepted calls. # Also note that dlopen() does not set errno, and we always want to notify the supervisor, # in send_msg_condition we have to skip the default exception on EINTR and EFAULT. generate("void *", "dlopen", "const char *filename, int flag", tpl="dlopen", before_lines=["thread_intercept_on = NULL;"], after_lines=["thread_intercept_on = \"dlopen\";"], send_msg_condition="true", msg_skip_fields=["error_no"]) generate("void *", "dlmopen", "Lmid_t lmid, const char *filename, int flag", platforms=['linux'], tpl="dlopen", msg="dlopen", before_lines=["thread_intercept_on = NULL;"], after_lines=["thread_intercept_on = \"dlmopen\";"], send_msg_condition="true", msg_skip_fields=["lmid", "error_no"]) # Intercept the pipe family generate("int", ["pipe", "SYS_pipe", "__pipe"], "int pipefd[2]", tpl="pipe", before_lines=["int flags = 0;"], after_lines=["if (i_am_intercepting && success) { clear_notify_on_read_write_state(pipefd[0]); clear_notify_on_read_write_state(pipefd[1]); }"], msg_skip_fields=["pipefd"], msg_add_fields=["if (success) {", " fbbcomm_builder_pipe_fds_set_fd0(&ic_msg, pipefd[0]);", " fbbcomm_builder_pipe_fds_set_fd1(&ic_msg, pipefd[1]);", "}"]) generate("int", ["pipe2", "SYS_pipe2"], "int pipefd[2], int flags", platforms=['linux'], tpl="pipe", after_lines=["if (i_am_intercepting && success) { clear_notify_on_read_write_state(pipefd[0]); clear_notify_on_read_write_state(pipefd[1]); }"], msg_skip_fields=["pipefd"], msg_add_fields=["if (success) {", " fbbcomm_builder_pipe_fds_set_fd0(&ic_msg, pipefd[0]);", " fbbcomm_builder_pipe_fds_set_fd1(&ic_msg, pipefd[1]);", "}"]) # Intercept the truncate family generate("int", ["truncate64", "SYS_truncate64"], "const char *pathname, off64_t len", before_lines=["if (i_am_intercepting) maybe_send_pre_open(AT_FDCWD, pathname, O_WRONLY | O_TRUNC);"], platforms=['linux'], msg="truncate", msg_skip_fields=["pathname", "len"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(truncate, pathname);"]) generate("int", ["truncate", "SYS_truncate"], "const char *pathname, off_t len", before_lines=["if (i_am_intercepting) maybe_send_pre_open(AT_FDCWD, pathname, O_WRONLY | O_TRUNC);"], msg_skip_fields=["pathname", "len"], msg_add_fields=["BUILDER_SET_ABSOLUTE_CANONICAL(truncate, pathname);"]) # For our purposes, ftruncate() is just like pwrite(): modifies the file's contents # in a way other than simply writing sequential data at the current offset. generate("int", ["ftruncate64", "SYS_ftruncate64"], "int fd, off64_t len", platforms=['linux'], tpl="write", is_pwrite="true", msg_skip_fields=["len"]) generate("int", ["ftruncate", "SYS_ftruncate"], "int fd, off_t len", tpl="write", is_pwrite="true", msg_skip_fields=["len"]) # Intercept the fallocate() family generate("int", "posix_fallocate", "int fd, off_t offset, off_t len", platforms=['linux'], is_pwrite="true", msg_skip_fields=["offset", "len"], tpl="write") generate("int", "posix_fallocate64", "int fd, off64_t offset, off64_t len", platforms=['linux'], is_pwrite="true", msg_skip_fields=["offset", "len"], tpl="write") generate("int", ["fallocate", "SYS_fallocate"], "int fd, int mode, off_t offset, off_t len", platforms=['linux'], is_pwrite="true", msg_skip_fields=["mode", "offset", "len"], tpl="write") generate("int", "fallocate64", "int fd, int mode, off64_t offset, off64_t len", platforms=['linux'], is_pwrite="true", msg_skip_fields=["mode", "offset", "len"], tpl="write") # Intercept the mkstemp family # Note: these update the pathname template in place. generate("int", ["mkstemp", "mkstemp64"], "char *pathname", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], send_msg_on_error=False, msg="open", msg_skip_fields=["pathname"], msg_add_fields=["fbbcomm_builder_open_set_flags(&ic_msg, O_RDWR | O_CREAT | O_EXCL);", "fbbcomm_builder_open_set_mode(&ic_msg, 0600);", "BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, false);" "fbbcomm_builder_open_set_tmp_file(&ic_msg, true);"], send_ret_on_success=True, ack_condition="true") generate("int", ["mkostemp", "mkostemp64"], "char *pathname, int flags", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], send_msg_on_error=False, msg="open", msg_skip_fields=["pathname", "flags"], msg_add_fields=["fbbcomm_builder_open_set_flags(&ic_msg, O_RDWR | O_CREAT | O_EXCL | (flags & (O_APPEND | O_CLOEXEC | O_SYNC)));", "fbbcomm_builder_open_set_mode(&ic_msg, 0600);", "BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, false);" "fbbcomm_builder_open_set_tmp_file(&ic_msg, true);"], send_ret_on_success=True, ack_condition="true") generate("int", ["mkstemps", "mkstemps64"], "char *pathname, int suffixlen", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], send_msg_on_error=False, msg="open", msg_skip_fields=["pathname", "suffixlen"], msg_add_fields=["fbbcomm_builder_open_set_flags(&ic_msg, O_RDWR | O_CREAT | O_EXCL);", "fbbcomm_builder_open_set_mode(&ic_msg, 0600);", "BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, false);" "fbbcomm_builder_open_set_tmp_file(&ic_msg, true);"], send_ret_on_success=True, ack_condition="true") generate("int", ["mkostemps", "mkostemps64"], "char *pathname, int suffixlen, int flags", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], send_msg_on_error=False, msg="open", msg_skip_fields=["pathname", "suffixlen", "flags"], msg_add_fields=["fbbcomm_builder_open_set_flags(&ic_msg, O_RDWR | O_CREAT | O_EXCL | (flags & (O_APPEND | O_CLOEXEC | O_SYNC)));", "fbbcomm_builder_open_set_mode(&ic_msg, 0600);", "BUILDER_SET_ABSOLUTE_CANONICAL(open, pathname);", "fbbcomm_builder_open_set_pre_open_sent(&ic_msg, false);" "fbbcomm_builder_open_set_tmp_file(&ic_msg, true);"], send_ret_on_success=True, ack_condition="true") generate("char *", "mkdtemp", "char *pathname", send_msg_on_error=False, msg="mkdir", msg_skip_fields=["pathname"], msg_add_fields=["fbbcomm_builder_mkdir_set_mode(&ic_msg, 0700);", "BUILDER_SET_ABSOLUTE_CANONICAL(mkdir, pathname);", "fbbcomm_builder_mkdir_set_tmp_dir(&ic_msg, true);"], ack_condition="true") # FIXME needs support on the supervisor for starting to track an fd without knowing the backing file generate("FILE *", ["tmpfile", "tmpfile64"], "", tpl="once") # "Never use these functions" # FIXME Supporting them would require supervisor work, or two messages (exclusive open, then close) generate("char *", ["mktemp", "tmpnam", "__mktemp"], "char *tmplt", tpl="once", diagnostic_ignored = ["-Warray-parameter="]) generate("char *", "tmpnam_r", "char *tmplt", platforms=['linux'], tpl="once", diagnostic_ignored = ["-Warray-parameter="]) generate("char *", ["tempnam"], "const char *dir, const char *pfx", tpl="once") # Disable shortcutting on chroot generate("int", ["chroot", "SYS_chroot"], "const char *path", tpl="once") # Skip bridge between low and high level io skip("fileno", "dirfd") # Skip directory reading skip("readdir", "readdir64", "readdir_r", "readdir64_r", "SYS_readdir", "rewinddir", "seekdir", "telldir") skip("getdents", "SYS_getdents", "getdents64", "SYS_getdents64", "getdirentries", "getdirentries64") # FIXME implement scandir(at)(64) # Disable shortcutting on mknod # mysterious hacks prior to glibc 2.33 (also note the pointer "dev_t *") generate("int", "__xmknod", "int ver, const char *path, mode_t mode, dev_t *dev", tpl="once") generate("int", "__xmknodat", "int ver, int dirfd, const char *path, mode_t mode, dev_t *dev", tpl="once") # cleaned up beginning with glibc 2.33 generate("int", ["mknod", "SYS_mknod"], "const char *path, mode_t mode, dev_t dev", tpl="once") generate("int", ["mknodat", "SYS_mknodat"], "int dirfd, const char *path, mode_t mode, dev_t dev", platforms=['linux'], tpl="once") generate("int", ["mknodat", "SYS_mknodat"], "int dirfd, const char *path, mode_t mode, dev_t dev", platforms=['darwin'], ifdef_guard=apple_ge(13, 0), tpl="once") # Disable shortcutting on mkfifo generate("int", "mkfifo", "const char *pathname, mode_t mode", tpl="once") generate("int", "mkfifoat", "int dirfd, const char *pathname, mode_t mode", platforms=['linux'], tpl="once") # Intercept the exit family. # - _exit() and _Exit() terminate the process immediately, no exit handlers are run. # So we need to modify the supervisor about the exit code and resource usage before execuing it. # - exit() calls the atexit() / on_exit() handlers first. We'll notify the supervisor using the # hopefully last on_exit() handler, which is our on_exit_handler() method. # - quick_exit() calls the at_quick_exit() handlers, which, similarly to the atexit() handlers, # do not receive the exit status. So notify the supervisor before calling this method, just like # for _exit(). generate("void", ["exit"], "int status", tpl="exit", no_saved_errno=True) generate("void", ["_exit", "_Exit", "SYS_exit"], "int status", tpl="_exit", no_saved_errno=True) generate("void", "quick_exit", "int status", platforms=['linux'], tpl="_exit", no_saved_errno=True) # No wrapper for exit_group() in glibc 2.30. skip("exit_group") generate("void", "SYS_exit_group", "int status", tpl="_exit") # Intercept fork() and variants generate("pid_t", ["fork", "SYS_fork", "__fork"], "", tpl="fork") generate("pid_t", ["_Fork"], "", platforms=['linux'], tpl="_fork") generate("pid_t", ["vfork", "SYS_vfork", "__vfork"], "", ifdef_guard=glibc_ge(2, 34), tpl="_fork") # NOTE: Mapping to fork() breaks the ABI since the fork handlers were not supposed to run # OTOH the fork handlers notify the supervisor about the call. generate("pid_t", ["vfork", "SYS_vfork", "__vfork"], "", ifdef_guard=not_glibc_ge(2, 34), tpl="fork") # Intercept system generate("int", "system", "const char *cmd", tpl="system") # Intercept popen and pclose generate("FILE *", "popen", "const char *cmd, const char *type", tpl="popen") generate("int", "pclose", "FILE *stream", tpl="pclose", # Maybe we pclose() an fopen()ed file, in that case send the "close" message (from tpl_pclose.c) # but don't send the "pclose" message. See #642 for more info. send_msg_condition="was_popened && (success || (errno != EINTR && errno != EFAULT))", msg_skip_fields=["stream"], msg_add_fields=["fbbcomm_builder_pclose_set_fd(&ic_msg, fd);"], send_ret_on_success=True, ack_condition="true") # Intercept the posix_spawn family generate("int", "posix_spawn", "pid_t *pid, const char *file, " + "const posix_spawn_file_actions_t *file_actions, " + "const posix_spawnattr_t *attrp, " + "char *const argv[], char *const envp[]", tpl="posix_spawn", success="ret == 0") generate("int", "posix_spawnp", "pid_t *pid, const char *file, " + "const posix_spawn_file_actions_t *file_actions, " + "const posix_spawnattr_t *attrp, " + "char *const argv[], char *const envp[]", tpl="posix_spawn", success="ret == 0") generate("int", ["posix_spawn_file_actions_init", "posix_spawn_file_actions_destroy"], "posix_spawn_file_actions_t *file_actions", tpl="posix_spawn_file_actions", success="ret == 0") generate("int", "posix_spawn_file_actions_addopen", "posix_spawn_file_actions_t *file_actions, int fd, const char *pathname, int flags, mode_t mode", tpl="posix_spawn_file_actions", success="ret == 0") generate("int", "posix_spawn_file_actions_addclose", "posix_spawn_file_actions_t *file_actions, int fd", tpl="posix_spawn_file_actions", success="ret == 0") generate("int", "posix_spawn_file_actions_addclosefrom_np", "posix_spawn_file_actions_t *file_actions, int lowfd", platforms=['linux'], tpl="posix_spawn_file_actions", success="ret == 0") generate("int", "posix_spawn_file_actions_adddup2", "posix_spawn_file_actions_t *file_actions, int oldfd, int newfd", tpl="posix_spawn_file_actions", success="ret == 0") generate("int", "posix_spawn_file_actions_addchdir_np", "posix_spawn_file_actions_t *file_actions, const char *pathname", ifdef_guard=glibc_ge(2, 29), tpl="posix_spawn_file_actions", success="ret == 0") generate("int", "posix_spawn_file_actions_addfchdir_np", "posix_spawn_file_actions_t *file_actions, int fd", ifdef_guard=glibc_ge(2, 29), tpl="posix_spawn_file_actions", success="ret == 0") # Insert a trace marker for clone and pthread_create generate("int", ["clone", "__clone"], "int (*fn)(void *), void *stack, int flags, void *arg, ...", platforms=['linux'], tpl="clone") # Don't bother spelling out SYS_clone's parameters, it's heavily architecture dependent generate("int", "SYS_clone", "...", platforms=['linux'], tpl="clone") generate("int", "pthread_create", "pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg", tpl="pthread_create") # Intercept the wait family # Need to wait for the supervisor to ACK that it has processed what the child had done # Lock only for the communication, see #338 # FIXME SYS_osf_old_wait, SYS_osf_wait4, SYS_osf_waitid generate("pid_t", ["wait", "__wait"], "int *wstatus", success="ret > 0", before_lines=["int wstatus_fallback;", "if (!wstatus) wstatus = &wstatus_fallback;"], global_lock='after', send_msg_on_error=False, msg="wait", msg_skip_fields=["wstatus"], msg_add_fields=["fbbcomm_builder_wait_set_pid(&ic_msg, ret);", "fbbcomm_builder_wait_set_wstatus(&ic_msg, *wstatus);"], ack_condition="true") generate("pid_t", ["waitpid", "SYS_waitpid", "__waitpid"], "pid_t pid, int *wstatus, int options", global_lock='after', before_lines=["int wstatus_fallback;", "if (!wstatus) wstatus = &wstatus_fallback;"], success="ret > 0", send_msg_on_error=False, send_msg_condition="success && !WIFSTOPPED(*wstatus) && !WIFCONTINUED(*wstatus)", msg="wait", msg_skip_fields=["pid", "wstatus", "options"], msg_add_fields=["fbbcomm_builder_wait_set_pid(&ic_msg, ret);", "fbbcomm_builder_wait_set_wstatus(&ic_msg, *wstatus);"], ack_condition="true") generate("pid_t", ["wait3", "__wait3_time64"], "int *wstatus, int options, struct rusage *rusage", global_lock='after', before_lines=["int wstatus_fallback;", "if (!wstatus) wstatus = &wstatus_fallback;"], success="ret > 0", send_msg_on_error=False, send_msg_condition="success && !WIFSTOPPED(*wstatus) && !WIFCONTINUED(*wstatus)", msg="wait", msg_skip_fields=["wstatus", "options", "rusage"], msg_add_fields=["fbbcomm_builder_wait_set_pid(&ic_msg, ret);", "fbbcomm_builder_wait_set_wstatus(&ic_msg, *wstatus);"], ack_condition="true") generate("pid_t", ["wait4", "SYS_wait4", "__wait4_time64"], "pid_t pid, int *wstatus, int options, struct rusage *rusage", global_lock='after', before_lines=["int wstatus_fallback;", "if (!wstatus) wstatus = &wstatus_fallback;"], success="ret > 0", send_msg_on_error=False, send_msg_condition="success && !WIFSTOPPED(*wstatus) && !WIFCONTINUED(*wstatus)", msg="wait", msg_skip_fields=["pid", "wstatus", "options", "rusage"], msg_add_fields=["fbbcomm_builder_wait_set_pid(&ic_msg, ret);", "fbbcomm_builder_wait_set_wstatus(&ic_msg, *wstatus);"], ack_condition="true") # Note: See BUGS in the waitid(2) manual page for the infop==NULL case. generate("int", "waitid", "idtype_t idtype, id_t id, siginfo_t *infop, int options", global_lock='after', before_lines=["siginfo_t infop_fallback;", "if (!infop) infop = &infop_fallback;"], send_msg_on_error=False, send_msg_condition="success && !((options & WNOHANG) && (infop->si_pid == 0)) && infop->si_code != CLD_STOPPED && infop->si_code != CLD_TRAPPED && infop->si_code != CLD_CONTINUED", msg="wait", msg_skip_fields=["idtype", "id", "infop", "options"], msg_add_fields=["fbbcomm_builder_wait_set_pid(&ic_msg, infop->si_pid);", "fbbcomm_builder_wait_set_si_code(&ic_msg, infop->si_code);", "fbbcomm_builder_wait_set_si_status(&ic_msg, infop->si_status);"], ack_condition="true") # Note: Takes one more parameter than the libc wrapper. generate("int", "SYS_waitid", "idtype_t idtype, id_t id, siginfo_t *infop, int options, struct rusage *usage", global_lock='after', before_lines=["siginfo_t infop_fallback;", "if (!infop) infop = &infop_fallback;"], send_msg_on_error=False, send_msg_condition="success && !((options & WNOHANG) && (infop->si_pid == 0)) && infop->si_code != CLD_STOPPED && infop->si_code != CLD_TRAPPED && infop->si_code != CLD_CONTINUED", msg="wait", msg_skip_fields=["idtype", "id", "infop", "options", "usage"], msg_add_fields=["fbbcomm_builder_wait_set_pid(&ic_msg, infop->si_pid);", "fbbcomm_builder_wait_set_si_code(&ic_msg, infop->si_code);", "fbbcomm_builder_wait_set_si_status(&ic_msg, infop->si_status);"], ack_condition="true") # Intercept the exec family # FIXME SYS_osf_execve generate("int", ["execl", "execlp"], "const char *file, const char *arg, ...", tpl="exec") # Note: execlpe exists on some systems, but not in glibc generate("int", "execle", "const char *file, const char *arg, ...", tpl="exec") generate("int", ["execv", "execvp"], "const char *file, char *const argv[]", tpl="exec") generate("int", ["execve", "SYS_execve"], "const char *file, char *const argv[], char *const envp[]", tpl="exec") generate("int", "execvpe", "const char *file, char *const argv[], char *const envp[]", platforms=['linux'], tpl="exec") # Note: execveat() appeared in glibc 2.34. generate("int", ["execveat", "SYS_execveat"], "int dirfd, const char *file, char *const argv[], char *const envp[], int flags", platforms=['linux'], tpl="exec") generate("int", "fexecve", "int fd, char *const argv[], char *const envp[]", platforms=['linux'], tpl="exec") # Signal handling # FIXME SYS_osf_signal, SYS_osf_old_sigsetmask, SYS_osf_old_sigaction generate("sighandler_t", ["signal", "sigset"], "int signum, " + ("sighandler_t" if target == 'linux' else 'sig_t') + " handler", tpl="signal", success="1") generate("long int", ["SYS_signal"], "int signum, sighandler_t handler", tpl="signal", success="1") generate("int", ["sigaction", "SYS_sigaction", "__sigaction"], "int signum, const struct sigaction *act, struct sigaction *oldact", tpl="signal") # futex() doesn't have a glibc wrapper, pthread_mutex_[un]lock() maps into syscall(SYS_futex, ...). # Don't need to notify the supervisor about these, stay out of the way as much as possible. generate("long", "SYS_futex", "uint32_t *uaddr, int futex_op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3", tpl="marker_only") # Skip the getcwd family: The initial working directory is part of the fingerprint skip("getcwd", "__getcwd_chk", "SYS_getcwd", "getwd", "__getwd_chk", "get_current_dir_name") # Time and alarm handling: These are fine, no action required # TODO(rbalint) those may affect output if the process measures time that way # usually the calls can be ignored skip("clock", "clock_getres", "SYS_clock_getres", "SYS_clock_getres_time64", "clock_getcpuclockid") skip("sleep", "usleep", "nanosleep", "SYS_nanosleep", "clock_nanosleep", "SYS_clock_nanosleep", "SYS_clock_nanosleep_time64") skip("pause", "SYS_pause", "alarm", "SYS_alarm", "ualarm", "getitimer", "SYS_getitimer", "setitimer", "SYS_setitimer") skip("timer_create", "SYS_timer_create", "timer_delete", "SYS_timer_delete", "timer_getoverrun", "SYS_timer_getoverrun") skip("timer_settime", "SYS_timer_settime", "SYS_timer_settime_time64", "timer_gettime", "SYS_timer_gettime", "SYS_timer_gettime_time64") # Getting high entropy randomness should disable shortcutting generate("ssize_t", ["getrandom", "SYS_getrandom"], "void* buf, size_t buflen, unsigned int flags", platforms=['linux'], msg_skip_fields = ["buf", "buflen"]) generate("int", "getentropy", "void* buffer, size_t length", msg="getrandom", msg_skip_fields = ["buffer", "length"], # implemented by glibc as getrandom(..., 0) msg_add_fields=["fbbcomm_builder_getrandom_set_flags(&ic_msg, 0);"]) generate("uint32_t", "arc4random", "", msg="getrandom", # implemented by glibc as getrandom(..., 0) msg_add_fields=["fbbcomm_builder_getrandom_set_flags(&ic_msg, 0);"], success="true", send_ret_on_success=False, no_saved_errno=True) generate("void", "arc4random_buf", "void* buf, size_t nbytes", msg="getrandom", msg_skip_fields=["buf", "nbytes"], # implemented by glibc as getrandom(..., 0) msg_add_fields=["fbbcomm_builder_getrandom_set_flags(&ic_msg, 0);"], success="true", send_ret_on_success=False, no_saved_errno=True) generate("uint32_t", "arc4random_uniform", "uint32_t upper_bound", msg="getrandom", msg_skip_fields = ["upper_bound"], # implemented by glibc as getrandom(..., 0) msg_add_fields=["fbbcomm_builder_getrandom_set_flags(&ic_msg, 0);"], success="true", send_ret_on_success=False, no_saved_errno=True) # Querying current time - probably disable shortcutting, let the supervisor decide generate("time_t", ["time", "SYS_time", "__time64"], "time_t *loc", msg="clock_gettime", tpl="once") generate("int", "ftime", "struct timeb *tp", msg="clock_gettime", tpl="once") # sys/time.h declares the funcion with void *tz and not matching that generates an error generate("int", ["gettimeofday", "__gettimeofday"], "struct timeval *tv, struct timezone *tz", platforms=['linux'], ifdef_guard=not_glibc_ge(2, 31), msg="clock_gettime", tpl="once") generate("int", ["gettimeofday", "__gettimeofday"], "struct timeval *tv, void *tz", ifdef_guard=glibc_ge(2, 31) + " || defined(__APPLE__)", msg="clock_gettime", tpl="once") generate("int", "__gettimeofday64", "struct timeval *tv, void *tz", platforms=['linux'], ifdef_guard=glibc_ge(2, 34), msg="clock_gettime", tpl="once") generate("int", "SYS_gettimeofday", "struct timeval *tv, void *tz", msg="clock_gettime", tpl="once") # FIXME SYS_clock_gettime64 generate("int", ["clock_gettime", "SYS_clock_gettime64", "SYS_clock_gettime", "__clock_gettime", "__clock_gettime64"], "clockid_t clk_id, struct timespec *tp", msg="clock_gettime", tpl="once") # FIXME SYS_osf_ntp_gettime generate("int", ["ntp_gettime", "ntp_gettimex", "__ntp_gettime64", "__ntp_gettimex64"], "struct ntptimeval *ntv", platforms=['linux'], msg="clock_gettime", tpl="once") # Disable setting the time generate("int", "stime", "const time_t *t", platforms=['linux'], tpl="once") generate("int", ["settimeofday", "SYS_settimeofday", "__settimeofday64"], "const struct timeval *tv, const struct timezone *tz", tpl="once") # FIXME SYS_clock_settime64 generate("int", ["clock_settime", "SYS_clock_settime64", "SYS_clock_settime", "__clock_settime64"], "clockid_t clk_id, const struct timespec *tp", tpl="once") # FIXME SYS_osf_adjtime, SYS_old_adjtimex generate("int", ["adjtime", "__adjtime64"], "const struct timeval *delta, struct timeval *olddelta", tpl="once") generate("int", ["adjtimex", "SYS_adjtimex", "__adjtimex"], "struct timex *buf", platforms=['linux'], tpl="once") # FIXME SYS_clock_adjtime64 generate("int", ["clock_adjtime", "SYS_clock_adjtime64", "SYS_clock_adjtime", "__clock_adjtime64"], "clockid_t clk_id, struct timex *buf", platforms=['linux'], tpl="once") # FIXME SYS_osf_ntp_adjtime, SYS_osf_utc_adjtime generate("int", "ntp_adjtime", "struct timex *buf", platforms=['linux'], tpl="once") # Skip scheduling parameters skip("nice", "SYS_nice", "getpriority", "SYS_getpriority", "setpriority", "SYS_setpriority") skip("getrusage", "SYS_getrusage", "profil", "SYS_profil", "acct", "SYS_acct") # Process stuff skip("prctl", "SYS_prctl") skip("getpid", "SYS_getpid", "getppid", "SYS_getppid", "gettid", "SYS_gettid") skip("getpgrp", "SYS_getpgrp", "getpgid", "__getpgid", "SYS_getpgid", "setpgrp", "SYS_setpgrp", "setpgid", "SYS_setpgid") skip("getsid", "SYS_getsid", "setsid", "SYS_setsid") skip("SYS_capget") generate("int", ["pidfd_open", "SYS_pidfd_open"], "pid_t pid, unsigned int flags", platforms=['linux'], tpl="once") # can be skipped, since pidfd_open already disables shortcutting skip("pidfd_getfd", "SYS_pidfd_getfd", "pidfd_send_signal", "SYS_pidfd_send_signal") generate("mode_t", ["umask", "SYS_umask"], "mode_t mask", success="true", send_ret_on_success=True, send_msg_on_error=False) # Note: getumask is gone since glibc 2.24 # Pretend that seccomp() is not implemented. # This allows interception of a few commands that otherwise would use seccomp sandboxing, such as man # and friends. generate("int", "SYS_seccomp", "unsigned int operation, unsigned int flags, void *args", msg_skip_fields=["operation", "flags", "args"], msg="seccomp", call_orig_lines=["if (i_am_intercepting) {", " (void)operation; (void)flags; (void)args;", " errno = EINVAL; ret = -1;" "} else {", " ret = ic_orig_SYS_seccomp(operation, flags, args);", "}", ]) # FIXME {get,set,p}rlimit # User and group stuff # FIXME shall we notify the supervisor? # FIXME SYS_*32 skip("getuid", "SYS_getuid", "getgid", "SYS_getgid", "geteuid", "SYS_geteuid", "getegid", "SYS_getegid") skip("getresuid", "SYS_getresuid", "getresgid", "SYS_getresgid") skip("getgroups", "SYS_getgroups", "group_member") generate("int", ["setuid", "SYS_setuid", "seteuid"], "uid_t uid", tpl="once") generate("int", ["setgid", "SYS_setgid", "setegid"], "gid_t gid", tpl="once") generate("int", ["setreuid", "SYS_setreuid"], "uid_t uid, uid_t euid", tpl="once") generate("int", ["setregid", "SYS_setregid"], "gid_t gid, gid_t egid", tpl="once") generate("int", ["setresuid", "SYS_setresuid"], "uid_t uid, uid_t euid, uid_t suid", platforms=['linux'], tpl="once") generate("int", ["setresgid", "SYS_setresgid"], "gid_t gid, gid_t egid, gid_t sgid", platforms=['linux'], tpl="once") generate("int", ["setgroups", "SYS_setgroups"], ("size_t" if target == 'linux' else "int") + " size, const gid_t *list", tpl="once") # Shells skip("getusershell", "endusershell", "setusershell") # FIXME What to do with the getpwent() family? # Tty stuff skip("ttyname", "ttyname_r", "ttyslot", "isatty", "vhangup", "SYS_vhangup", "getpass") skip("tcgetpgrp", "tcsetpgrp", "getlogin", "getlogin_r"," __getlogin_r_chk") skip("ctermid", "cuserid") generate("int", "setlogin", "const char *name", tpl="once") # Revoke - glibc function exists, no manpage. Does it work? skip("revoke") # System stuff skip("daemon") skip("getpagesize", "__getpagesize", "SYS_getpagesize") skip("getdtablesize", "SYS_getdtablesize") # Memory management skip("malloc", "calloc", "realloc", "free", "brk", "SYS_brk", "sbrk") # FIXME intercept mmap skip("mmap", "SYS_mmap", "SYS_mmap2", "mremap", "SYS_mremap", "munmap", "SYS_munmap") # Hostname stuff - report gethostname, getdomainname, gethostid. # The supervisor might decide whether to disable shortcutting, or ignore this difference generate("int", ["gethostname", "SYS_gethostname"], "char *name, size_t len") generate("int", "__gethostname_chk", "char *name, size_t len, size_t fortify_size", tpl="once") generate("int", "getdomainname", "char *name, " + ("size_t" if target == 'linux' else "int") + " len", tpl="once") generate("int", "__getdomainname_chk", "char *name, size_t len, size_t fortify_size", tpl="once") # FIXME SYS_osf_gethostid generate("long", "gethostid", "", tpl="once") # Disable shortcutting on sethostname, setdomainname, sethostid generate("int", ["sethostname", "SYS_sethostname"], "const char *name, " + ("size_t" if target == 'linux' else "int") + " len", tpl="once") generate("int", ["setdomainname", "SYS_setdomainname"], "const char *name, " + ("size_t" if target == 'linux' else "int") + " len", tpl="once") # FIXME SYS_osf_gethostid generate("int" if target == 'linux' else "void", "sethostid", "long hostid", tpl="once") generate("int", ["socket", "SYS_socket", "__socket"], "int domain, int type, int protocol", after_lines=["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"], send_ret_on_success=True) generate("int", ["connect", "SYS_connect", "__connect"], "int sockfd, const struct sockaddr *addr, socklen_t addrlen", tpl="once") generate("int", ["bind", "SYS_bind"], "int sockfd, const struct sockaddr *addr, socklen_t addrlen", tpl="once") generate("int", ["listen", "SYS_listen"], "int sockfd, int backlog", tpl="once") generate("int", ["socketpair", "SYS_socketpair"], "int domain, int type, int protocol, int sv[2]", after_lines=["if (i_am_intercepting && success) {", " clear_notify_on_read_write_state(sv[0]);", " clear_notify_on_read_write_state(sv[1]);", "}"], msg_skip_fields=["sv"], msg_add_fields=["if (success) {", " fbbcomm_builder_socketpair_set_fd0(&ic_msg, sv[0]);", " fbbcomm_builder_socketpair_set_fd1(&ic_msg, sv[1]);", "}"]) generate("int", "SYS_socketcall", "int call, unsigned long *args", tpl="once") # Intercept the pathconf family # FIXME Sometimes a retval == -1 does not denote error. Same for sysconf. # FIXME SYS_osf_[f]pathconf generate("long", "pathconf", "const char *path, int name", send_ret_on_success=True) generate("long", "fpathconf", "int fd, int name", send_ret_on_success=True) # Intercept sysconf # FIXME Sometimes a retval == -1 does not denote error. Same for pathconf. generate("long", ["sysconf", "__sysconf"], "int name", send_ret_on_success=True) # Intercept syscall # FIXME What does syscall(SYS_syscall, ...) do? :) generate("long" if target == 'linux' else "int", "syscall", ("long" if target == 'linux' else "int") + " number, ...", tpl="syscall", send_ret_on_success=True, ack_condition="true") # FIXME This is temporary only, until intercepting confstr is implemented: #generate("size_t", "confstr", "int name, char *buf, size_t len") #generate("size_t", "__confstr_chk", "int name, char *buf, size_t len, size_t fortify_size") skip("confstr", "__confstr_chk") # Network # FIXME use the "once" template instead: we need to know it can't be shortcut skip("getsockname", "SYS_getsockname", "getpeername", "SYS_getpeername") skip("getsockopt", "SYS_getsockopt", "setsockopt", "SYS_setsockopt") skip("accept", "SYS_accept", "accept4", "SYS_accept4", "shutdown", "SYS_shutdown", "sockatmark") # Mounts skip("setmntent", "getmntent", "getmntent_r", "addmntent", "endmntent", "hasmntopt") # FIXME Implement intercepting these: # mkdir(at) # freopen(64) - looks complicated # locale related ones: # (bind)textdomain, *gettext, {set,use,new}locale, nl_langinfo, iconv* # iswalpha, towlower and friends (the locale dependant ones) # strcasecmp # strerror*, strsignal, strftime # *_l # Finally rename the output files. for gen in set(libc_outputs + syscall_outputs): os.rename(outdir + "/gen_" + gen + tmpsuffix, outdir + "/gen_" + gen) firebuild-0.8.2/src/interceptor/ic_file_ops.c000066400000000000000000000133341447164520700212500ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #define _LARGEFILE64_SOURCE 1 #define _GNU_SOURCE #include "interceptor/ic_file_ops.h" #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #ifdef __linux__ #include #endif #include #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #include "interceptor/intercept.h" int intercept_fopen_mode_to_open_flags_helper(const char * mode) { int flags; const char * p = mode; /* invalid mode , NULL will crash fopen() anyway */ if (p == NULL) { return -1; } /* open() flags */ switch (*p) { case 'r': { p++; if (*p != '+') { flags = O_RDONLY; } else { p++; flags = O_RDWR; } break; } case 'w': { p++; if (*p != '+') { flags = O_WRONLY | O_CREAT | O_TRUNC; } else { p++; flags = O_RDWR | O_CREAT | O_TRUNC; } break; } case 'a': { p++; if (*p != '+') { flags = O_WRONLY | O_CREAT | O_APPEND; } else { p++; flags = O_RDWR | O_CREAT | O_APPEND; } break; } default: /* would cause EINVAL in open()*/ return -1; } /* glibc extensions */ while (*p != '\0') { switch (*p) { case 'b': /* ignore, not interesting from interception POV */ p++; continue; case 'c': { /* ignore, not interesting from interception POV */ p++; continue; } case 'e': { flags |= O_CLOEXEC; break; } case 'm': { /* ignore, not interesting from interception POV */ p++; continue; } case 'x': { flags |= O_EXCL; break; } case ',': { /* ,ccs=string is not interesting from intercepion POV */ return flags; } default: /* */ break; } p++; } return flags; } int popen_type_to_flags(const char * type) { int type_flags = 0; for (const char *c = type; c != NULL && *c != '\0'; c++) { switch (*c) { case 'w': { type_flags |= O_WRONLY; break; } case 'r': { type_flags |= O_RDONLY; break; } case 'e': { type_flags |= O_CLOEXEC; break; } default: /* Popen will return -1 due to the unknown type. */ break; } } return type_flags; } void clear_notify_on_read_write_state(const int fd) { if (fd >= 0 && fd < IC_FD_STATES_SIZE) { ic_fd_states[fd].notify_on_read = false; ic_fd_states[fd].notify_on_pread = false; ic_fd_states[fd].notify_on_write = false; ic_fd_states[fd].notify_on_pwrite = false; ic_fd_states[fd].notify_on_tell = false; ic_fd_states[fd].notify_on_seek = false; } } void set_notify_on_read_write_state(const int fd) { if (fd >= 0 && fd < IC_FD_STATES_SIZE) { ic_fd_states[fd].notify_on_read = true; ic_fd_states[fd].notify_on_pread = true; ic_fd_states[fd].notify_on_write = true; ic_fd_states[fd].notify_on_pwrite = true; ic_fd_states[fd].notify_on_tell = true; ic_fd_states[fd].notify_on_seek = true; } } void set_all_notify_on_read_write_states() { for (int fd = 0; fd < IC_FD_STATES_SIZE; fd++) { ic_fd_states[fd].notify_on_read = true; ic_fd_states[fd].notify_on_pread = true; ic_fd_states[fd].notify_on_write = true; ic_fd_states[fd].notify_on_pwrite = true; ic_fd_states[fd].notify_on_tell = true; ic_fd_states[fd].notify_on_seek = true; } } void copy_notify_on_read_write_state(const int to_fd, const int from_fd) { if ((to_fd >= 0) && (to_fd < IC_FD_STATES_SIZE) && (from_fd >= 0) && (from_fd < IC_FD_STATES_SIZE)) { ic_fd_states[to_fd] = ic_fd_states[from_fd]; } } void set_notify_on_read_state(const int fd, const bool is_pread) { if (fd >= 0 && fd < IC_FD_STATES_SIZE) { ic_fd_states[fd].notify_on_read = false; if (is_pread) { ic_fd_states[fd].notify_on_pread = false; } } } void set_notify_on_write_state(const int fd, const bool is_pwrite) { if (fd >= 0 && fd < IC_FD_STATES_SIZE) { ic_fd_states[fd].notify_on_write = false; if (is_pwrite) { ic_fd_states[fd].notify_on_pwrite = false; } } } bool notify_on_read(const int fd, const bool is_pread) { return (fd < 0 || fd >= IC_FD_STATES_SIZE || (is_pread == false && ic_fd_states[fd].notify_on_read == true) || (is_pread == true && ic_fd_states[fd].notify_on_pread == true)); } bool notify_on_write(const int fd, const bool is_pwrite) { return (fd < 0 || fd >= IC_FD_STATES_SIZE || (is_pwrite == false && ic_fd_states[fd].notify_on_write == true) || (is_pwrite == true && ic_fd_states[fd].notify_on_pwrite == true)); } firebuild-0.8.2/src/interceptor/ic_file_ops.h000066400000000000000000000042231447164520700212520ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef FIREBUILD_IC_FILE_OPS_H_ #define FIREBUILD_IC_FILE_OPS_H_ #ifdef __linux__ #include #endif #include #include #include "interceptor/intercept.h" #include "interceptor/interceptors.h" int intercept_fopen_mode_to_open_flags_helper(const char * mode); int popen_type_to_flags(const char * type); void clear_notify_on_read_write_state(const int fd); void set_notify_on_read_write_state(const int fd); void set_all_notify_on_read_write_states(); void copy_notify_on_read_write_state(const int to_fd, const int from_fd); void set_notify_on_read_state(const int fd, const bool is_pread); void set_notify_on_write_state(const int fd, const bool is_pwrite); bool notify_on_read(const int fd, const bool is_pread); bool notify_on_write(const int fd, const bool is_pwrite); /* Same as fileno(), but with safe NULL pointer handling. */ static inline int safe_fileno(FILE *stream) { int ret = stream ? get_ic_orig_fileno()(stream) : -1; if (ret == fb_sv_conn) { assert(0 && "fileno() returned the connection fd"); } return ret; } /* Same as dirfd(), but with safe NULL pointer handling. */ static inline int safe_dirfd(DIR *dirp) { int ret = dirp ? get_ic_orig_dirfd()(dirp) : -1; if (ret == fb_sv_conn) { assert(0 && "dirfd() returned the connection fd"); } return ret; } #endif // FIREBUILD_IC_FILE_OPS_H_ firebuild-0.8.2/src/interceptor/intercept.c000066400000000000000000001450601447164520700207740ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "interceptor/intercept.h" #include #include #include #ifdef __APPLE__ #include "libproc.h" #endif #ifdef __linux__ #include #endif #include #ifdef __linux__ #include #endif #include #include #include #include "interceptor/env.h" #include "interceptor/ic_file_ops.h" #include "interceptor/interceptors.h" #include "common/firebuild_common.h" #if defined(__s390x__) || defined (__powerpc64__) #define VDSO_NAME "linux-vdso64.so.1" #elif defined(__i386__) #define VDSO_NAME "linux-gate.so.1" #else #define VDSO_NAME "linux-vdso.so.1" #endif static void fb_ic_init_constructor(int argc, char **argv) __attribute__((constructor)); static void fb_ic_cleanup() __attribute__((destructor)); fd_state ic_fd_states[IC_FD_STATES_SIZE]; struct rusage initial_rusage; pthread_mutex_t ic_system_popen_lock = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t ic_global_lock = PTHREAD_MUTEX_INITIALIZER; char fb_conn_string[FB_PATH_BUFSIZE] = {'\0'}; size_t fb_conn_string_len = 0; int fb_sv_conn = -1; bool ic_called_syscall[IC_CALLED_SYSCALL_SIZE] = {0}; bool ic_init_started = false; bool ic_init_done = false; /** System locations to not ask ACK for when opening them, as set in the environment variable. */ char read_only_locations_env_buf[4096]; /** Ignore locations to not ask ACK for when opening them, as set in the environment variable. */ char ignore_locations_env_buf[4096]; /** * Jobserver users for which the jobserver fds have to be detected, as set in the environment * variable. */ char jobserver_users_env_buf[4096]; STATIC_CSTRING_VIEW_ARRAY(read_only_locations, 32); STATIC_CSTRING_VIEW_ARRAY(ignore_locations, 32); STATIC_CSTRING_VIEW_ARRAY(jobserver_users, 8); bool intercepting_enabled = true; char ic_cwd[FB_PATH_BUFSIZE] = {0}; size_t ic_cwd_len = 0; /** Program's argc and argv. */ static int ic_argc; static char **ic_argv; int ic_pid; __thread const char *thread_intercept_on = NULL; __thread sig_atomic_t thread_signal_danger_zone_depth = 0; __thread bool thread_has_global_lock = false; __thread sig_atomic_t thread_signal_handler_running_depth = 0; __thread sig_atomic_t thread_libc_nesting_depth = 0; __thread uint64_t thread_delayed_signals_bitmap = 0; #ifdef __APPLE__ /* OS X does not support RT signals https://flylib.com/books/en/3.126.1.110/1/ , * but we can handle 64 signals safely. */ #define SIGRTMAX ((int)sizeof(thread_delayed_signals_bitmap) * 8) #endif void (*orig_signal_handlers[IC_WRAP_SIGRTMAX])(void) = {NULL}; bool signal_is_wrappable(int signum) { /* Safety check, so that we don't crash if the user passes an invalid value to signal(), * sigset() or sigaction(). Just let the original function handle it somehow. */ if (signum < 1 || signum > IC_WRAP_SIGRTMAX) { return false; } return true; } void wrapper_signal_handler_1arg(int signum) { char debug_msg[256]; if (thread_signal_danger_zone_depth > 0) { snprintf(debug_msg, sizeof(debug_msg), "signal %d arrived in danger zone, delaying\n", signum); insert_debug_msg(debug_msg); thread_delayed_signals_bitmap |= (1LLU << (signum - 1)); return; } thread_signal_handler_running_depth++; snprintf(debug_msg, sizeof(debug_msg), "signal-handler-1arg-begin %d\n", signum); insert_debug_msg(debug_msg); ((void (*)(int))(*orig_signal_handlers[signum - 1]))(signum); snprintf(debug_msg, sizeof(debug_msg), "signal-handler-1arg-end %d\n", signum); insert_debug_msg(debug_msg); thread_signal_handler_running_depth--; } void wrapper_signal_handler_3arg(int signum, siginfo_t *info, void *ucontext) { char debug_msg[256]; if (thread_signal_danger_zone_depth > 0) { snprintf(debug_msg, sizeof(debug_msg), "signal %d arrived in danger zone, delaying\n", signum); insert_debug_msg(debug_msg); thread_delayed_signals_bitmap |= (1LLU << (signum - 1)); // FIXME(egmont) stash "info" return; } thread_signal_handler_running_depth++; snprintf(debug_msg, sizeof(debug_msg), "signal-handler-3arg-begin %d\n", signum); insert_debug_msg(debug_msg); // FIXME(egmont) if this is a re-raised signal from thread_raise_delayed_signals() // [can this be detected fully reliably, without the slightest race condition?] // then replace "info" with the stashed version ((void (*)(int, siginfo_t *, void *))(*orig_signal_handlers[signum - 1]))(signum, info, ucontext); snprintf(debug_msg, sizeof(debug_msg), "signal-handler-3arg-end %d\n", signum); insert_debug_msg(debug_msg); thread_signal_handler_running_depth--; } void thread_raise_delayed_signals() { /* Execute the delayed signals, by re-raising them. */ char debug_msg[256]; for (int signum = 1; signum <= IC_WRAP_SIGRTMAX; signum++) { if (thread_delayed_signals_bitmap & (1LLU << (signum - 1))) { snprintf(debug_msg, sizeof(debug_msg), "raising delayed signal %d\n", signum); insert_debug_msg(debug_msg); thread_delayed_signals_bitmap &= ~(1LLU << (signum - 1)); raise(signum); } } } int ic_pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset) { /* pthread_sigmask() is only available if we're linked against libpthread. * Otherwise use the single-threaded sigprocmask(). */ #if defined(__APPLE__) || FB_GLIBC_PREREQ(2, 34) return pthread_sigmask(how, set, oldset); #else static int (*ic_orig_pthread_sigmask)(int, const sigset_t *, sigset_t *); static bool tried_dlsym = false; if (ic_orig_pthread_sigmask) { return ic_orig_pthread_sigmask(how, set, oldset); } else { if (!tried_dlsym) { ic_orig_pthread_sigmask = dlsym(RTLD_NEXT, "pthread_sigmask"); tried_dlsym = true; /* Try again with possibly resolved symbol. */ return ic_pthread_sigmask(how, set, oldset); } else { return sigprocmask(how, set, oldset); } } #endif } void grab_global_lock(bool *i_locked, const char * const function_name) { thread_signal_danger_zone_enter(); /* Some internal integrity assertions */ if ((thread_has_global_lock) != (thread_intercept_on != NULL)) { char debug_buf[256]; snprintf(debug_buf, sizeof(debug_buf), "Internal error while intercepting %s: thread_has_global_lock (%s) and " "thread_intercept_on (%s) must go hand in hand", function_name, thread_has_global_lock ? "true" : "false", thread_intercept_on); insert_debug_msg(debug_buf); assert(0 && "Internal error: thread_has_global_lock and " "thread_intercept_on must go hand in hand"); } if (thread_signal_handler_running_depth == 0 && thread_libc_nesting_depth == 0 && thread_intercept_on != NULL) { char debug_buf[256]; snprintf(debug_buf, sizeof(debug_buf), "Internal error while intercepting %s: already intercepting %s " "(and no signal or atfork handler running in this thread)", function_name, thread_intercept_on); insert_debug_msg(debug_buf); assert(0 && "Internal error: nested interceptors (no signal handler running)"); } if (!thread_has_global_lock) { pthread_mutex_lock(&ic_global_lock); thread_has_global_lock = true; thread_intercept_on = function_name; *i_locked = true; } thread_signal_danger_zone_leave(); assert(thread_signal_danger_zone_depth == 0); } void release_global_lock() { thread_signal_danger_zone_enter(); pthread_mutex_unlock(&ic_global_lock); thread_has_global_lock = false; thread_intercept_on = NULL; thread_signal_danger_zone_leave(); assert(thread_signal_danger_zone_depth == 0); } /** debugging flags */ int32_t debug_flags = 0; char env_ld_library_path[FB_PATH_BUFSIZE] = {0}; bool insert_trace_markers = false; /** Next ACK id*/ static uint16_t ack_id = 1; voidp_set popened_streams; psfa *psfas = NULL; int psfas_num = 0; int psfas_alloc = 0; void insert_debug_msg(const char* m) { #ifdef FB_EXTRA_DEBUG if (insert_trace_markers) { int saved_errno = errno; char tpl[256] = "/FIREBUILD ### "; get_ic_orig_open()(strncat(tpl, m, sizeof(tpl) - strlen(tpl) - 1), 0); errno = saved_errno; } #else (void)m; #endif } void insert_begin_marker(const char* m) { if (insert_trace_markers) { char tpl[256] = "intercept-begin: "; insert_debug_msg(strncat(tpl, m, sizeof(tpl) - strlen(tpl) - 1)); } } void insert_end_marker(const char* m) { if (insert_trace_markers) { char tpl[256] = "intercept-end: "; insert_debug_msg(strncat(tpl, m, sizeof(tpl) - strlen(tpl) - 1)); } } /** Get next ACK id */ static uint16_t get_next_ack_id() { ack_id++; /* Start over after 65535, but skip the value of 0 because that means no ACK is expected. */ if (ack_id == 0) { ack_id = 1; } return ack_id; } /** * Receive a message consisting solely of an ack_id. * * It's the caller's responsibility to lock. * * @param fd the communication file descriptor * @return the received ack_id */ static uint16_t fb_recv_ack(int fd) { /* read the header */ msg_header header; #ifndef NDEBUG ssize_t ret = #endif fb_read(fd, &header, sizeof(header)); assert(ret == sizeof(header)); assert(header.msg_size == 0); assert(header.fd_count == 0); return header.ack_id; } /** Send the serialized version of the given message over the wire, * prefixed with the ack num and the message length */ static void fb_send_msg(int fd, const void /*FBBCOMM_Builder*/ *ic_msg, uint16_t ack_num) { int len = fbbcomm_builder_measure(ic_msg); char *buf = alloca(sizeof(msg_header) + len); memset(buf, 0, sizeof(msg_header)); fbbcomm_builder_serialize(ic_msg, buf + sizeof(msg_header)); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" memset(buf, 0, sizeof(msg_header)); ((msg_header *)buf)->ack_id = ack_num; ((msg_header *)buf)->msg_size = len; #pragma GCC diagnostic pop fb_write(fd, buf, sizeof(msg_header) + len); } void fb_fbbcomm_send_msg(const void /*FBBCOMM_Builder*/ *ic_msg, int fd) { thread_signal_danger_zone_enter(); fb_send_msg(fd, ic_msg, 0); thread_signal_danger_zone_leave(); } uint16_t fb_fbbcomm_send_msg_with_ack(const void /*FBBCOMM_Builder*/ *ic_msg, int fd) { thread_signal_danger_zone_enter(); uint16_t ack_num = get_next_ack_id(); fb_send_msg(fd, ic_msg, ack_num); return ack_num; } void fb_fbbcomm_check_ack(int fd, uint16_t ack_num) { #ifdef NDEBUG (void)ack_num; #else uint16_t ack_num_resp = #endif fb_recv_ack(fd); assert(ack_num_resp == ack_num); thread_signal_danger_zone_leave(); } void fb_fbbcomm_send_msg_and_check_ack(const void /*FBBCOMM_Builder*/ *ic_msg, int fd) { uint16_t ack_num = fb_fbbcomm_send_msg_with_ack(ic_msg, fd); fb_fbbcomm_check_ack(fd, ack_num); } static void send_pre_open_internal(const int dirfd, const char* pathname, bool need_ack) { FBBCOMM_Builder_pre_open ic_msg; fbbcomm_builder_pre_open_init(&ic_msg); fbbcomm_builder_pre_open_set_dirfd(&ic_msg, dirfd); BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(pre_open, dirfd, pathname); if (need_ack) { fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } else { fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } } void send_pre_open(const int dirfd, const char* pathname) { send_pre_open_internal(dirfd, pathname, true); } void send_pre_open_without_ack_request(const int dirfd, const char* pathname) { send_pre_open_internal(dirfd, pathname, false); } bool maybe_send_pre_open(const int dirfd, const char* pathname, int flags) { if (pathname && is_write(flags) && (flags & O_TRUNC) && !(flags & (O_EXCL | O_DIRECTORY)) #ifdef O_TMPFILE && (flags & O_TMPFILE) != O_TMPFILE #endif && !is_path_at_locations(pathname, -1, &ignore_locations)) { send_pre_open(dirfd, pathname); return true; } else { return false; } } void pre_clone_disable_interception(const int flags, bool *i_locked) { FBBCOMM_Builder_clone ic_msg; fbbcomm_builder_clone_init(&ic_msg); /* Skipping 'fn' */ /* Skipping 'stack' */ fbbcomm_builder_clone_set_flags(&ic_msg, flags); /* Skipping 'arg' */ /* Not sending return value */ /* Send and go on, no ack */ fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); /* clone() can be really tricky to intercept, for example when the cloned process shares * the file descriptor table with the parent (CLONE_FILES). In this case the interceptor * would have to protect two communication fds or implement locking across separate processes. */ intercepting_enabled = false; env_purge(environ); /* Releasing the global lock (if we grabbed it in this pass) to not keep it locked * in the forked process. */ if (*i_locked) { release_global_lock(); *i_locked = false; } } int clone_trampoline(void *arg) { clone_trampoline_arg *trampoline_arg = (clone_trampoline_arg *)arg; thread_signal_danger_zone_leave(); if (trampoline_arg->i_locked) { release_global_lock(); } atfork_child_handler(); return(trampoline_arg->orig_fn(trampoline_arg->orig_arg)); } size_t make_canonical(char *path, size_t original_length) { char *src = path, *dst = path; /* dst <= src all the time */ bool add_slash = true; if (path[0] == '\0') return 0; if (!(path[0] == '.' && path[1] == '/')) { char *a = strstr(path, "//"); char *b = strstr(path, "/./"); if (a == NULL && b == NULL) { /* This is the quick code path for most of the well-behaved paths: * doesn't start with "./", doesn't contain "//" or "/./". * If a path passes this check then the only thing that might need * fixing is a trailing "/" or "/.". */ size_t len = original_length; if (len >= 2 && path[len - 1] == '.' && path[len - 2] == '/') { /* Strip the final "." if the path ends in "/.". */ len--; path[len] = '\0'; } if (len >= 2 && path[len - 1] == '/') { /* Strip the final "/" if the path ends in "/" and that's not the entire path. */ len--; path[len] = '\0'; } /* The quick code path is done here. */ return len; } /* Does not start with "./", but contains at least a "//" or "/./". * Everything is fine up to that point. Fast forward src and dst. */ if (a != NULL && b != NULL) { src = dst = a < b ? a : b; } else if (a != NULL) { src = dst = a; } else { src = dst = b; } } else { /* Starts with "./", needs fixing from the beginning. */ src++; add_slash = false; /* Don't add "/" to dst when skipping the first one(s) in src. */ } while (src[0] != '\0') { /* Skip through a possible run of slashes and non-initial "." components, e.g. "//././". */ if (src[0] == '/') { while (src[0] == '/' || (src[0] == '.' && (src[1] == '/' || src[1] == '\0'))) src++; if (add_slash) { *dst++ = '/'; } } /* Handle a regular (not ".") component. */ while (src[0] != '/' && src[0] != '\0') { *dst++ = *src++; } add_slash = true; } /* If got empty path then it should be a "." instead. */ if (dst == path) { *dst++ = '.'; } /* Strip trailing slash, except if the entire path is "/". */ if (dst > path + 1 && dst[-1] == '/') { dst--; } *dst = '\0'; return dst - path; } #if 0 /* unittests for make_canonical() */ /* Macro so that assert() reports useful line numbers. */ #define test(A, B) { \ char *str = strdup(A); \ make_canonical(str, strlen(str)); \ if (strcmp(str, B)) { \ fprintf(stderr, "Error: input: %s\n", A); \ fprintf(stderr, " expected: %s\n", B); \ fprintf(stderr, " got instead: %s\n", str); \ } \ } int main() { test("/", "/"); test("/etc/hosts", "/etc/hosts"); test("/usr/include/vte-2.91/vte/vteterminal.h", "/usr/include/vte-2.91/vte/vteterminal.h"); test("/usr/bin/", "/usr/bin"); test("/usr/bin/.", "/usr/bin"); test("/usr/./bin", "/usr/bin"); test("/./usr/bin", "/usr/bin"); test("//", "/"); test("", ""); test(".", "."); test("/.", "/"); test("./", "."); test("/./././", "/"); test("./././.", "."); test("//foo//bar//", "/foo/bar"); test("/././foo/././bar/././", "/foo/bar"); test("///.//././/.///foo//.//bar//.", "/foo/bar"); test("////foo/../bar", "/foo/../bar"); test("/foo/bar/../../../../../", "/foo/bar/../../../../.."); test("/.foo/.bar/..quux", "/.foo/.bar/..quux"); test("foo", "foo"); test("foo/bar", "foo/bar"); test("././foo/./bar/./.", "foo/bar"); } #endif /* unittests for canonicalize_path() */ /** Compare pointers to char* like strcmp() for char* */ static int cmpstringpp(const void *p1, const void *p2) { /* The actual arguments to this function are "pointers to pointers to char", but strcmp(3) arguments are "pointers to char", hence the following cast plus dereference */ return strcmp(* (char * const *) p1, * (char * const *) p2); } /** Store from entries from environment variable. */ static void store_entries(const char* env_var, cstring_view_array *entries, char * entries_env_buf, size_t buffer_size) { char* env_entries = getenv(env_var); if (env_entries) { strncpy(entries_env_buf, env_entries, buffer_size); const size_t env_entries_len = strlen(env_entries); if (env_entries_len + 1 > buffer_size) { /* Trim to the fitting parts. The entries are used only for improving * performance and the space is allocated statically. */ entries_env_buf[buffer_size - 1] = '\0'; char * last_separator = strrchr(entries_env_buf, ':'); if (!last_separator) { /* This is a quite long single path that may be incomplete, thus ignore it. */ entries_env_buf[0] = '\0'; } else { /* Drop the possibly incomplete path after the last separator.*/ *last_separator = '\0'; } } char *prefix = entries_env_buf; /* Process all entries that fit location without reallocation. */ while (prefix && !is_cstring_view_array_full(entries)) { char *next_prefix = strchr(prefix, ':'); if (next_prefix) { *next_prefix = '\0'; next_prefix++; } /* Skip "". */ if (*prefix != '\0') { cstring_view_array_append_noalloc(entries, prefix); prefix = next_prefix; } } } } #ifdef __APPLE__ static void collect_shared_libs(cstring_view_array* libs, char *canonized_libs) { /* Skip first image because it is the binary itself. */ for (int32_t i = _dyld_image_count() - 1; i > 1 ; i--) { const char *image_name = _dyld_get_image_name(i); const size_t len = strlen(image_name); if (is_canonical(image_name, len)) { assert(!is_cstring_view_array_full(libs)); cstring_view_array_append_noalloc(libs, (/* not const */ char *)image_name); } else { char *canonical_name = &canonized_libs[i * FB_PATH_BUFSIZE]; memcpy(canonical_name, image_name, len + 1); make_canonical(canonical_name, len); assert(!is_cstring_view_array_full(libs)); cstring_view_array_append_noalloc(libs, canonical_name); } } } #else static bool skip_shared_lib(const char *name, const size_t len) { if (name[0] == '\0') { /* FIXME does this really happen? */ return true; } const char *libfirebuild = "/" LIBFIREBUILD_SO; if (len >= strlen(libfirebuild) && strcmp(name + strlen(name) - strlen(libfirebuild), libfirebuild) == 0) { /* This is internal to Firebuild, filter it out. */ return true; } if (strcmp(name, VDSO_NAME) == 0) { /* This is an in-kernel library, filter it out. */ return true; } return false; } /** * State struct for shared_libs_cb() */ typedef struct shared_libs_cb_data_ { /** Array of collected shared library names. */ cstring_view_array *array; /** Number of entries that could be collected to `array`. */ int collectable_entries; /** Number of entries that are not in canonical form, thus need to be made canonical. */ int not_canonical_entries; /** Buffert to store canonized library names. Size is canonized_libs_size * FB_PATH_BUFSIZE. */ char *canonized_libs; /** Number of canonized names canonized_libs can store. */ int canonized_libs_size; /** Number of canonized names stored in canonized_libs. */ int canonized_libs_count; } shared_libs_cb_data_t; /** Add shared library's name to the file list */ static int shared_libs_cb(struct dl_phdr_info *info, const size_t size, void *data) { (void) size; /* unused */ shared_libs_cb_data_t *cb_data = (shared_libs_cb_data_t *)data; cstring_view_array *array = cb_data->array; const char* name = info->dlpi_name; const size_t len = strlen(name); if (skip_shared_lib(name, len)) { return 0; } cb_data->collectable_entries++; if (is_canonical(name, len)) { if (!is_cstring_view_array_full(array)) { cstring_view_array_append_noalloc(array, (/* non-const */ char *) name); } } else { /* !is_canonical() */ cb_data->not_canonical_entries++; assert(cb_data->canonized_libs_count <= cb_data->canonized_libs_size); if (cb_data->canonized_libs_count < cb_data->canonized_libs_size) { /* The there is enough space for the new canonized entry. */ char * canonical_name = &cb_data->canonized_libs[cb_data->canonized_libs_count++ * FB_PATH_BUFSIZE]; memcpy(canonical_name, name, len + 1); make_canonical(canonical_name, len); if (!is_cstring_view_array_full(array)) { cstring_view_array_append_noalloc(array, canonical_name); } } } return 0; } #endif void atfork_parent_handler(void) { /* The variable i_am_intercepting from the intercepted fork() is * not available here, and storing it in a thread-global variable is * probably not worth the trouble. */ if (intercepting_enabled) { FBBCOMM_Builder_fork_parent ic_msg; fbbcomm_builder_fork_parent_init(&ic_msg); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } } void atfork_child_handler(void) { /* ic_pid still have parent process' pid */ pid_t ppid = ic_pid; /* Reset, getrusage will report the correct self resource usage. */ timerclear(&initial_rusage.ru_stime); timerclear(&initial_rusage.ru_utime); /* Reinitialize the lock, see #207. * * We don't know if the lock was previously held, we'd need to check * the variable i_am_intercepting from the intercepted fork() which is * not available here, and storing it in a thread-global variable is * probably not worth the trouble. The intercepted fork() will attempt * to unlock if it grabbed the lock, which will silently fail, that's * okay. */ if (intercepting_enabled) { pthread_mutex_init(&ic_global_lock, NULL); /* Add a useful trace marker */ if (insert_trace_markers) { char buf[256]; snprintf(buf, sizeof(buf), "launched via fork() by ppid %d", ppid); insert_debug_msg(buf); } /* Reinitialize other stuff */ reset_interceptors(); ic_pid = get_ic_orig_getpid()(); /* Reconnect to supervisor */ fb_init_supervisor_conn(); /* Inform the supervisor about who we are */ FBBCOMM_Builder_fork_child ic_msg; fbbcomm_builder_fork_child_init(&ic_msg); fbbcomm_builder_fork_child_set_pid(&ic_msg, ic_pid); fbbcomm_builder_fork_child_set_ppid(&ic_msg, ppid); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } } static void atexit_handler() { insert_debug_msg("our_atexit_handler-begin"); handle_exit(); insert_debug_msg("our_atexit_handler-end"); /* Destruction of global objects is not done here, because other exit handlers * may perform actions that need to be reported to the supervisor. * TODO(rbalint) add Valgrind suppress file */ } void handle_exit() { /* On rare occasions (e.g. two threads attempting to exit at the same * time) this method is called multiple times. The server can safely * handle it. */ /* Use the same pattern for locking as in tpl.c, simplified (fewer debugging messages). */ if (intercepting_enabled) { bool i_locked = false; thread_signal_danger_zone_enter(); if (!thread_has_global_lock) { pthread_mutex_lock(&ic_global_lock); thread_has_global_lock = true; thread_intercept_on = "handle_exit"; i_locked = true; } thread_signal_danger_zone_leave(); FBBCOMM_Builder_rusage ic_msg; fbbcomm_builder_rusage_init(&ic_msg); struct rusage ru; get_ic_orig_getrusage()(RUSAGE_SELF, &ru); timersub(&ru.ru_stime, &initial_rusage.ru_stime, &ru.ru_stime); timersub(&ru.ru_utime, &initial_rusage.ru_utime, &ru.ru_utime); fbbcomm_builder_rusage_set_utime_u(&ic_msg, (int64_t)ru.ru_utime.tv_sec * 1000000 + (int64_t)ru.ru_utime.tv_usec); fbbcomm_builder_rusage_set_stime_u(&ic_msg, (int64_t)ru.ru_stime.tv_sec * 1000000 + (int64_t)ru.ru_stime.tv_usec); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); if (i_locked) { thread_signal_danger_zone_enter(); pthread_mutex_unlock(&ic_global_lock); thread_has_global_lock = false; thread_intercept_on = NULL; thread_signal_danger_zone_leave(); } } } void *pthread_start_routine_wrapper(void *routine_and_arg) { if (insert_trace_markers) { char buf[256]; snprintf(buf, sizeof(buf), "launched via pthread_create() in pid %d", get_ic_orig_getpid()()); insert_debug_msg(buf); } void *(*start_routine)(void *) = ((void **)routine_and_arg)[0]; void *arg = ((void **)routine_and_arg)[1]; free(routine_and_arg); return (*start_routine)(arg); } /** * Parses and returns GNU Make jobserver fds if they are present in makeflags. * e.g. --jobserver-auth=R,W where ‘R’ and ‘W’ are non-negative integers representing fds * * @param[in] makeflags Make flags as set in environment's MAKEFLAGS * @param[out] fd_r R fd * @param[out] fd_w W fd * @return true, if jobserver fds were set and parsed */ static bool extract_jobserver_fds(const char* makeflags_env, int *fd_r, int *fd_w) { const char *makeflags = getenv(makeflags_env); if (!makeflags) { return false; } const char *needle = "--jobserver-auth="; const char *jobserver_option = strstr(makeflags, needle); if (!jobserver_option) { needle = "--jobserver-fds="; jobserver_option = strstr(makeflags, needle); } if (jobserver_option) { if (sscanf(jobserver_option + strlen(needle), "%d,%d", fd_r, fd_w) == 2) { return true; } } return false; } /** * Set up a supervisor connection * @return fd of the connection */ int fb_connect_supervisor() { #ifdef SOCK_CLOEXEC int conn = TEMP_FAILURE_RETRY(get_ic_orig_socket()(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)); #else int conn = TEMP_FAILURE_RETRY(get_ic_orig_socket()(AF_UNIX, SOCK_STREAM, 0)); #ifndef NDEBUG int fcntl_ret = #endif TEMP_FAILURE_RETRY(get_ic_orig_fcntl()(conn, F_SETFD, FD_CLOEXEC)); assert(fcntl_ret != -1); #endif assert(conn != -1); struct sockaddr_un remote; memset(&remote, 0, sizeof(remote)); remote.sun_family = AF_UNIX; #ifdef FB_EXTRA_DEBUG assert(strlen(fb_conn_string) < sizeof(remote.sun_path)); #endif strncpy(remote.sun_path, fb_conn_string, sizeof(remote.sun_path)); int conn_ret = TEMP_FAILURE_RETRY( get_ic_orig_connect()(conn, (struct sockaddr *)&remote, sizeof(remote))); if (conn_ret == -1) { get_ic_orig_perror()("connect"); assert(0 && "connection to supervisor failed"); } return conn; } void fb_init_supervisor_conn() { if (fb_conn_string[0] == '\0') { strncpy(fb_conn_string, getenv("FB_SOCKET"), sizeof(fb_conn_string)); fb_conn_string_len = strlen(fb_conn_string); } /* Reconnect to supervisor. * POSIX says to retry close() on EINTR (e.g. wrap in TEMP_FAILURE_RETRY()) * but Linux probably disagrees, see #723. */ get_ic_orig_close()(fb_sv_conn); fb_sv_conn = fb_connect_supervisor(); } /** * Detect main()'s argc and argv with heuristics. * * The reliable and portable initialization happens in fb_ic_init_constructor(), but in case * an intercepted function is called before the constructor of this shared library, argc * and argv still needs to be reported to the supervisor in the first message. * The heuristics below works for most programs, but not for mpicc. Luckily when intercepting * mpicc the constructor is called first, thus this heuristics is not used. */ static void init_argc_argv() { #ifdef __APPLE__ char** __environ = environ; #endif if (ic_argv == NULL) { char* arg = *(__environ - 2); unsigned long int argc_guess = 0; /* argv is NULL terminated */ assert(*(__environ - 1) == NULL); /* walk back on argv[] to find the first value matching the counted argument number */ while (argc_guess != (unsigned long int)arg) { argc_guess++; arg = *(__environ - 2 - argc_guess); } ic_argc = argc_guess; ic_argv = __environ - 1 - argc_guess; } } /** * Initialize interceptor's data structures and sync with supervisor. * * Collect information about process the earliest possible, right * when interceptor library loads or when the first interceped call happens */ void fb_ic_init() { /* Run only once, at startup. */ if (ic_init_started) { /* Should not be called recursively. */ assert(ic_init_done); return; } ic_init_started = true; get_ic_orig_getrusage()(RUSAGE_SELF, &initial_rusage); if (getenv("FB_INSERT_TRACE_MARKERS") != NULL) { insert_trace_markers = true; } store_entries("FB_READ_ONLY_LOCATIONS", &read_only_locations, read_only_locations_env_buf, sizeof(read_only_locations_env_buf)); store_entries("FB_IGNORE_LOCATIONS", &ignore_locations, ignore_locations_env_buf, sizeof(ignore_locations_env_buf)); store_entries("FB_JOBSERVER_USERS", &jobserver_users, jobserver_users_env_buf, sizeof(jobserver_users_env_buf)); #ifndef __mips__ /* We use an uint64_t as bitmap for delayed signals. Make sure it's okay. * On MIPS it is not enough, signals > 64 will not be wrapped. */ assert(SIGRTMAX <= IC_WRAP_SIGRTMAX); #endif voidp_set_init(&popened_streams); reset_interceptors(); assert(thread_intercept_on == NULL); thread_intercept_on = "init"; insert_debug_msg("initialization-begin"); set_all_notify_on_read_write_states(); /* Useful for debugging deadlocks with strace, since the same values appear in futex() * if we need to wait for the lock. */ if (insert_trace_markers) { char buf[256]; snprintf(buf, sizeof(buf), "ic_global_lock = %p", &ic_global_lock); insert_debug_msg(buf); snprintf(buf, sizeof(buf), "ic_system_popen_lock = %p", &ic_system_popen_lock); insert_debug_msg(buf); } /* init global variables */ /* Save a copy of LD_LIBRARY_PATH before someone might modify it. */ char *llp = getenv("LD_LIBRARY_PATH"); if (llp != NULL) { strncpy(env_ld_library_path, llp, sizeof(env_ld_library_path) - 1); } fb_init_supervisor_conn(); pthread_atfork(NULL, atfork_parent_handler, atfork_child_handler); atexit(atexit_handler); init_argc_argv(); pid_t pid, ppid; ic_pid = pid = get_ic_orig_getpid()(); ppid = get_ic_orig_getppid()(); if (get_ic_orig_getcwd()(ic_cwd, sizeof(ic_cwd)) == NULL) { assert(0 && "getcwd() returned NULL"); } ic_cwd_len = strlen(ic_cwd); FBBCOMM_Builder_scproc_query ic_msg; fbbcomm_builder_scproc_query_init(&ic_msg); fbbcomm_builder_scproc_query_set_version(&ic_msg, FIREBUILD_VERSION); fbbcomm_builder_scproc_query_set_pid(&ic_msg, pid); fbbcomm_builder_scproc_query_set_ppid(&ic_msg, ppid); fbbcomm_builder_scproc_query_set_cwd(&ic_msg, ic_cwd); fbbcomm_builder_scproc_query_set_arg_with_count(&ic_msg, (const char **) ic_argv, ic_argc); mode_t initial_umask = get_ic_orig_umask()(0077); get_ic_orig_umask()(initial_umask); fbbcomm_builder_scproc_query_set_umask(&ic_msg, initial_umask); /* make a sorted and filtered copy of env */ #ifdef __APPLE__ char **env = environ; #else char **env = __environ; #endif int env_len = 0, env_copy_len = 0; for (char** cursor = env; *cursor != NULL; cursor++) { env_len++; } char *env_copy[sizeof(env[0]) * (env_len + 1)]; for (char** cursor = env; *cursor != NULL; cursor++) { const char *fb_socket = "FB_SOCKET="; const char *fb_read_only_locations = "FB_READ_ONLY_LOCATIONS="; const char *fb_ignore_locations = "FB_IGNORE_LOCATIONS="; const char *fb_jobserver_users = "FB_JOBSERVER_USERS="; if (strncmp(*cursor, fb_socket, strlen(fb_socket)) != 0 && strncmp(*cursor, fb_read_only_locations, strlen(fb_read_only_locations)) != 0 && strncmp(*cursor, fb_ignore_locations, strlen(fb_ignore_locations)) != 0 && strncmp(*cursor, fb_jobserver_users, strlen(fb_jobserver_users)) != 0) { env_copy[env_copy_len++] = *cursor; } } env_copy[env_copy_len] = NULL; qsort(env_copy, env_copy_len, sizeof(env_copy[0]), cmpstringpp); fbbcomm_builder_scproc_query_set_env_var(&ic_msg, (const char **) env_copy); const char* slash_pos = strrchr(ic_argv[0], '/'); const char* cmd_name = slash_pos ? slash_pos + 1 : ic_argv[0]; int jobserver_fds[] = {-1, -1}; if (is_in_sorted_cstring_view_array(cmd_name, strlen(cmd_name), &jobserver_users)) { if (extract_jobserver_fds("CARGO_MAKEFLAGS", &jobserver_fds[0], &jobserver_fds[1])) { fbbcomm_builder_scproc_query_set_jobserver_fds(&ic_msg, jobserver_fds, 2); } else if (extract_jobserver_fds("MAKEFLAGS", &jobserver_fds[0], &jobserver_fds[1])) { fbbcomm_builder_scproc_query_set_jobserver_fds(&ic_msg, jobserver_fds, 2); } } /* get full executable path * see http://stackoverflow.com/questions/1023306/finding-current-executables-path-without-proc-self-exe * and readlink(2) */ char linkname[FB_PATH_BUFSIZE]; #ifdef __APPLE__ uint32_t r = sizeof(linkname); if (_NSGetExecutablePath(linkname, &r) == 0) { r = strlen(linkname); } else { /* A bigger buffer is needed. */ char* linkname2 = alloca(++r); if (_NSGetExecutablePath(linkname2, &r) != 0) { assert(0 && "Could not get the executable path even with the buffer " "that should have been enough."); r = 0; } else { r = strlen(linkname2); fbbcomm_builder_scproc_query_set_executable_with_length(&ic_msg, linkname2, r); } } #else ssize_t r = get_ic_orig_readlink()("/proc/self/exe", linkname, FB_PATH_BUFSIZE - 1); #endif if (r > 0 && r < FB_PATH_BUFSIZE) { linkname[r] = '\0'; fbbcomm_builder_scproc_query_set_executable_with_length(&ic_msg, linkname, r); } #ifdef __APPLE__ char original_executed_path[PROC_PIDPATHINFO_MAXSIZE]; if (proc_pidpath(getpid(), original_executed_path, sizeof(original_executed_path)) != -1 #else const char *original_executed_path = (const char*)getauxval(AT_EXECFN); if (original_executed_path #endif && strcmp(original_executed_path, linkname) != 0) { /* The macro relies on the field name matching the variable name. */ const char *executed_path = original_executed_path; BUILDER_SET_ABSOLUTE_CANONICAL(scproc_query, executed_path); if (strcmp(fbbcomm_builder_scproc_query_get_executed_path(&ic_msg), original_executed_path) != 0) { fbbcomm_builder_scproc_query_set_original_executed_path(&ic_msg, original_executed_path); } } /* list loaded shared libs */ #ifdef __APPLE__ const int image_count = _dyld_image_count(); cstring_view *libs_ptrs = alloca((image_count + 1) * sizeof(cstring_view)); cstring_view_array libs = {libs_ptrs, 0, image_count + 1}; char *canonized_libs = alloca(image_count * FB_PATH_BUFSIZE); collect_shared_libs(&libs, canonized_libs); #else STATIC_CSTRING_VIEW_ARRAY(libs, 64); int canonized_libs_size = 8; char *canonized_libs = alloca(canonized_libs_size * FB_PATH_BUFSIZE); shared_libs_cb_data_t cb_data = {&libs, 0, 0, canonized_libs, canonized_libs_size, 0}; dl_iterate_phdr(shared_libs_cb, &cb_data); if (cb_data.collectable_entries > cb_data.array->len) { if (cb_data.not_canonical_entries > canonized_libs_size) { /* canonized_libs was not big enough. */ canonized_libs_size = cb_data.not_canonical_entries; canonized_libs = alloca(canonized_libs_size * FB_PATH_BUFSIZE); } /* The initially allocated space was not enough to collect all shared libs, trying again. */ if (cb_data.collectable_entries > cb_data.array->size_alloc - 1) { /* libs array was not big enough. */ libs.p = alloca((cb_data.collectable_entries + 1) * sizeof(char*)); libs.size_alloc = cb_data.collectable_entries + 1; } else { /* The size was big enough, reset the contents*/ memset(libs.p, 0, libs.len * sizeof(char*)); } libs.len = 0; shared_libs_cb_data_t cb_data2 = {&libs, 0, 0, canonized_libs, canonized_libs_size, 0}; dl_iterate_phdr(shared_libs_cb, &cb_data2); assert(cb_data.collectable_entries == cb_data2.array->len); } #endif fbbcomm_builder_scproc_query_set_libs_cstring_views(&ic_msg, libs.p, libs.len); fb_send_msg(fb_sv_conn, &ic_msg, 0); /* Read the scproc_resp message header. */ msg_header header; #ifndef NDEBUG ssize_t ret = #endif fb_read(fb_sv_conn, &header, sizeof(header)); assert(ret == sizeof(header)); assert(header.msg_size > 0); uint16_t fd_count = header.fd_count; /* Read the scproc_resp message body. * * This message may have file descriptors attached as ancillary data. */ FBBCOMM_Serialized *sv_msg_generic = alloca(header.msg_size); void *anc_buf = NULL; size_t anc_buf_size = 0; if (fd_count > 0) { anc_buf_size = CMSG_SPACE(fd_count * sizeof(int)); anc_buf = alloca(anc_buf_size); memset(anc_buf, 0, anc_buf_size); } struct iovec iov = { 0 }; iov.iov_base = sv_msg_generic; iov.iov_len = header.msg_size; struct msghdr msgh = { 0 }; msgh.msg_iov = &iov; msgh.msg_iovlen = 1; msgh.msg_control = anc_buf; msgh.msg_controllen = anc_buf_size; /* This is the first message arriving on the socket, and it's reasonably small. * We can safely expect that the header and the payload are fully available (no short read). * However, a signal interrupt might occur. */ #ifndef NDEBUG ret = #endif TEMP_FAILURE_RETRY(get_ic_orig_recvmsg()(fb_sv_conn, &msgh, 0)); assert(ret >= 0 && ret == (ssize_t)header.msg_size); assert(fbbcomm_serialized_get_tag(sv_msg_generic) == FBBCOMM_TAG_scproc_resp); FBBCOMM_Serialized_scproc_resp *sv_msg = (FBBCOMM_Serialized_scproc_resp *) sv_msg_generic; debug_flags = fbbcomm_serialized_scproc_resp_get_debug_flags_with_fallback(sv_msg, 0); /* we may return immediately if supervisor decides that way */ if (fbbcomm_serialized_scproc_resp_get_shortcut(sv_msg)) { insert_debug_msg("this process was shortcut by the supervisor"); for (fbb_size_t i = 0; i < fbbcomm_serialized_scproc_resp_get_fds_appended_to_count(sv_msg); i++) { int fd = fbbcomm_serialized_scproc_resp_get_fds_appended_to_at(sv_msg, i); insert_debug_msg("seeking forward in fd"); get_ic_orig_lseek()(fd, 0, SEEK_END); } insert_debug_msg("exiting"); #ifdef __APPLE__ _exit(fbbcomm_serialized_scproc_resp_get_exit_status(sv_msg)); #else void(*orig_underscore_exit)(int) = (void(*)(int)) dlsym(RTLD_NEXT, "_exit"); (*orig_underscore_exit)(fbbcomm_serialized_scproc_resp_get_exit_status(sv_msg)); #endif assert(0 && "_exit() did not exit"); } if (fbbcomm_serialized_scproc_resp_has_dont_intercept(sv_msg)) { /* if set, must be true */ assert(fbbcomm_serialized_scproc_resp_get_dont_intercept(sv_msg)); intercepting_enabled = false; env_purge(environ); } /* Reopen the fds. * * The current temporary fd numbers were received as ancillary data, and are in the corresponding * slot of CMSG_DATA(cmsg). * * The list of desired final file descriptors are in the received FBB message. There might be * multiple desired slots (dups of each other) for each received fd. * * We're always reopening to a slot that should currently be open in the interceptor (and thus * we'll implicitly close that by the dup2). So the set of source fds and the set of targets fds * are disjoint. We don't have to worry about dup2ing to a target fd which we'd later need to use * as a source fd. */ assert(fd_count == fbbcomm_serialized_scproc_resp_get_reopen_fds_count(sv_msg)); if (fd_count > 0) { struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msgh); assert(cmsg); assert(cmsg->cmsg_level == SOL_SOCKET); assert(cmsg->cmsg_type == SCM_RIGHTS); assert(cmsg->cmsg_len == CMSG_LEN(fd_count * sizeof(int))); #ifdef FB_EXTRA_DEBUG /* Assert that the set of source fds and the set of target fds are disjoint. */ for (size_t i = 0; i < fbbcomm_serialized_scproc_resp_get_reopen_fds_count(sv_msg); i++) { FBBCOMM_Serialized_scproc_resp_reopen_fd *fds = (FBBCOMM_Serialized_scproc_resp_reopen_fd *) fbbcomm_serialized_scproc_resp_get_reopen_fds_at(sv_msg, i); assert(fbbcomm_serialized_scproc_resp_reopen_fd_get_fds_count(fds) >= 1); for (size_t j = 0; j < fbbcomm_serialized_scproc_resp_reopen_fd_get_fds_count(fds); j++) { int dst_fd = fbbcomm_serialized_scproc_resp_reopen_fd_get_fds_at(fds, j); for (size_t k = 0; k < fd_count; k++) { int src_fd; memcpy(&src_fd, CMSG_DATA(cmsg) + k * sizeof(int), sizeof(int)); assert(src_fd != dst_fd); } } } #endif /* For each source fd, dup2 it to all the desired target fds and then close the source fd. */ for (size_t i = 0; i < fbbcomm_serialized_scproc_resp_get_reopen_fds_count(sv_msg); i++) { FBBCOMM_Serialized_scproc_resp_reopen_fd *fds = (FBBCOMM_Serialized_scproc_resp_reopen_fd *) fbbcomm_serialized_scproc_resp_get_reopen_fds_at(sv_msg, i); int src_fd; memcpy(&src_fd, CMSG_DATA(cmsg) + i * sizeof(int), sizeof(int)); /* Preserve the fcntl(..., F_SETFL, ...) mode. * The supervisor doesn't track this value, so take the old local fd as reference. If there * are more than one local fds, they are supposedly dups of each other (at least this is what * the supervisor's bookkeeping said) and thus share these flags, so arbitrarily use the first * one as reference. * Similarly, since the targets will be dups of each other, it's enough to set the flags once. * In fact, set them on the source fd just because it's simpler this way. */ int flags = get_ic_orig_fcntl()(fbbcomm_serialized_scproc_resp_reopen_fd_get_fds_at(fds, 0), F_GETFL); assert(flags != -1); #ifndef NDEBUG int fcntl_ret = #endif get_ic_orig_fcntl()(src_fd, F_SETFL, flags); assert(fcntl_ret != -1); /* Dup2 the source fd to the desired places and then close the original. */ for (size_t j = 0; j < fbbcomm_serialized_scproc_resp_reopen_fd_get_fds_count(fds); j++) { int dst_fd = fbbcomm_serialized_scproc_resp_reopen_fd_get_fds_at(fds, j); #ifndef NDEBUG int dup2_ret = #endif get_ic_orig_dup2()(src_fd, dst_fd); assert(dup2_ret == dst_fd); } get_ic_orig_close()(src_fd); } } /* Report back each inherited fd not seeked to the end. */ for (fbb_size_t i = 0; i < fbbcomm_serialized_scproc_resp_get_seekable_fds_count(sv_msg); i++) { int fd = fbbcomm_serialized_scproc_resp_get_seekable_fds_at(sv_msg, i); int64_t size = fbbcomm_serialized_scproc_resp_get_seekable_fds_size_at(sv_msg, i); insert_debug_msg("get offset of fd"); #ifdef __APPLE__ off_t offset = get_ic_orig_lseek()(fd, 0, SEEK_CUR); #else off64_t offset = get_ic_orig_lseek64()(fd, 0, SEEK_CUR); #endif if (offset != size) { FBBCOMM_Builder_inherited_fd_offset ic_msg; fbbcomm_builder_inherited_fd_offset_init(&ic_msg); fbbcomm_builder_inherited_fd_offset_set_fd(&ic_msg, fd); fbbcomm_builder_inherited_fd_offset_set_offset(&ic_msg, offset); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } } insert_debug_msg("initialization-end"); thread_intercept_on = NULL; ic_init_done = true; } static void fb_ic_init_constructor(int argc, char **argv) { if (!ic_init_started) { ic_argc = argc; ic_argv = argv; fb_ic_init(); } } static void fb_ic_cleanup() { /* Don't put anything here, unless you really know what you're doing! * Our atexit_handler, which reports the resource usage to the supervisor, * is run _after_ this destructor, and still needs pretty much all the * functionality that we have (including the communication channel). */ } ssize_t fb_read(int fd, void *buf, size_t count) { FB_READ_WRITE(*get_ic_orig_read(), fd, buf, count); } ssize_t fb_write(int fd, const void *buf, size_t count) { FB_READ_WRITE(*get_ic_orig_write(), fd, buf, count); } /** Send error message to supervisor */ extern void fb_error(const char* msg) { FBBCOMM_Builder_fb_error ic_msg; fbbcomm_builder_fb_error_init(&ic_msg); fbbcomm_builder_fb_error_set_msg(&ic_msg, msg); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } /** Send debug message to supervisor if debug level is at least lvl */ void fb_debug(const char* msg) { FBBCOMM_Builder_fb_debug ic_msg; fbbcomm_builder_fb_debug_init(&ic_msg); fbbcomm_builder_fb_debug_set_msg(&ic_msg, msg); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } void psfa_init(const posix_spawn_file_actions_t *p) { // FIXME guard with mutex! /* This provides extra safety, in case a previous record belonging to this pointer wasn't cleaned * up, and now the same pointer is getting reused for a brand new posix_spawn_file_actions. */ psfa_destroy(p); /* grow buffer if necessary */ if (psfas_alloc == 0) { psfas_alloc = 4 /* whatever */; psfas = (psfa *) malloc(sizeof(psfa) * psfas_alloc); } else if (psfas_num == psfas_alloc) { psfas_alloc *= 2; psfas = (psfa *) realloc(psfas, sizeof(psfa) * psfas_alloc); } psfas[psfas_num].p = p; voidp_array_init(&psfas[psfas_num].actions); psfas_num++; } static void psfa_item_free(void *p) { /* For addopen() and addchdir_np() actions the filename needs to be freed. */ if (fbbcomm_builder_get_tag(p) == FBBCOMM_TAG_posix_spawn_file_action_open) { FBBCOMM_Builder_posix_spawn_file_action_open *builder = p; char *pathname = (/* non-const */ char *)fbbcomm_builder_posix_spawn_file_action_open_get_pathname(builder); free(pathname); } else if (fbbcomm_builder_get_tag(p) == FBBCOMM_TAG_posix_spawn_file_action_chdir) { FBBCOMM_Builder_posix_spawn_file_action_chdir *builder = p; char *pathname = (/* non-const */ char *)fbbcomm_builder_posix_spawn_file_action_chdir_get_pathname(builder); free(pathname); } free(p); } void psfa_destroy(const posix_spawn_file_actions_t *p) { // FIXME guard with mutex! for (int i = 0; i < psfas_num; i++) { if (psfas[i].p == p) { voidp_array_deep_free(&psfas[i].actions, psfa_item_free); if (i < psfas_num - 1) { /* Keep the array dense by moving the last item to this slot. */ psfas[i] = psfas[psfas_num - 1]; } psfas_num--; /* There can't be more than 1 match. */ break; } } } void psfa_addopen(const posix_spawn_file_actions_t *p, int fd, const char *pathname, int flags, mode_t mode) { voidp_array *obj = psfa_find(p); assert(obj); FBBCOMM_Builder_posix_spawn_file_action_open *fbbcomm_builder = malloc(sizeof(FBBCOMM_Builder_posix_spawn_file_action_open)); fbbcomm_builder_posix_spawn_file_action_open_init(fbbcomm_builder); fbbcomm_builder_posix_spawn_file_action_open_set_fd(fbbcomm_builder, fd); fbbcomm_builder_posix_spawn_file_action_open_set_pathname(fbbcomm_builder, strdup(pathname)); fbbcomm_builder_posix_spawn_file_action_open_set_flags(fbbcomm_builder, flags); fbbcomm_builder_posix_spawn_file_action_open_set_mode(fbbcomm_builder, mode); voidp_array_append(obj, fbbcomm_builder); } void psfa_addclose(const posix_spawn_file_actions_t *p, int fd) { voidp_array *obj = psfa_find(p); assert(obj); FBBCOMM_Builder_posix_spawn_file_action_close *fbbcomm_builder = malloc(sizeof(FBBCOMM_Builder_posix_spawn_file_action_close)); fbbcomm_builder_posix_spawn_file_action_close_init(fbbcomm_builder); fbbcomm_builder_posix_spawn_file_action_close_set_fd(fbbcomm_builder, fd); voidp_array_append(obj, fbbcomm_builder); } void psfa_addclosefrom_np(const posix_spawn_file_actions_t *p, int lowfd) { voidp_array *obj = psfa_find(p); assert(obj); FBBCOMM_Builder_posix_spawn_file_action_closefrom *fbbcomm_builder = malloc(sizeof(FBBCOMM_Builder_posix_spawn_file_action_closefrom)); fbbcomm_builder_posix_spawn_file_action_closefrom_init(fbbcomm_builder); fbbcomm_builder_posix_spawn_file_action_closefrom_set_lowfd(fbbcomm_builder, lowfd); voidp_array_append(obj, fbbcomm_builder); } void psfa_adddup2(const posix_spawn_file_actions_t *p, int oldfd, int newfd) { voidp_array *obj = psfa_find(p); assert(obj); FBBCOMM_Builder_posix_spawn_file_action_dup2 *fbbcomm_builder = malloc(sizeof(FBBCOMM_Builder_posix_spawn_file_action_dup2)); fbbcomm_builder_posix_spawn_file_action_dup2_init(fbbcomm_builder); fbbcomm_builder_posix_spawn_file_action_dup2_set_oldfd(fbbcomm_builder, oldfd); fbbcomm_builder_posix_spawn_file_action_dup2_set_newfd(fbbcomm_builder, newfd); voidp_array_append(obj, fbbcomm_builder); } void psfa_addchdir_np(const posix_spawn_file_actions_t *p, const char *pathname) { voidp_array *obj = psfa_find(p); assert(obj); FBBCOMM_Builder_posix_spawn_file_action_chdir *fbbcomm_builder = malloc(sizeof(FBBCOMM_Builder_posix_spawn_file_action_chdir)); fbbcomm_builder_posix_spawn_file_action_chdir_init(fbbcomm_builder); fbbcomm_builder_posix_spawn_file_action_chdir_set_pathname(fbbcomm_builder, strdup(pathname)); voidp_array_append(obj, fbbcomm_builder); } void psfa_addfchdir_np(const posix_spawn_file_actions_t *p, int fd) { voidp_array *obj = psfa_find(p); assert(obj); FBBCOMM_Builder_posix_spawn_file_action_fchdir *fbbcomm_builder = malloc(sizeof(FBBCOMM_Builder_posix_spawn_file_action_fchdir)); fbbcomm_builder_posix_spawn_file_action_fchdir_init(fbbcomm_builder); fbbcomm_builder_posix_spawn_file_action_fchdir_set_fd(fbbcomm_builder, fd); voidp_array_append(obj, fbbcomm_builder); } voidp_array *psfa_find(const posix_spawn_file_actions_t *p) { for (int i = 0; i < psfas_num; i++) { if (psfas[i].p == p) { return &psfas[i].actions; } } return NULL; } firebuild-0.8.2/src/interceptor/intercept.h000066400000000000000000000465221447164520700210040ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* * Interceptor library definitions */ #ifndef FIREBUILD_INTERCEPT_H_ #define FIREBUILD_INTERCEPT_H_ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #ifdef __linux__ #include #endif #include #ifdef __linux__ #include #endif #include #include #include #include #include #include #include #include #include #include "common/firebuild_common.h" #include "./fbbcomm.h" /** A poor man's (plain C) implementation of a hashmap: * posix_spawn_file_actions_t -> char** * implemented as a dense array with linear lookup. * * Each file action is encoded as a simple string, e.g. * - open: "o 10 0 0 /etc/hosts" * - close: "c 11" * - dup2: "d 3 5" */ typedef struct { const posix_spawn_file_actions_t *p; voidp_array actions; } psfa; extern psfa *psfas; extern int psfas_num; extern int psfas_alloc; /** This tells whether the supervisor needs to be notified on a read or write * event. The supervisor needs to be notified only on the first of each kind, * and only for file descriptors that were inherited by the process. * The "p" ones are stronger than their "non-p" counterparts, e.g. after notifying * the supervisor about a "pwrite" we don't need to notify it on a "write". * Similarly, "seek" is stronger than "tell", i.e. after a "seek" we don't send a "tell". */ typedef struct { /* Whether to notify on a read()-like operation at the current file offset, * including preadv2() with offset == -1. */ bool notify_on_read:1; /* Whether to notify on a pread()-like operation that reads at an arbitrary offset, * but not preadv2() with offset == -1. */ bool notify_on_pread:1; /* Whether to notify on a write()-like operation at the current file offset, * including pwrite2() with offset == -1. */ bool notify_on_write:1; /* Whether to notify on a pwrite()-like operation that writes at an arbitrary offset, * but not pwrite2() with offset == -1. */ bool notify_on_pwrite:1; /* Whether to notify on an lseek()-like operation that queries (but does not modify) the * offset. */ bool notify_on_tell:1; /* Whether to notify on an lseek()-like operation that modifies (and possibly also queries) * the offset. */ bool notify_on_seek:1; } fd_state; typedef struct { int (*orig_fn)(void *); void *orig_arg; bool i_locked; } clone_trampoline_arg; /** file fd states */ #define IC_FD_STATES_SIZE 4096 extern fd_state ic_fd_states[]; /** An uint64_t bitmap is used for delayed signals. */ #define IC_WRAP_SIGRTMAX 64 /** Called unknown syscalls */ #define IC_CALLED_SYSCALL_SIZE 1024 extern bool ic_called_syscall[]; /** Resource usage at the process' last exec() */ extern struct rusage initial_rusage; /** Global lock for preventing parallel system and popen calls */ extern pthread_mutex_t ic_system_popen_lock; /** Current working directory as reported to the supervisor */ extern char ic_cwd[FB_PATH_BUFSIZE]; extern size_t ic_cwd_len; /** Reset globally maintained information about intercepted functions */ extern void reset_fn_infos(); /** Connect to supervisor */ extern int fb_connect_supervisor(); /** Set up main supervisor connection */ extern void fb_init_supervisor_conn(); /** Global lock for serializing critical interceptor actions */ extern pthread_mutex_t ic_global_lock; /** Send message, delaying all signals in the current thread. * The caller has to take care of thread locking. */ void fb_fbbcomm_send_msg(const void /*FBBCOMM_Builder*/ *ic_msg, int fd); /** Send delaying all signals in the current thread, returning the ACK number sent. * The caller has to take care of thread locking. */ uint16_t fb_fbbcomm_send_msg_with_ack(const void /*FBBCOMM_Builder*/ *ic_msg, int fd); /** Wait for ACK, processing delayed signals in the current thread. * The caller has to take care of thread locking. */ void fb_fbbcomm_check_ack(int fd, uint16_t ack_num); /** Send message and wait for ACK, delaying all signals in the current thread. * The caller has to take care of thread locking. */ void fb_fbbcomm_send_msg_and_check_ack(const void /*FBBCOMM_Builder*/ *ic_msg, int fd); /** * Send pre_open message to supervisor if it is needed. * @return if message has been sent */ void send_pre_open(int dirfd, const char* pathname); void send_pre_open_without_ack_request(int dirfd, const char* pathname); bool maybe_send_pre_open(int dirfd, const char* pathname, int flags); /** Send message and disable interception before clone() */ void pre_clone_disable_interception(const int flags, bool *i_locked); /** Function to call by the intercepted clone(), registering into the supervisor then calling the original clone() fn param. */ int clone_trampoline(void *arg); /** Connection string to supervisor */ extern char fb_conn_string[FB_PATH_BUFSIZE]; /** Connection string length */ extern size_t fb_conn_string_len; /** Connection file descriptor to supervisor */ extern int fb_sv_conn; /** pthread_sigmask() if available (libpthread is loaded), otherwise sigprocmask() */ int ic_pthread_sigmask(int, const sigset_t *, sigset_t *); /** Fast check for whether interceptor init has been started. */ extern bool ic_init_started; /** Fast check for whether interceptor init has finished. */ extern bool ic_init_done; extern bool intercepting_enabled; /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_init(): * Add an entry, with a new empty string array, to our pool. */ extern void psfa_init(const posix_spawn_file_actions_t *p); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_destroy(): * Remove the entry, freeing up the string array, from our pool. * Do not shrink psfas. */ extern void psfa_destroy(const posix_spawn_file_actions_t *p); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_addopen(): * Append a corresponding FBBCOMM_Builder_posix_spawn_file_action_open builder to our structures. */ extern void psfa_addopen(const posix_spawn_file_actions_t *p, int fd, const char *pathname, int flags, mode_t mode); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_addclose(): * Append a corresponding FBBCOMM_Builder_posix_spawn_file_action_close builder to our structures. */ extern void psfa_addclose(const posix_spawn_file_actions_t *p, int fd); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_addclosefrom_np(): * Append a corresponding FBBCOMM_Builder_posix_spawn_file_action_closefrom builder to our structures. */ extern void psfa_addclosefrom_np(const posix_spawn_file_actions_t *p, int fd); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_adddup2(): * Append a corresponding FBBCOMM_Builder_posix_spawn_file_action_dup2 builder to our structures. */ extern void psfa_adddup2(const posix_spawn_file_actions_t *p, int oldfd, int newfd); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_chdir_np(): * Append a corresponding FBBCOMM_Builder_posix_spawn_file_action_chdir builder to our structures. */ extern void psfa_addchdir_np(const posix_spawn_file_actions_t *p, const char *pathname); /** * Additional bookkeeping to do after a successful posix_spawn_file_actions_fchdir_np(): * Append a corresponding FBBCOMM_Builder_posix_spawn_file_action_fchdir builder to our structures. */ extern void psfa_addfchdir_np(const posix_spawn_file_actions_t *p, int fd); /** * Find the voidp_array for a given posix_spawn_file_actions. */ extern voidp_array *psfa_find(const posix_spawn_file_actions_t *p); extern voidp_set popened_streams; /** Initial LD_LIBRARY_PATH so that we can fix it up if needed */ extern char env_ld_library_path[FB_PATH_BUFSIZE]; /** Insert marker open()-s for strace, ltrace, etc. */ extern bool insert_trace_markers; /** System and ignore locations to not ask ACK for when opening them. */ extern cstring_view_array read_only_locations; extern cstring_view_array ignore_locations; /** Insert debug message */ extern void insert_debug_msg(const char*); /** Insert begin marker strace, ltrace, etc. */ extern void insert_begin_marker(const char*); /** Insert end marker strace, ltrace, etc. */ extern void insert_end_marker(const char*); /** * Stored PID * When getpid() returns a different value, we missed a fork() :-) */ extern int ic_pid; /** * Make the filename canonical in place. * * String operation only, does not look at the actual file system. * Removes double slashes, trailing slashes (except if the entire path is "/") * and "." components. * Preserves ".." components, since they might point elsewhere if a symlink led to * its containing directory. * See #401 for further details and gotchas. * * Returns the length of the canonicalized path. */ size_t make_canonical(char *path, size_t original_length); #ifdef FB_EXTRA_DEBUG static inline bool ic_cwd_ok() { char buf[FB_PATH_BUFSIZE]; /* getcwd() is not intercepted */ char * getcwd_ret = getcwd(buf, sizeof(buf)); assert(getcwd_ret); return (strcmp(ic_cwd, buf) == 0); } #else static inline bool ic_cwd_ok() { return true; } #endif #define BUILDER_SET_CANONICAL2(msg, field, make_abs) do { \ const int orig_len = strlen(field); \ const bool fix_abs = make_abs && field[0] != '/'; \ const bool canonical = is_canonical(field, orig_len); \ if (!fix_abs && canonical) { \ fbbcomm_builder_##msg##_set_##field##_with_length(&ic_msg, field, \ orig_len); \ } else if (fix_abs \ && (orig_len == 0 \ || (orig_len == 1 && field[0] == '.'))) { \ fbbcomm_builder_##msg##_set_##field##_with_length(&ic_msg, ic_cwd, \ ic_cwd_len); \ } else { \ char * c_buf = \ alloca(orig_len + (fix_abs ? (ic_cwd_len + 2) : 1)); \ int c_len; \ if (fix_abs) { \ assert(ic_cwd_ok()); \ const size_t adjusted_cwd_len = ic_cwd_len == 1 ? 0 : ic_cwd_len; \ memcpy(c_buf, ic_cwd, adjusted_cwd_len); \ c_buf[adjusted_cwd_len] = '/'; \ memcpy(&c_buf[adjusted_cwd_len + 1], field, orig_len + 1); \ c_len = \ make_canonical(&c_buf[adjusted_cwd_len], orig_len + 1) \ + adjusted_cwd_len; \ if (c_len > 1 && c_buf[c_len - 1] == '/') { \ c_buf[c_len - 1] = '\0'; \ c_len--; \ } \ } else { \ memcpy(c_buf, field, orig_len + 1); \ c_len = make_canonical(c_buf, orig_len); \ } \ fbbcomm_builder_##msg##_set_##field##_with_length(&ic_msg, c_buf, \ c_len); \ } \ } while (0) /* Note: These macros must be called in the function serializing the FBB message because the buffer * holding the canonical field is allocated on the stack. */ #define BUILDER_SET_ABSOLUTE_CANONICAL(msg, field) BUILDER_SET_CANONICAL2(msg, field, true) #define BUILDER_SET_CANONICAL(msg, field) BUILDER_SET_CANONICAL2(msg, field, false) #define BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(msg, dirfd, field) \ BUILDER_SET_CANONICAL2(msg, field, (dirfd == AT_FDCWD)); /** The method name the current thread is intercepting, or NULL. In case of nested interceptions * (which can happen with signal handlers), it contains the outermost intercepted method. The value * is used for internal assertions and debugging messages only, not for actual business logic. */ extern __thread const char *thread_intercept_on; /** Whether the current thread is in "signal danger zone" where we don't like if a signal handler * kicks in, because our data structures are inconsistent. Blocking/unblocking signals would be too * slow for us (a pair of pthread_sigmask() kernel syscalls). So we just detect this scenario from * the wrapped signal handler. It's a counter, similar to a recursive lock. */ extern __thread sig_atomic_t thread_signal_danger_zone_depth; /** Only if thread_signal_danger_zone_depth == 0: tells whether the current thread holds * ic_global_lock. Querying the lock itself would not be async-signal-safe (and there aren't direct * methods for querying either), hence this accompanying value. * If thread_signal_danger_zone_depth > 0, the contents are undefined and must not be relied on. */ extern __thread bool thread_has_global_lock; /** Counting the depth of nested signal handlers running in the current thread. */ extern __thread sig_atomic_t thread_signal_handler_running_depth; /** Counting the nested depth of libc calls that might call other libc methods externally. * Currently fork() (atfork handlers) and dlopen() (constructor method) increment this level. * exit(), err(), error() etc. might also be ported to this infrastructure one day. */ extern __thread sig_atomic_t thread_libc_nesting_depth; /** Bitmap of signals that we're delaying. Multiplicity is irrelevant. Since signals are counted * from 1 to 64 (on Linux x86), it's bit number (signum-1) that corresponds to signal signum. */ extern __thread uint64_t thread_delayed_signals_bitmap; /** Array of the original signal handlers. * The items are actually either void (*)(int) a.k.a. sighandler_t, * or void (*)(int, siginfo_t *, void *), depending on how the handler was installed. */ extern void (*orig_signal_handlers[IC_WRAP_SIGRTMAX])(void); /** Whether we can intercept the given signal. */ bool signal_is_wrappable(int); /** When a signal handler is installed using signal(), sigset(), or sigaction() without the * SA_SIGINFO flag, this wrapper gets installed instead. * * See tpl_signal.c for how this wrapper is installed instead of the actual handler. * * This wrapper makes sure that the actual signal handler is only executed straight away if the * thread is not inside a "signal danger zone". Otherwise execution is deferred until the danger * zone is left (thread_signal_danger_zone_leave()). */ void wrapper_signal_handler_1arg(int); /** When a signal handler is installed using sigaction() with the SA_SIGINFO flag, * this wrapper gets installed instead. * * See wrapper_signal_handler_1arg() for further details. */ void wrapper_signal_handler_3arg(int, siginfo_t *, void *); /** Internal helper for thread_signal_danger_zone_leave(), see there for details. */ void thread_raise_delayed_signals(); /** Enter a "signal danger zone" where if a signal arrives then its execution is delayed. * This delaying is done manually because sigprocmask() or pthread_sigmask() are too expensive * for us. Our signal handler wrapper returns immediately (after the necessary bookkeeping, * but without invoking the actual handler). * The signal is later re-raised from thread_signal_danger_zone_leave(). * There can be multiple levels nested. * Inline so that it's as fast as possible. */ static inline void thread_signal_danger_zone_enter() { thread_signal_danger_zone_depth++; } /** Leave one level of "signal danger zone". * See thread_signal_danger_zone_enter() for how we delay signals. * If leaving the outermost level, re-raise the delayed signals. * Inline so that the typical branch is as fast as possible. */ static inline void thread_signal_danger_zone_leave() { /* Leave this danger zone first. * * If leaving the outermost danger zone, a signal can now kick in any time after this decrement, * potentially even before we reach that raise() below to emit the delayed signal, and its real * handler is executed immediately. This reordering is not a problem (fingers crossed). * * (The other possible order of code lines, i.e. raise the delayed signals first, then leave the * danger zone, would suffer from a race condition if a signal kicks in between these steps.) */ thread_signal_danger_zone_depth--; /* If this wasn't the outermost danger zone, there's nothing more to do, just return. * Also, obviously nothing to do if there's no delayed signal. * * Otherwise, re-raise them. (Note that in this case thread_delayed_signals_bitmap is stable now, * a randomly arriving signal cannot surprisingly modify it.) */ if (thread_delayed_signals_bitmap != 0 && thread_signal_danger_zone_depth == 0) { /* The rarely executed heavy stuff is factored out to a separate method to reduce code size. */ thread_raise_delayed_signals(); } } /** Take the global lock if the thread does not hold it already */ void grab_global_lock(bool *i_locked, const char * const function_name); void release_global_lock(); /** * Notify the supervisor after a fork(). Do it from the first registered pthread_atfork * handler so that it happens before other such handlers are run. * See #819 for further details. */ void atfork_parent_handler(void); /** * Reconnect to the supervisor and reinitialize other stuff in the child * after a fork(). Do it from the first registered pthread_atfork * handler so that it happens before other such handlers are run. * See #237 for further details. */ void atfork_child_handler(void); extern void fb_ic_init(); extern void handle_exit(); /** * A wrapper in front of the start_routine of a pthread_create(), inserting a useful trace marker. * pthread_create()'s two parameters start_routine and arg are accessed via one, * malloc()'ed in the intercepted pthread_create() and free()'d here. */ void *pthread_start_routine_wrapper(void *routine_and_arg); extern int __libc_start_main(int (*main)(int, char **, char **), int argc, char **ubp_av, void (*init)(void), void (*fini)(void), void (*rtld_fini)(void), void *stack_end); #endif // FIREBUILD_INTERCEPT_H_ firebuild-0.8.2/src/interceptor/interceptors.c000066400000000000000000000031351447164520700215140ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* interceptors.{cc,h} are the minimum necessary boilerplate * around the auto-generated gen_* interceptor files. */ #include "interceptor/interceptors.h" #include #include "common/firebuild_common.h" #include "common/platform.h" #include "interceptor/env.h" #include "interceptor/ic_file_ops.h" #include "interceptor/intercept.h" void reset_interceptors() { /* Include the auto-generated resetting of the internal states */ #include "interceptor/gen_reset.c" memset(ic_called_syscall, false, sizeof(ic_called_syscall[0]) * IC_CALLED_SYSCALL_SIZE); } /* Include the auto-generated definitions of the get_ic_orig function pointers */ #include "interceptor/gen_def.c" /* Include the auto-generated implementations of the interceptor functions */ #include "interceptor/gen_impl.c" firebuild-0.8.2/src/interceptor/interceptors.h000066400000000000000000000047571447164520700215340ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* interceptors.{cc,h} are the minimum necessary boilerplate * around the auto-generated gen_* interceptor files. */ #ifndef FIREBUILD_INTERCEPTORS_H_ #define FIREBUILD_INTERCEPTORS_H_ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif #include #include #include #include #ifdef __APPLE__ #include #include #include #else #include #endif #include #include #include #include #include #include #include #include #ifdef __APPLE__ #include #include #endif #include #include #ifdef __APPLE__ #include #endif #include #include #include #include #include #ifdef __linux__ #include #endif // #include struct ntptimeval; #include #include #ifdef __linux__ #include #endif #include #include // #include struct statx; struct ustat; #include #include #include "./fbbcomm.h" #include "common/platform.h" #ifndef __GLIBC_PREREQ #define FB_SSIZE_T ssize_t #define FB_VA_LIST va_list #else #if __GLIBC_PREREQ (2, 28) #define FB_SSIZE_T ssize_t #define FB_VA_LIST va_list #else #define FB_SSIZE_T _IO_ssize_t #define FB_VA_LIST _G_va_list #endif #endif void reset_interceptors(); /* Include the auto-generated declarations of the get_ic_orig function pointers, * and some convenience #define redirects */ #include "interceptor/gen_decl.h" #endif // FIREBUILD_INTERCEPTORS_H_ firebuild-0.8.2/src/interceptor/tpl.c000066400000000000000000000364461447164520700176050ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# This is the base template file that other templates derive from. #} {# In the mean time, this template is suitable for generating the #} {# required code for the majority of the intercepted functions. #} {# ------------------------------------------------------------------ #} {# Parameters: #} {# global_lock: Whether to acquire the global lock 'before', #} {# or 'after' the operation, or 'never' #} {# (default: 'before') #} {# before_lines: Things to place right before the call #} {# call_orig_lines: How to call the orig method #} {# after_lines: Things to place right after the call #} {# success: Success condition (default: "ret >= 0") #} {# msg_skip_fields: Don't automatically set these fields #} {# msg_add_fields: Additional code lines to set fields #} {# send_ret_on_success: Whether to send the actual return value #} {# on success (default: false) #} {# send_msg_on_error: Whether to send the message (with errno) on #} {# error (default: true) or only report success #} {# send_msg_condition: Custom condition to send message #} {# ack_condition: Whether to ask for ack 'true', 'false' or #} {# '' (default: 'false') #} {# after_send_lines: Things to place after sending msg #} {# diagnostic_ignored: GCC diagnostic ignored for the function #} {# ifdef_guard #if or #ifdef guard wrapping declarations, #} {# definitions and other func related parts #} {# ------------------------------------------------------------------ #} {# Jinja lacks native support for generating multiple files. #} {# Work it around by running multiple times, each time with a #} {# different value of `gen`, thus processing a different "segment" #} {# of this file. #} {# ------------------------------------------------------------------ #} {# #} {# Convenient handling of default-true booleans and other defaults #} ### if send_msg_on_error is not defined ### set send_msg_on_error = true ### endif ### if not send_msg_condition ### if send_msg_on_error {# Send it in case of error too, but not on EFAULT or EINTR, see #713 and #723. #} ### set send_msg_condition = "success || (errno != EINTR && errno != EFAULT)" ### else ### set send_msg_condition = "success" ### endif ### endif ### if global_lock is not defined ### set global_lock = 'before' ### endif {# #} {# --- Template for 'decl.h' ---------------------------------------- #} {# #} ### if gen == 'decl.h' ### if ifdef_guard {{ ifdef_guard }} ### endif ### block decl_h ### if not syscall ### if target == "darwin" #define get_ic_orig_{{ func }}() {{ func }} ### else extern {{ rettype }} (*get_ic_orig_{{ func }}(void)) ({{ sig_str }}); ### endif ### else #define ic_orig_{{ func }}(...) get_ic_orig_syscall()({{ func }} __VA_OPT__(,) __VA_ARGS__) ### endif ### endblock decl_h ### if ifdef_guard #endif ### endif ### endif {# #} {# --- Template for 'def.c' ----------------------------------------- #} {# #} ### if gen == 'def.c' ### if ifdef_guard {{ ifdef_guard }} ### endif ### block def_c ### if not syscall ### if target == "darwin" {{ rettype }} interposing_{{ func }} ({{ sig_str }}); static struct {const void* new; const void* original;} interpose_map_{{ func }} __attribute__ ((used, section("__DATA, __interpose"))) = {(const void*)interposing_{{ func }}, (const void*){{ func }}}; ### else inline {{ rettype }} (*get_ic_orig_{{ func }}(void)) ({{ sig_str }}) { static {{ rettype }}(*resolved)({{ sig_str }}) = NULL; if (resolved == NULL) { resolved = dlsym(RTLD_NEXT, "{{ func }}"); } return resolved; } ### endif ### endif ### endblock def_c ### if ifdef_guard #endif ### endif ### endif {# #} {# --- Template for 'reset.c' --------------------------------------- #} {# #} ### if gen == 'reset.c' ### if ifdef_guard {{ ifdef_guard }} ### endif ### block reset_c ### endblock reset_c ### if ifdef_guard #endif ### endif ### endif {# #} {# --- Template for 'list.txt' -------------------------------------- #} {# #} ### if gen == 'list.txt' {# Since ifdef_guard-s are not working here list.txt may have #} {# duplicates. #} ### block list_txt ### if not syscall ### if target == "darwin" _{{ func }} ### else {{ func }} ### endif ### endif ### endblock list_txt ### endif {# #} {# --- Template for 'impl.c' and 'impl_syscalls.c.inc' -------------- #} {# #} {# If func does not begin with 'SYS_' then it is an actual libc #} {# function (perhaps a thin wrapper around a kernel syscall). #} {# We generate a complete function definition into 'impl.c'. #} {# #} {# If func begins with 'SYS_' then it denotes the first parameter of #} {# a syscall(). We generate a 'case' label into 'impl_syscalls.c.inc' #} {# which is to be '#include'd within a 'switch' statement. #} {# #} ### if gen in ['impl.c', 'impl_syscalls.c.inc'] ### macro grab_lock_if_needed(grab_condition) /* Grabbing the global lock (unless it's already ours, e.g. we're in a signal handler) */ bool i_locked = false; /* "i" as in "me, myself and I" */ if ({{ grab_condition }}) { grab_global_lock(&i_locked, "{{ func }}"); } /* Global lock grabbed */ ### endmacro ### macro release_lock_if_needed() /* Releasing the global lock (if we grabbed it in this pass) */ if (i_locked) { release_global_lock(); } /* Global lock released */ ### endmacro /* Generated from {{ tpl }} */ ### block impl_c ### if ifdef_guard {{ ifdef_guard }} ### endif ### if not syscall /* Make the intercepting function visible */ #pragma GCC visibility push(default) #pragma GCC diagnostic push ### if diagnostic_ignored ### for item in diagnostic_ignored #pragma GCC diagnostic ignored "{{ item }}" ### endfor ### endif /* Undefine potential macro */ #ifdef {{ func }} #undef {{ func }} #endif ### if target == "darwin" {{ rettype }} {{ func }} ({{ sig_str }}); {{ rettype }} interposing_{{ func }} ({{ sig_str }}) { ### else {{ rettype }} {{ func }} ({{ sig_str }}) { ### endif ### else #ifdef {{ func }} /* this is prone against typos in the syscall name, but handles older kernels */ case {{ func }}: { #define IC_SYSCALL_{{ func }}_IS_INTERCEPTED /* The 64 bit variant has to be defined earlier. */ #if defined {{ func }}64 && !defined IC_SYSCALL_{{ func }}64_IS_INTERCEPTED #error "Missing {{ func }}64 interception" #endif va_list ap_args; va_start(ap_args, number); ### for arg in args ### if arg['vatype'] == "mode_t" ### if target == "darwin" int {{ arg['name'] }} = va_arg(ap_args, int); ### else {{ arg['vatype_and_name'] }} = va_arg(ap_args, {{ arg['vatype'] }}); ### endif ### else {{ arg['vatype_and_name'] }} = va_arg(ap_args, {{ arg['vatype'] }}); ### endif ### endfor va_end(ap_args); ### endif ### if rettype != 'void' {{ rettype }} ret; ### endif bool i_am_intercepting = intercepting_enabled; /* use a copy, in case another thread modifies it */ (void)i_am_intercepting; /* sometimes it's unused, silence warning */ /* Guard the communication channel */ ### block guard_connection_fd ### for arg in args {# It is ugly to check for the variable name to end with "fd", but is simple and works well in practice. #} ### if arg['type'] == "int" and arg['name'][-2:] == "fd" if ({{ arg['name'] }} == fb_sv_conn) { errno = EBADF; return {% if '*' in rettype %}NULL{% else %}-1{% endif %}; } ### endif ### endfor ### endblock ### if vararg /* Auto-generated for vararg functions */ ### if not syscall va_list ap; va_start(ap, {{ args[-1]['name'] }}); ### else va_list ap; va_start(ap, number); ### for arg in args va_arg(ap_args, {{ arg['type'] }}); /* consume {{ arg['name'] }} */ ### endfor ### endif ### endif /* Maybe don't intercept? */ ### block no_intercept ### endblock no_intercept /* Warm up */ ### if not no_saved_errno int saved_errno = errno; ### endif if (!ic_init_done) fb_ic_init(); #ifdef FB_EXTRA_DEBUG if (insert_trace_markers) { char debug_buf[256]; snprintf(debug_buf, sizeof(debug_buf), "%s%s{{ debug_before_fmt }}", i_am_intercepting ? "" : "[not intercepting] ", "{{ func }}"{{ debug_before_args }}); insert_begin_marker(debug_buf); } #endif ### block grab_lock ### if global_lock == 'before' {{ grab_lock_if_needed('i_am_intercepting') }} ### endif ### endblock grab_lock ### block body bool success = false; /* Beforework */ ### block before ### if before_lines ### for item in before_lines {{ item }} ### endfor ### endif ### endblock before /* Perform the call */ ### if not no_saved_errno errno = saved_errno; ### endif ### block call_orig ### if call_orig_lines ### for item in call_orig_lines {{ item }} ### endfor ### else ### if not vararg {%+ if rettype != 'void' %}ret = {% endif -%} {{ call_ic_orig_func }}({{ names_str }}); ### else #error "Need to implement call_orig for vararg function {{ func }}()" ### endif ### endif ### endblock call_orig ### if not no_saved_errno saved_errno = errno; ### endif success = {{ success }}; (void)success; /* sometimes it's unused, silence warning */ /* Afterwork */ ### block after ### if after_lines ### for item in after_lines {{ item }} ### endfor ### endif ### endblock after ### if global_lock == 'after' {{ grab_lock_if_needed('i_am_intercepting') }} ### endif ### block send_msg ### if msg /* Maybe notify the supervisor */ if (i_am_intercepting && ({{ send_msg_condition }})) { FBBCOMM_Builder_{{ msg }} ic_msg; fbbcomm_builder_{{ msg }}_init(&ic_msg); ### block set_fields /* Auto-generated from the function signature */ ### for arg in args ### if not msg_skip_fields or arg['name'] not in msg_skip_fields fbbcomm_builder_{{ msg }}_set_{{ arg['name'] }}(&ic_msg, {{ arg['name'] }}); ### else /* Skipping '{{ arg['name'] }}' */ ### endif ### endfor ### if msg_add_fields /* Additional ones from 'msg_add_fields' */ ### for item in msg_add_fields {{ item }} ### endfor ### endif ### endblock set_fields ### if send_ret_on_success /* Send return value on success */ if (success) fbbcomm_builder_{{ msg }}_set_ret(&ic_msg, ret); ### else /* Not sending return value */ ### endif ### if send_msg_on_error /* Send errno on failure */ ### if not msg_skip_fields or 'error_no' not in msg_skip_fields ### if not no_saved_errno if (!success) fbbcomm_builder_{{ msg }}_set_error_no(&ic_msg, saved_errno); ### else if (!success) fbbcomm_builder_{{ msg }}_set_error_no(&ic_msg, errno); ### endif ### endif ### endif ### if ack_condition /* Sending ack is conditional */ if ({{ ack_condition }}) { /* Send and wait for ack */ fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } else { /* Send and go on, no ack */ fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } ### else /* Send and go on, no ack */ fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); ### endif } ### endif ### endblock send_msg ### if after_send_lines ### for item in after_send_lines {{ item }} ### endfor ### endif ### endblock body /* Cool down */ #ifdef FB_EXTRA_DEBUG if (insert_trace_markers) { char debug_buf[256]; snprintf(debug_buf, sizeof(debug_buf), "%s%s{{ debug_after_fmt }}", i_am_intercepting ? "" : "[not intercepting] ", "{{ func }}"{{ debug_after_args }}); insert_end_marker(debug_buf); } #endif ### if global_lock == 'before' or global_lock == 'after' {{ release_lock_if_needed() }} ### endif ### if not no_saved_errno errno = saved_errno; ### endif ### if vararg /* Auto-generated for vararg functions */ va_end(ap); ### endif ### if rettype != 'void' return ret; ### endif } ### if not syscall #pragma GCC diagnostic pop #pragma GCC visibility pop ### else break; #endif /* {{ func }} */ ### endif ### if ifdef_guard #endif ### endif ### endblock impl_c ### endif {# #} {# ------------------------------------------------------------------ #} firebuild-0.8.2/src/interceptor/tpl__exit.c000066400000000000000000000043701447164520700207640ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the _exit() family (which exit immediately, skipping #} {# the atexit / on_exit handlers). #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block body /* Release the lock, to resemble tpl_exit.c. * handle_exit() will re-grab it. */ thread_signal_danger_zone_enter(); if (thread_has_global_lock) { pthread_mutex_unlock(&ic_global_lock); thread_has_global_lock = false; thread_intercept_on = NULL; } thread_signal_danger_zone_leave(); assert(thread_signal_danger_zone_depth == 0); /* Mark the end now */ insert_end_marker("{{ func }}"); /* Notify the supervisor by calling handle_exit() */ handle_exit(); /* Perform the call */ {{ call_ic_orig_func }}({{ names_str }}); /* Make scan-build happy */ (void)i_locked; /* Should not be reached */ assert(0 && "{{ func }} did not exit"); abort(); /* for NDEBUG */ ### endblock body firebuild-0.8.2/src/interceptor/tpl__fork.c000066400000000000000000000047671447164520700207660ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the _Fork() and vfork() calls. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block call_orig ### if func == 'vfork' /* vfork interception would be a bit complicated to implement properly * and most of the programs will work properly with _Fork */ ### endif ret = get_ic_orig__Fork()(); ### endblock call_orig ### block after if (!success) { /* Error */ // FIXME: disable shortcutting } /* In the child, what we need to do here is done via our atfork_child_handler(). * In the parent there's nothing to do here at all. */ ### endblock after ### block send_msg /* Notify the supervisor */ if (!success) { /* Error, nothing here to do */ } else if (ret == 0) { /* Make sure the child cannot receive a signal until it builds up * the new connection to the supervisor. To do this, we must block * signals before forking. */ sigset_t set_orig, set_block_all; sigfillset(&set_block_all); ic_pthread_sigmask(SIG_SETMASK, &set_block_all, &set_orig); atfork_child_handler(); ic_pthread_sigmask(SIG_SETMASK, &set_orig, NULL); } else { atfork_parent_handler(); } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_clone.c000066400000000000000000000107571447164520700207620ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for clone(). #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" {% set msg_skip_fields = ["fn", "stack", "arg"] %} ### block call_orig ### if syscall /* Need to extract 'flags'. See clone(2) NOTES about differences between architectures. */ #if defined(__s390__) || defined(__cris__) va_arg(ap, void*); /* skip over 'stack' */ #endif unsigned long flags = va_arg(ap, unsigned long); // TODO(rbalint) decode flags from varargs and intercept syscall() variants, too bool intercepted_clone = false; ### else // TODO(rbalint) cover more flag combinations bool intercepted_clone = (flags == (CLONE_VFORK | SIGCHLD)); ### endif uint16_t ack_num = 0; if (i_am_intercepting) { if (intercepted_clone) { FBBCOMM_Builder_fork_parent ic_msg; fbbcomm_builder_fork_parent_init(&ic_msg); ack_num = fb_fbbcomm_send_msg_with_ack(&ic_msg, fb_sv_conn); } else { pre_clone_disable_interception(flags, &i_locked); } } ### if not syscall int vararg_count = 0; if (flags & (CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID)) { vararg_count = 3; } else if (flags & CLONE_SETTLS) { vararg_count = 2; } else if (flags & (CLONE_PARENT_SETTID | CLONE_PIDFD)) { vararg_count = 1; } int (*passed_fn)(void *) = intercepted_clone ? clone_trampoline : fn; clone_trampoline_arg trampoline_arg = {fn, arg, i_locked}; void *passed_arg = intercepted_clone ? &trampoline_arg : arg; if (vararg_count == 0) { ret = get_ic_orig_{{ func }}()(passed_fn, stack, flags, passed_arg); } else { pid_t *parent_tid = va_arg(ap, pid_t *); if (vararg_count == 1) { ret = get_ic_orig_{{ func }}()(passed_fn, stack, flags, passed_arg, parent_tid); } else { void *tls = va_arg(ap, void *); if (vararg_count == 2) { ret = get_ic_orig_{{ func }}()(passed_fn, stack, flags, passed_arg, parent_tid, tls); } else { pid_t *child_tid = va_arg(ap, pid_t *); ret = get_ic_orig_{{ func }}()(passed_fn, stack, flags, passed_arg, parent_tid, tls, child_tid); } } } ### else /* The order of parameters is heavily architecture dependent. */ /* Pass on several long parameters unchanged, as in tpl_syscall.c. */ va_list ap_pass; va_start(ap_pass, number); long arg1 = va_arg(ap_pass, long); long arg2 = va_arg(ap_pass, long); long arg3 = va_arg(ap_pass, long); long arg4 = va_arg(ap_pass, long); long arg5 = va_arg(ap_pass, long); long arg6 = va_arg(ap_pass, long); long arg7 = va_arg(ap_pass, long); long arg8 = va_arg(ap_pass, long); va_end(ap_pass); ret = ic_orig_{{ func }}(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); ### endif ### endblock call_orig ### block send_msg /* Notify the supervisor */ if (!success) { if (intercepted_clone) { // TODO(rbalint) fix this case assert(0 && "The supervisor still waits for the child"); fb_fbbcomm_check_ack(fb_sv_conn, ack_num); } } else if (ret == 0) { /* Child is running child_atfork_handler in clone_trampoline in the intercepted cases. */ } else { if (intercepted_clone) { fb_fbbcomm_check_ack(fb_sv_conn, ack_num); } } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_close_range.c000066400000000000000000000052761447164520700221430ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the close_range() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block guard_connection_fd /* Skip our standard connection fd guarding. */ ### endblock guard_connection_fd ### block call_orig /* Reset our file states for fds that will be closed. */ if (i_am_intercepting && !(flags & CLOSE_RANGE_CLOEXEC)) { unsigned int i; for (i = first; i <= last && i < IC_FD_STATES_SIZE; i++) { set_notify_on_read_write_state(i); } } const unsigned int u_fb_sv_conn = (unsigned int)fb_sv_conn; if (first > u_fb_sv_conn || last < u_fb_sv_conn) { /* Just go ahead. */ ret = get_ic_orig_close_range()(first, last, flags); } else if (first == u_fb_sv_conn && last == u_fb_sv_conn) { /* Wishing to close only fb_sv_conn. Just pretend it succeeded. */ ret = 0; } else if (first == u_fb_sv_conn) { /* Need to skip the first fd. */ ret = get_ic_orig_close_range()(first + 1, last, flags); } else if (last == u_fb_sv_conn) { /* Need to skip the last fd. */ ret = get_ic_orig_close_range()(first, last - 1, flags); } else { /* Need to leave a hole in the range. */ int ret1 = get_ic_orig_close_range()(first, u_fb_sv_conn - 1, 0); int ret2 = get_ic_orig_close_range()(u_fb_sv_conn + 1, last, 0); ret = (ret1 == 0 && ret2 == 0) ? 0 : -1; } ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_closefrom.c000066400000000000000000000042311447164520700216410ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the closefrom() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block guard_connection_fd /* Skip our standard connection fd guarding. */ ### endblock guard_connection_fd ### block call_orig /* Reset our file states for fds that will be closed. */ if (i_am_intercepting) { int i; for (i = lowfd; i < IC_FD_STATES_SIZE; i++) { set_notify_on_read_write_state(i); } } if (lowfd > fb_sv_conn) { /* Just go ahead. */ get_ic_orig_closefrom()(lowfd); } else if (lowfd == fb_sv_conn) { /* Need to skip the first fd. */ get_ic_orig_closefrom()(lowfd + 1); } else { /* Need to leave a hole in the range. */ get_ic_orig_close_range()(lowfd, fb_sv_conn - 1, 0); get_ic_orig_closefrom()(fb_sv_conn + 1); } ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_copy_file_range.c000066400000000000000000000102511447164520700227740ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2023 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for copy_file_range and sendfile. Only need to notify the #} {# supervisor once per such methods if they fail. Otherwise notify #} {# Supervisor as if the intercepted process performed a (p)read() and #} {# a (p)write(). #} {# ------------------------------------------------------------------ #} {% set msg = 'gen_call' %} {% set sendfile_variant = func in ['sendfile', 'SYS_sendfile', 'sendfile64', 'SYS_sendfile64'] %} {% set copy_file_range_variant = func in ['copy_file_range', 'SYS_copy_file_range'] %} {# No locking around the write(): see issue #279 #} {% set global_lock = 'never' %} ### extends "tpl_once.c" ### block send_msg if (i_am_intercepting) { if (success) { ### if copy_file_range_variant const bool is_pread = off_in; const bool is_pwrite = off_out; ### elif sendfile_variant const bool is_pread = offset; const bool is_pwrite = false; ### endif ### if sendfile_variant ### if target == "darwin" const int fd_in = fd, fd_out = s; ### else const int fd_in = in_fd, fd_out = out_fd; ### endif ### endif if (notify_on_read(fd_in, is_pread) || notify_on_write(fd_out, is_pwrite)) { bool i_locked = false; grab_global_lock(&i_locked, "{{ func }}"); if (notify_on_read(fd_in, is_pread)) { FBBCOMM_Builder_read_from_inherited ic_msg; fbbcomm_builder_read_from_inherited_init(&ic_msg); fbbcomm_builder_read_from_inherited_set_fd(&ic_msg, fd_in); fbbcomm_builder_read_from_inherited_set_is_pread(&ic_msg, is_pread); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); set_notify_on_read_state(fd_in, is_pread); } if (notify_on_write(fd_out, is_pwrite)) { FBBCOMM_Builder_write_to_inherited ic_msg; fbbcomm_builder_write_to_inherited_init(&ic_msg); fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd_out); fbbcomm_builder_write_to_inherited_set_is_pwrite(&ic_msg, is_pwrite); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } set_notify_on_write_state(fd_out, is_pwrite); if (i_locked) { release_global_lock(); } } } else if (!ic_called_{{ func }} && !(ret == -1 && (saved_errno == EINVAL || saved_errno == EBADF))) { /* Notify the supervisor */ ic_called_{{ func }} = true; FBBCOMM_Builder_{{ msg }} ic_msg; fbbcomm_builder_{{ msg }}_init(&ic_msg); ### if msg == 'gen_call' fbbcomm_builder_{{ msg }}_set_call(&ic_msg, "{{ func }}"); fbbcomm_builder_{{ msg }}_set_error_no(&ic_msg, saved_errno); ### endif ### if ack /* Send and wait for ack */ fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); ### else /* Send and go on, no ack */ fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); ### endif } } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_dlopen.c000066400000000000000000000073641447164520700211430ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the dlopen() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" {% set msg_add_fields = ["if (absolute_filename == NULL && filename && strrchr(filename, '/')) {", " /* This is a relative or absolute name which will be made absolute in the next step. */", " absolute_filename = filename;", "}", "if (absolute_filename != NULL) BUILDER_SET_ABSOLUTE_CANONICAL(" + msg + ", absolute_filename);", "fbbcomm_builder_dlopen_set_error(&ic_msg, !success);"] %} ### block before /* TODO(rbalint) Save all loaded images before the dlopen() to collect also loaded shared * library dependencies. */ thread_libc_nesting_depth++; ### endblock before ### block after thread_libc_nesting_depth--; const char *absolute_filename = NULL; if (success) { ### if target == "darwin" /* From https://github.com/JuliaLang/julia/blob/0027ed143e90d0f965694de7ea8c692d75ffa1a5/src/sys.c#L572-L583 . * MIT licensed originally. */ /* Iterate through all images currently in memory */ int32_t i; for (i = _dyld_image_count() - 1; i > 1 ; i--) { /* dlopen() each image, check handle */ const char *image_name = _dyld_get_image_name(i); void *probe_handle = get_ic_orig_dlopen()(image_name, RTLD_LAZY | RTLD_NOLOAD); /* If the handle is the same as what was passed in (modulo mode bits), return this image name */ dlclose(probe_handle); if (((intptr_t)ret & (-4)) == ((intptr_t)probe_handle & (-4))) { absolute_filename = image_name; break; } } ### else struct link_map *map; if (dlinfo(ret, RTLD_DI_LINKMAP, &map) == 0) { /* Note: contrary to the dlinfo(3) manual page, this is not necessarily absolute. See #657. * We'll resolve to absolute when setting the FBB field. */ absolute_filename = map->l_name; } else { /* As per #920, dlinfo() returning an error _might_ cause problems later on in the intercepted * app, should it call dlerror(). A call to dlerror() would return a non-NULL string * describing dlinfo()'s failure, rather than NULL describing dlopen()'s success. But why * would any app invoke dlerror() after a successful dlopen()? Let's hope that in practice no * application does this. */ } ### endif } ### endblock after firebuild-0.8.2/src/interceptor/tpl_dup2.c000066400000000000000000000062361447164520700205310ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the dup2() and dup3() calls. #} {# See issue #632 for detailed explanation. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block guard_connection_fd /* Only handle oldfd here, newfd is handled a bit later. */ if (oldfd == fb_sv_conn) { errno = EBADF; return -1; } ### endblock ### block before int fb_sv_conn_new = -1; if (newfd == fb_sv_conn) { /* In order to make this dup2() or dup3() actually happen to the desired newfd * and still be able to talk to the supervisor, * we need to move fb_sv_conn to some other file descriptor. */ fb_sv_conn_new = TEMP_FAILURE_RETRY(get_ic_orig_dup()(fb_sv_conn)); if (fb_sv_conn_new < 0) { /* This dup() failed, which is very unlikely (out of available fds). * There's no hope to succeed with the actual dup2() and still be able to talk * to the supervisor. So just bail out. */ if (i_locked) { release_global_lock(); } errno = EBADF; return -1; } /* The communication fd has the close-on-exec flag set, and dup() doesn't copy it. */ TEMP_FAILURE_RETRY(get_ic_orig_fcntl()(fb_sv_conn_new, F_SETFD, FD_CLOEXEC)); } ### endblock ### block after if (newfd == fb_sv_conn) { if (success) { /* The actual dup2() succeeded and thus automatically closed fb_sv_conn. * Use the new fd number from now on for the communication. */ fb_sv_conn = fb_sv_conn_new; } else { /* The actual dup2() failed for whatever reason. Close the dupped connection fd. * POSIX says to retry close() on EINTR (e.g. wrap in TEMP_FAILURE_RETRY()) * but Linux probably disagrees, see #723. */ get_ic_orig_close()(fb_sv_conn_new); } } if (i_am_intercepting && success) copy_notify_on_read_write_state(newfd, oldfd); ### endblock firebuild-0.8.2/src/interceptor/tpl_error.c000066400000000000000000000075751447164520700210170ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for these calls: #} {# - error(), error_at_line(): #} {# these call the atexit / on_exit handlers if status != 0 #} {# - err(), errx(), verr(), verrx(): #} {# these always call the atexit / on_exit handlers #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block before /* First notify the supervisor that stderr has been written to, * similarly to tpl_write.c. */ int fd = safe_fileno(stderr); if (i_am_intercepting && (fd < 0 || fd >= IC_FD_STATES_SIZE || ic_fd_states[fd].notify_on_write == true)) { FBBCOMM_Builder_write_to_inherited ic_msg; fbbcomm_builder_write_to_inherited_init(&ic_msg); fbbcomm_builder_write_to_inherited_set_fd(&ic_msg, fd); fbbcomm_builder_write_to_inherited_set_is_pwrite(&ic_msg, false); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } if (fd >= 0 && fd < IC_FD_STATES_SIZE) { ic_fd_states[fd].notify_on_write = false; } ### endblock before ### block call_orig /* Then call the original. If error()'s or error_at_line()'s status is non-zero, * or if the method is err(), errx(), verr(), verrx(), then the original will call exit() * and in turn the atexit / on_exit handlers, which can call intercepted functions. * So release the lock, just like in tpl_exit.c. */ ### if func in ['error', 'error_at_line'] if (status == 0) { {{ super() }} ### else if (false) { ### endif } else { /* Exit handlers may call intercepted functions, so release the lock */ thread_signal_danger_zone_enter(); if (thread_has_global_lock) { pthread_mutex_unlock(&ic_global_lock); thread_has_global_lock = false; thread_intercept_on = NULL; } thread_signal_danger_zone_leave(); assert(thread_signal_danger_zone_depth == 0); /* Mark the end now */ insert_end_marker("{{ func }}"); /* Perform the call. * This will call the registered atexit / on_exit handlers, * including our handle_exit() which will notify the supervisor. */ {{ super() }} /* Make scan-build happy */ (void)i_locked; /* Should not be reached */ ### if func in ['error', 'error_at_line'] assert(0 && "{{ func }} with nonzero \"status\" parameter did not exit"); abort(); /* for NDEBUG */ ### else assert(0 && "{{ func }} did not exit"); abort(); /* for NDEBUG */ ### endif } ### endblock call_orig ### block send_msg /* Nothing else to tell to the supervisor */ ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_exec.c000066400000000000000000000143621447164520700206020ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the exec() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" {# Nasty hacks. Note that the func[4:] stuff also work with "fexecve". #} {% set l = ('l' in func[4:]) %} {% set v = ('v' in func[4:]) %} {# Exactly one of 'l' and 'v' is True. #} {% set p = ('p' in func[4:]) %} {% set e = ('e' in func[4:]) %} {% set f = (func[0] == 'f') %} {% set at = (func[-2:] == 'at') %} ### block body ### if l /* Convert "arg, ..." to "argv[]" on the stack (async-signal-safe) */ unsigned int argc = 1; unsigned int i; while (va_arg(ap, char*) != NULL) { argc++; } va_end(ap); char *argv[argc + 1]; argv[0] = (/* non-const */ char *) arg; va_start(ap, {{ args[-1]['name'] }}); for (i = 1; i <= argc ; i++) { argv[i] = va_arg(ap, char*); } ### if e /* Also locate the environment */ char **envp = va_arg(ap, char**); ### endif ### endif ### if not e /* Use the global environment */ char **envp = environ; ### endif /* Fix up the environment */ void *env_fixed_up; if (i_am_intercepting && env_needs_fixup((char **) envp)) { int env_fixup_size = get_env_fixup_size((char **) envp); env_fixed_up = alloca(env_fixup_size); env_fixup((char **) envp, env_fixed_up); } else { env_fixed_up = (void *) envp; } if (i_am_intercepting) { /* Notify the supervisor before the call */ FBBCOMM_Builder_exec ic_msg; fbbcomm_builder_exec_init(&ic_msg); ### if not f fbbcomm_builder_exec_set_file(&ic_msg, file); ### else /* Set for fexec*() */ fbbcomm_builder_exec_set_fd(&ic_msg, fd); ### endif ### if at /* Set for exec*at() */ fbbcomm_builder_exec_set_dirfd(&ic_msg, dirfd); // TODO(rbalint) see #32 fbbcomm_builder_exec_set_flags(&ic_msg, flags); ### endif ### if p /* Set for exec*p*() */ fbbcomm_builder_exec_set_with_p(&ic_msg, true); char *path_env; size_t confstr_buf_len = 0; if ((path_env = getenv("PATH"))) { fbbcomm_builder_exec_set_path(&ic_msg, path_env); } else { /* We have to fall back as described in man execvp. * This code is for glibc >= 2.24. For older versions * we'd need to prepend ".:", see issue 153. */ confstr_buf_len = get_ic_orig_confstr()(_CS_PATH, NULL, 0); } /* Use the stack rather than the heap, make sure it lives * until we send the message. */ if (confstr_buf_len > 0) { char *path_confstr = alloca(confstr_buf_len); get_ic_orig_confstr()(_CS_PATH, path_confstr, confstr_buf_len); fbbcomm_builder_exec_set_path(&ic_msg, path_confstr); } ### endif /* Command line arguments */ fbbcomm_builder_exec_set_arg(&ic_msg, (const char **) argv); /* Environment variables */ fbbcomm_builder_exec_set_env(&ic_msg, (const char **) env_fixed_up); /* Get CPU time used up to this exec() */ struct rusage ru; get_ic_orig_getrusage()(RUSAGE_SELF, &ru); timersub(&ru.ru_stime, &initial_rusage.ru_stime, &ru.ru_stime); timersub(&ru.ru_utime, &initial_rusage.ru_utime, &ru.ru_utime); fbbcomm_builder_exec_set_utime_u(&ic_msg, (int64_t)ru.ru_utime.tv_sec * 1000000 + (int64_t)ru.ru_utime.tv_usec); fbbcomm_builder_exec_set_stime_u(&ic_msg, (int64_t)ru.ru_stime.tv_sec * 1000000 + (int64_t)ru.ru_stime.tv_usec); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } /* Perform the call. */ {% set ic_orig_func = "ic_orig_" + func %} ### if l /* Instead of execl*(), call its execv*() counterpart. */ {% set ic_orig_func = ic_orig_func.replace("l", "v") %} ### endif ### if not e and not (target == "darwin" and ic_orig_func == "ic_orig_execvp") /* Instead of exec*() without "e", call its exec*e() counterpart. */ {% set ic_orig_func = ic_orig_func + "e" %} ### endif ### if syscall {% set call_ic_orig_func = ic_orig_func %} ### else {% set call_ic_orig_func = "get_" + ic_orig_func + "()" %} ### endif errno = saved_errno; ### if ic_orig_func == "ic_orig_execvp" char **env_saved = environ; environ = env_fixed_up; ### endif ret = {{ call_ic_orig_func }}({% if at %}dirfd, {% endif %}{% if f %}fd{% else %}file{% endif %}, argv{% if not ic_orig_func == "ic_orig_execvp" %}, env_fixed_up{% endif %}{% if at %}, flags{% endif %}); ### if ic_orig_func == "ic_orig_execvp" environ = env_saved; ### endif saved_errno = errno; if (i_am_intercepting) { /* Notify the supervisor after the call */ FBBCOMM_Builder_exec_failed ic_msg; fbbcomm_builder_exec_failed_init(&ic_msg); fbbcomm_builder_exec_failed_set_error_no(&ic_msg, saved_errno); /* It's important to wait for ACK, so that if this process now exits and its parent * successfully waits for it then the supervisor won't incorrectly see it in * exec_pending state and won't incorrectly believe that a statically linked binary * was execed. See #324 for details. */ fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } ### endblock body firebuild-0.8.2/src/interceptor/tpl_exit.c000066400000000000000000000044461447164520700206310ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the exit() call (which calls the atexit / on_exit #} {# handlers). #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block body /* Exit handlers may call intercepted functions, so release the lock */ thread_signal_danger_zone_enter(); if (thread_has_global_lock) { pthread_mutex_unlock(&ic_global_lock); thread_has_global_lock = false; thread_intercept_on = NULL; } thread_signal_danger_zone_leave(); assert(thread_signal_danger_zone_depth == 0); /* Mark the end now */ insert_end_marker("{{ func }}"); /* Perform the call. * This will call the registered atexit / on_exit handlers, * including our handle_exit() which will notify the supervisor. */ get_ic_orig_{{ func }}()({{ names_str }}); /* Make scan-build happy */ (void)i_locked; /* Should not be reached */ assert(0 && "{{ func }} did not exit"); abort(); /* for NDEBUG */ ### endblock body firebuild-0.8.2/src/interceptor/tpl_fcntl.c000066400000000000000000000104071447164520700207600ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the fcntl() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" {% set msg_add_fields = ["if (has_int_arg) fbbcomm_builder_" + msg + "_set_arg(&ic_msg, int_arg);", "if (send_ret) fbbcomm_builder_" + msg + "_set_ret(&ic_msg, ret);"] %} {% set send_msg_condition = "to_send" %} ### block before /* Preparations */ bool to_send = false; bool send_ret = false; bool has_int_arg = false; int int_arg = -1; switch (cmd) { /* Commands the supervisor doesn't need to know about. */ case F_GETFD: case F_GETFL: case F_SETFL: case F_GETLK: case F_SETLK: case F_SETLKW: #ifdef __linux__ case F_OFD_GETLK: case F_OFD_SETLK: case F_OFD_SETLKW: #endif case F_GETOWN: case F_SETOWN: #ifdef __linux__ case F_GETOWN_EX: case F_SETOWN_EX: case F_GETSIG: case F_SETSIG: case F_GETLEASE: case F_SETLEASE: case F_NOTIFY: case F_GETPIPE_SZ: case F_SETPIPE_SZ: case F_ADD_SEALS: case F_GET_SEALS: case F_GET_RW_HINT: case F_SET_RW_HINT: case F_GET_FILE_RW_HINT: case F_SET_FILE_RW_HINT: #endif { break; } /* Commands taking an int arg that the supervisor needs to know about, * and the return value is also relevant. */ case F_DUPFD: case F_DUPFD_CLOEXEC: { send_ret = true; } __attribute__ ((fallthrough)); /* Commands taking an int arg that the supervisor needs to know about, * but the return value is irrelevant (other than not being an error value). */ case F_SETFD: { to_send = true; has_int_arg = true; /* Start another vararg business that doesn't conflict with the one in call_orig, see #178. */ va_list ap_int; ### if not syscall /* Find 'arg' of an fcntl(fd, cmd, arg) */ va_start(ap_int, cmd); ### else /* Find 'arg' of a syscall(SYS_fcntl, fd, cmd, arg) */ va_start(ap_int, number); va_arg(ap_int, int); /* skip over fd */ va_arg(ap_int, int); /* skip over cmd */ ### endif int_arg = va_arg(ap_int, int); va_end(ap_int); break; } /* Commands that don't take an arg (or the arg doesn't matter to * the supervisor), but the supervisor needs to know about. This * includes all the unrecognized commands. Let's spell out the * recognized ones, rather than just catching them by "default", * for better readability. */ default: { to_send = true; break; } } ### endblock before ### block after switch (cmd) { case F_DUPFD: case F_DUPFD_CLOEXEC: if (i_am_intercepting && success) copy_notify_on_read_write_state(ret, fd); break; default: break; } ### endblock after ### block call_orig /* Treating the optional parameter as 'void *' should work, see #178. */ void *voidp_arg = va_arg(ap, void *); ret = {{ call_ic_orig_func }}(fd, cmd, voidp_arg); ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_fork.c000066400000000000000000000054361447164520700206210ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the fork() and vfork() calls. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block before /* Make sure the child cannot receive a signal until it builds up * the new connection to the supervisor. To do this, we must block * signals before forking. */ sigset_t set_orig, set_block_all; sigfillset(&set_block_all); ic_pthread_sigmask(SIG_SETMASK, &set_block_all, &set_orig); thread_libc_nesting_depth++; ### endblock before ### block call_orig ### if func in ['vfork', '__vfork'] /* vfork interception would be a bit complicated to implement properly * and most of the programs will work properly with fork */ ### endif ret = get_ic_orig_fork()(); ### endblock call_orig ### block after thread_libc_nesting_depth--; if (!success) { /* Error */ // FIXME: disable shortcutting } /* In the child, what we need to do here is done via our atfork_child_handler(). * In the parent there's nothing to do here at all. */ ### endblock after ### block send_msg /* Notify the supervisor */ if (!success) { /* Error, nothing here to do */ } else if (ret == 0) { /* The child signed in to the supervisor in atfork_child_handler(), nothing else here to do. */ } else { /* Parent sends the fork_parent message in atfork_parent_handler(). */ } /* Common for all three outcomes: re-enable signal delivery */ ic_pthread_sigmask(SIG_SETMASK, &set_orig, NULL); ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_ioctl.c000066400000000000000000000042621447164520700207660ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the ioctl() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" {% set send_msg_condition = "to_send" %} ### block before /* Preparations */ bool to_send = false; switch (cmd) { /* Commands that don't take an arg (or the arg doesn't matter to * the supervisor), but the supervisor needs to know about. */ case FIOCLEX: case FIONCLEX: { to_send = true; break; } /* Commands the supervisor doesn't need to know about. There are way * too many to list them all, so just use the wildcard. */ default: { break; } } ### endblock before ### block call_orig /* Treating the optional parameter as 'void *' should work, see #178. */ void *voidp_arg = va_arg(ap, void *); ret = {{ call_ic_orig_func }}(fd, cmd, voidp_arg); ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_marker_only.c000066400000000000000000000032261447164520700221750ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for methods where we only insert a trace marker, but #} {# other than that do not intercept (no locking or anything). #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" {% set msg = None %} {% set global_lock = 'never' %} ### block no_intercept i_am_intercepting = false; ### endblock no_intercept firebuild-0.8.2/src/interceptor/tpl_once.c000066400000000000000000000055351447164520700206040ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for methods where we only need to notify the supervisor #} {# once per such method. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block decl_h extern bool ic_called_{{ func }}; {{ super() }} ### endblock decl_h ### block reset_c ic_called_{{ func }} = false; ### endblock reset_c ### block def_c bool ic_called_{{ func }}; {{ super() }} ### endblock def_c ### block grab_lock ### if global_lock == 'before' {% set grab_lock_condition = "i_am_intercepting && !ic_called_" + func %} {{ grab_lock_if_needed(grab_lock_condition) }} ### endif ### endblock grab_lock ### block send_msg /* Notify the supervisor */ if (!ic_called_{{ func }}) { ic_called_{{ func }} = true; FBBCOMM_Builder_{{ msg }} ic_msg; fbbcomm_builder_{{ msg }}_init(&ic_msg); ### if msg == 'gen_call' fbbcomm_builder_{{ msg }}_set_call(&ic_msg, "{{ func }}"); ### endif ### if send_msg_on_error /* Send errno on failure */ ### if not msg_skip_fields or 'error_no' not in msg_skip_fields ### if not no_saved_errno if (!success) fbbcomm_builder_{{ msg }}_set_error_no(&ic_msg, saved_errno); ### else if (!success) fbbcomm_builder_{{ msg }}_set_error_no(&ic_msg, errno); ### endif ### endif ### endif ### if ack /* Send and wait for ack */ fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); ### else /* Send and go on, no ack */ fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); ### endif } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_open.c000066400000000000000000000062141447164520700206140ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the open() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### if msg_add_fields is not defined ### if vararg ### set msg_add_fields = ["if (__OPEN_NEEDS_MODE(flags)) fbbcomm_builder_" + msg + "_set_mode(&ic_msg, mode);"] ### else ### set msg_add_fields = [] ### endif ### if "dirfd" in sig_str ### do msg_add_fields.append("BUILDER_MAYBE_SET_ABSOLUTE_CANONICAL(" + msg + ", dirfd, pathname);") ### else ### do msg_add_fields.append("BUILDER_SET_ABSOLUTE_CANONICAL(" + msg + ", pathname);") ### endif ### do msg_add_fields.append("fbbcomm_builder_" + msg + "_set_pre_open_sent(&ic_msg, pre_open_sent);") ### endif ### set after_lines = ["if (i_am_intercepting && success) clear_notify_on_read_write_state(ret);"] ### set send_ret_on_success=True ### set ack_condition = "success && !is_path_at_locations(fbbcomm_builder_" + msg + "_get_pathname(&ic_msg), fbbcomm_builder_" + msg + "_get_pathname_len(&ic_msg), &read_only_locations) && !is_path_at_locations(fbbcomm_builder_" + msg + "_get_pathname(&ic_msg), fbbcomm_builder_" + msg + "_get_pathname_len(&ic_msg), &ignore_locations)" ### block before {{ super() }} ### if vararg ### if target == "darwin" int mode = 0; ### else mode_t mode = 0; ### endif if (__OPEN_NEEDS_MODE(flags)) { ### if target == "darwin" mode = va_arg(ap, int); ### else mode = va_arg(ap, mode_t); ### endif } ### endif ### if "dirfd" not in sig_str const int dirfd = AT_FDCWD; ### endif const int pre_open_sent = i_am_intercepting && maybe_send_pre_open(dirfd, pathname, flags); ### endblock before ### block call_orig ### if vararg ret = {{ call_ic_orig_func }}({{ names_str }}, mode); ### else ret = {{ call_ic_orig_func }}({{ names_str }}); ### endif ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_pclose.c000066400000000000000000000037621447164520700211450ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the pclose() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block before /* save it here, we can't do fileno() after the pclose() */ int fd = safe_fileno(stream); int was_popened = voidp_set_contains(&popened_streams, stream); if (was_popened) { voidp_set_erase(&popened_streams, stream); } if (i_am_intercepting) { /* Send a synthetic close before the pclose() to avoid a deadlock in wait4. */ FBBCOMM_Builder_close ic_msg; fbbcomm_builder_close_init(&ic_msg); fbbcomm_builder_close_set_fd(&ic_msg, fd); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); } ### endblock before firebuild-0.8.2/src/interceptor/tpl_pipe.c000066400000000000000000000130441447164520700206070ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the pipe() and pipe2() calls. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block call_orig if (i_am_intercepting) { /* No signal between sending the "pipe_request" message and receiving its "pipe_fds" response. */ thread_signal_danger_zone_enter(); /* Step 1/3. See #656 for design rationale. * Request the supervisor to create an intercepted unnamed pipe for us. */ FBBCOMM_Builder_pipe_request ic_msg1; fbbcomm_builder_pipe_request_init(&ic_msg1); fbbcomm_builder_pipe_request_set_flags(&ic_msg1, flags); fb_fbbcomm_send_msg(&ic_msg1, fb_sv_conn); /* Step 2/3. Receive the response from the supervisor, which carries * the file descriptors as ancillary data (SCM_RIGHTS). * The real data we're expecting to arrive is the usual message header * followed by a serialized FBB "pipe_created" message. */ msg_header sv_msg_hdr; uint64_t sv_msg_buf[8]; /* Should be large enough for the serialized "pipe_created" message. */ /* Read the header. */ #ifndef NDEBUG ssize_t received = #endif fb_read(fb_sv_conn, &sv_msg_hdr, sizeof(sv_msg_hdr)); assert(received == sizeof(sv_msg_hdr)); assert(sv_msg_hdr.ack_id == 0); // FIXME maybe send a real ack_id /* Taken from cmsg(3). */ union { /* Ancillary data buffer, wrapped in a union in order to ensure it is suitably aligned */ char buf[CMSG_SPACE(2 * sizeof(int))]; struct cmsghdr align; } u = { 0 }; struct iovec iov = { 0 }; iov.iov_base = sv_msg_buf; iov.iov_len = sv_msg_hdr.msg_size; struct msghdr msgh = { 0 }; msgh.msg_iov = &iov; msgh.msg_iovlen = 1; msgh.msg_control = u.buf; msgh.msg_controllen = sizeof(u.buf); /* Read the payload, with possibly two attached fds as ancillary data. * * The supervisor places this in the socket as an atomic step when the queue is almost empty, so * we don't expect a short read. However, a signal interrupt might occur. * * Set the O_CLOEXEC bit to the desired value. * The fcntl(..., F_SETFL, ...) bits were set by the supervisor. */ ### if target == "darwin" /* MSG_CMSG_CLOEXEC is not defined on OS X, but it would not be used anyway because pipe2 is * missing, too. */ #define MSG_CMSG_CLOEXEC 0 ### endif #ifndef NDEBUG received = #endif TEMP_FAILURE_RETRY(get_ic_orig_recvmsg()(fb_sv_conn, &msgh, (flags & O_CLOEXEC) ? MSG_CMSG_CLOEXEC : 0)); assert(received >= 0 && received == (ssize_t)sv_msg_hdr.msg_size); assert(fbbcomm_serialized_get_tag((FBBCOMM_Serialized *) sv_msg_buf) == FBBCOMM_TAG_pipe_created); thread_signal_danger_zone_leave(); FBBCOMM_Serialized_pipe_created *sv_msg = (FBBCOMM_Serialized_pipe_created *) sv_msg_buf; if (fbbcomm_serialized_pipe_created_has_error_no(sv_msg)) { /* Supervisor reported an error. */ assert(sv_msg_hdr.fd_count == 0); errno = fbbcomm_serialized_pipe_created_get_error_no(sv_msg); ret = -1; } else { /* The supervisor successfully created the pipes. See their local fds. */ assert(sv_msg_hdr.fd_count == 2); struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msgh); if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS || cmsg->cmsg_len != CMSG_LEN(2 * sizeof(int))) { /* fds missing, maybe file limit in this process exceeded? */ errno = EMFILE; ret = -1; } else { /* Two fds found as expected. */ memcpy(pipefd, CMSG_DATA(cmsg), 2 * sizeof(int)); ret = 0; } } } else { /* just create the pipe */ ### if target == "darwin" ret = get_ic_orig_pipe()(pipefd); ### else ret = get_ic_orig_pipe2()(pipefd, flags); ### endif } ### endblock call_orig {% set send_msg_on_error = False %} {% set msg = "pipe_fds" %} {% set msg_skip_fields = ["pipefd", "flags"] %} {% set msg_add_fields = ["if (success) {", " fbbcomm_builder_pipe_fds_set_fd0(&ic_msg, pipefd[0]);", " fbbcomm_builder_pipe_fds_set_fd1(&ic_msg, pipefd[1]);", "}"] %} ### block send_msg /* Step 3/3. Send the interceptor-side fd numbers to the supervisor. */ {{ super() }} ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_popen.c000066400000000000000000000150341447164520700207740ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the popen() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block before /* * The popen() call interception loops the output of the popen()-ed command through the supervisor * using a fifo. The original fd backing the FILE* stream returned by the popen() call is replaced * with a fifo endpoint which will be closed by the pclose() call eventually. */ int type_flags = popen_type_to_flags(type); if (i_am_intercepting) { pthread_mutex_lock(&ic_system_popen_lock); /* Notify the supervisor before the call */ FBBCOMM_Builder_popen ic_msg; fbbcomm_builder_popen_init(&ic_msg); fbbcomm_builder_popen_set_cmd(&ic_msg, cmd); fbbcomm_builder_popen_set_type_flags(&ic_msg, type_flags); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } ### endblock before ### block call_orig ENVIRON_SAVE_AND_FIXUP(did_env_fixup, environ_saved); {{ super() }} ENVIRON_RESTORE(did_env_fixup, environ_saved); ### endblock call_orig ### block after if (success) { assert(!voidp_set_contains(&popened_streams, ret)); voidp_set_insert(&popened_streams, ret); } ### endblock ### block send_msg if (i_am_intercepting) { /* Notify the supervisor after the call */ if (success) { /* No signal between sending the "popen_parent" message and receiving its "popen_fd" response. */ thread_signal_danger_zone_enter(); int ret_fileno = get_ic_orig_fileno()(ret); FBBCOMM_Builder_popen_parent ic_msg; fbbcomm_builder_popen_parent_init(&ic_msg); fbbcomm_builder_popen_parent_set_fd(&ic_msg, ret_fileno); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); /* Receive the response from the supervisor, which carries * the file descriptor as ancillary data (SCM_RIGHTS). * The real data we're expecting to arrive is the usual message header * followed by a serialized FBB "popen_fd" message. */ msg_header sv_msg_hdr; uint64_t sv_msg_buf[8]; /* Should be large enough for the serialized "popen_fd" message. */ /* Read the header. */ #ifndef NDEBUG ssize_t received = #endif fb_read(fb_sv_conn, &sv_msg_hdr, sizeof(sv_msg_hdr)); assert(received == sizeof(sv_msg_hdr)); assert(sv_msg_hdr.ack_id == 0); // FIXME maybe send a real ack_id /* Taken from cmsg(3). */ union { /* Ancillary data buffer, wrapped in a union in order to ensure it is suitably aligned */ char buf[CMSG_SPACE(1 * sizeof(int))]; struct cmsghdr align; } u = { 0 }; struct iovec iov = { 0 }; iov.iov_base = sv_msg_buf; iov.iov_len = sv_msg_hdr.msg_size; struct msghdr msgh = { 0 }; msgh.msg_iov = &iov; msgh.msg_iovlen = 1; msgh.msg_control = u.buf; msgh.msg_controllen = sizeof(u.buf); /* Read the payload, with the attached fd as ancillary data. * * The supervisor places this in the socket as an atomic step when the queue is almost empty, * so we don't expect a short read. However, a signal interrupt might occur. */ #ifndef NDEBUG received = #endif TEMP_FAILURE_RETRY(get_ic_orig_recvmsg()(fb_sv_conn, &msgh, 0)); assert(received >= 0 && received == (ssize_t)sv_msg_hdr.msg_size); assert(fbbcomm_serialized_get_tag((FBBCOMM_Serialized *) sv_msg_buf) == FBBCOMM_TAG_popen_fd); assert(sv_msg_hdr.fd_count == 1); thread_signal_danger_zone_leave(); struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msgh); if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS || cmsg->cmsg_len != CMSG_LEN(1 * sizeof(int))) { assert(0 && "expected ancillary fd missing"); } else { /* fd found as expected. */ int ancillary_fd; memcpy(&ancillary_fd, CMSG_DATA(cmsg), sizeof(int)); /* Move to the desired slot. Set the O_CLOEXEC bit to the desired value. * The fcntl(..., F_SETFL, ...) bits were set by the supervisor. */ assert(ancillary_fd != ret_fileno); /* because ret_fileno is still open */ ### if target == "linux" if (TEMP_FAILURE_RETRY(get_ic_orig_dup3()(ancillary_fd, ret_fileno, type_flags & O_CLOEXEC)) != ret_fileno) { assert(0 && "dup3() on the popened fd failed"); } ### else if (TEMP_FAILURE_RETRY(get_ic_orig_dup2()(ancillary_fd, ret_fileno)) != ret_fileno) { assert(0 && "dup2() on the popened fd failed"); } if (TEMP_FAILURE_RETRY(get_ic_orig_fcntl()(ret_fileno, F_SETFD, type_flags & FD_CLOEXEC)) != 0) { assert(0 && "fcntl() on the popened fd failed"); } ### endif /* POSIX says to retry close() on EINTR (e.g. wrap in TEMP_FAILURE_RETRY()) * but Linux probably disagrees, see #723. */ if (get_ic_orig_close()(ancillary_fd) < 0) { assert(0 && "close() on the dup3()d popened fd failed"); } } } else { FBBCOMM_Builder_popen_failed ic_msg; fbbcomm_builder_popen_failed_init(&ic_msg); fbbcomm_builder_popen_failed_set_error_no(&ic_msg, saved_errno); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } pthread_mutex_unlock(&ic_system_popen_lock); } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_posix_spawn.c000066400000000000000000000100701447164520700222200ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the posix_spawn() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block before { pthread_mutex_lock(&ic_system_popen_lock); /* Notify the supervisor before the call */ FBBCOMM_Builder_posix_spawn ic_msg; fbbcomm_builder_posix_spawn_init(&ic_msg); fbbcomm_builder_posix_spawn_set_file(&ic_msg, file); if (file_actions) { voidp_array *p = psfa_find(file_actions); assert(p); fbbcomm_builder_posix_spawn_set_file_actions(&ic_msg, (const FBBCOMM_Builder **) (p->p)); } ### if func == 'posix_spawnp' fbbcomm_builder_posix_spawn_set_is_spawnp(&ic_msg, true); ### else fbbcomm_builder_posix_spawn_set_is_spawnp(&ic_msg, false); ### endif fbbcomm_builder_posix_spawn_set_arg(&ic_msg, (const char **) argv); fbbcomm_builder_posix_spawn_set_env(&ic_msg, (const char **) envp); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } ### endblock before ### block call_orig /* Fix up the environment */ void *env_fixed_up; if (i_am_intercepting && env_needs_fixup((char **) envp)) { int env_fixup_size = get_env_fixup_size((char **) envp); env_fixed_up = alloca(env_fixup_size); env_fixup((char **) envp, env_fixed_up); } else { env_fixed_up = (char **)envp; } /* Fix up missing out parameter for internal use */ pid_t tmp_pid; if (!pid) { pid = &tmp_pid; } ret = get_ic_orig_{{ func }}()({{ names_str | replace("envp", "env_fixed_up")}}); ### endblock call_orig ### block send_msg { /* Notify the supervisor after the call */ if (success) { FBBCOMM_Builder_posix_spawn_parent ic_msg; fbbcomm_builder_posix_spawn_parent_init(&ic_msg); fbbcomm_builder_posix_spawn_parent_set_arg(&ic_msg, (const char **) argv); if (file_actions) { voidp_array *p = psfa_find(file_actions); assert(p); fbbcomm_builder_posix_spawn_parent_set_file_actions(&ic_msg, (const FBBCOMM_Builder **) (p->p)); } fbbcomm_builder_posix_spawn_parent_set_pid(&ic_msg, *pid); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } else { /* Unlike at most other methods where we skip on EINTR or EFAULT, here we always have to send * a counterpart to the posix_spawn message. */ FBBCOMM_Builder_posix_spawn_failed ic_msg; fbbcomm_builder_posix_spawn_failed_init(&ic_msg); fbbcomm_builder_posix_spawn_failed_set_arg(&ic_msg, (const char **) argv); /* errno is not documented to be set, the error code is in the return value. */ fbbcomm_builder_posix_spawn_failed_set_error_no(&ic_msg, ret); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } pthread_mutex_unlock(&ic_system_popen_lock); } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_posix_spawn_file_actions.c000066400000000000000000000043141447164520700247430ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the posix_spawn_file_actions_...() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block guard_connection_fd {# Override the main template's corresponding block so that the #} {# connection fd is _not_ guarded here. This is because matching the #} {# raw fd number against the _current_ connection fd number is #} {# incorrect. By the time the actions we register here will be #} {# executed, the communication fd might have moved elsewhere due to #} {# an intercepted dup2(), or reopened as a regular file due to a #} {# preceding posix_spawn_file_action. See #875 for further details. #} ### endblock ### block after if (success) { {{ func | replace("posix_spawn_file_actions_", "psfa_") }} ({{ names_str }}); } ### endblock after ### block send_msg /* No supervisor communication */ ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_pthread_create.c000066400000000000000000000041761447164520700226320ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for pthread_create, inherited from marker_only. #} {# Insert another trace markers, telling the pid. #} {# ------------------------------------------------------------------ #} ### extends "tpl_marker_only.c" {% set msg = None %} {% set global_lock = False %} ### block no_intercept i_am_intercepting = false; (void)i_am_intercepting; ### endblock no_intercept ### block call_orig /* Need to pass two pointers using one. Allocate room on the heap, * placing it on the stack might not live long enough. * Will be free()d in pthread_start_routine_wrapper(). */ void **routine_and_arg = malloc(2 * sizeof(void *)); routine_and_arg[0] = start_routine; routine_and_arg[1] = arg; ret = get_ic_orig_pthread_create()(thread, attr, pthread_start_routine_wrapper, routine_and_arg); ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_read.c000066400000000000000000000051471447164520700205720ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for functions reading from a (regular or special) file, #} {# including #} {# - low-level [p]read*() family #} {# - high-level stdio like fread(), getc(), scanf() etc. #} {# - low-level socket reading recv*() family #} {# and perhaps more. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### if is_pread is not defined ### set is_pread = "false" ### endif ### if msg_skip_fields is not defined ### set msg_skip_fields = [] ### endif ### do msg_skip_fields.append("error_no") {% set msg = "read_from_inherited" %} {# No locking around the read(): see issue #279 #} {% set global_lock = 'never' %} ### block set_fields {{ super() }} fbbcomm_builder_{{ msg }}_set_is_pread(&ic_msg, is_pread); ### endblock set_fields ### block send_msg bool is_pread = {{ is_pread }}; {# Acquire the lock if sending a message #} if (notify_on_read(fd, is_pread)) { /* Need to notify the supervisor */ {{ grab_lock_if_needed('true') | indent(2) }} {{ super() | indent(2) }} set_notify_on_read_state(fd, is_pread); {{ release_lock_if_needed() | indent(2) }} } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_readlink.c000066400000000000000000000037131447164520700214450ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the readlink() family. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block set_fields {{ super() }} /* Create a zero-terminated copy on the stack. * Make sure it lives until we send the message. */ int len = 0; if (ret >= 0 && (size_t)labs(ret) <= bufsiz) { len = ret; } char ret_target[len + 1]; if (len > 0) { memcpy(ret_target, buf, len); ret_target[len] = '\0'; /* Returned path is a raw string, not to be resolved. */ fbbcomm_builder_{{ msg }}_set_ret_target(&ic_msg, ret_target); } ### endblock set_fields firebuild-0.8.2/src/interceptor/tpl_recvmsg.c000066400000000000000000000104351447164520700213210ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the recv[m]msg() calls. #} {# In addition to the "read" template, i.e. notifying the supervisor #} {# on a read from an inherited fd, we also need to notify the #} {# supervisor if a file descriptor was received using SCM_RIGHTS. #} {# This is done in another FBB message. #} {# ------------------------------------------------------------------ #} ### extends "tpl_read.c" ### set is_pread = "false" {% set msg = "read_from_inherited" %} {# No locking around the recv[m]msg(): see issue #279 #} {% set global_lock = 'never' %} ### block send_msg {{ super() }} {# tpl_read.c's stuff #} {% if 'recvmmsg' in func %} /* recvmmsg() can return multiple messages. Loop over them. For simplicity, handle them * separately, i.e. send separate FBB messages if more of them have SCM_RIGHTS fds. */ int i; for (i = 0; i < ret; i++) { struct msghdr *msg = &msgvec[i].msg_hdr; {% endif %} /* A message can have multiple ancillary messages attached to it, each of them can carry multiple * file descriptors. It's possible, even though unlikely, that multiple ancillary messages contain * some fds. Especially since the Linux kernel seems to flatten them out into a single ancillary * message of the type SCM_RIGHTS. * * To simplify our lives, send a separate FBB message to the supervisor for each such ancillary * message of type SCM_RIGHTS. However, for each ancillary message, send all its fds in a single * FBB message as an array. */ struct cmsghdr *cmsg; for (cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL; cmsg = CMSG_NXTHDR(msg, cmsg)) { if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { /* How do we figure out how many fds we actually received? It's unclear to me. * My best bet is that we'd need the inverse of CMSG_LEN(), for which there is no macro. * We could find the inverse by calling CMSG_LEN() with our guesses in a loop. Or open * up the definition from the glibc header to invert it, which is what I'm doing here. */ int len = cmsg->cmsg_len - (CMSG_DATA(cmsg) - (unsigned char *)cmsg); assert(len >= 0); int num_fds = len / sizeof(int); if (num_fds > 0) { /* Notify the supervisor */ FBBCOMM_Builder_recvmsg_scm_rights ic_msg1; fbbcomm_builder_recvmsg_scm_rights_init(&ic_msg1); #ifdef MSG_CMSG_CLOEXEC fbbcomm_builder_recvmsg_scm_rights_set_cloexec(&ic_msg1, flags & MSG_CMSG_CLOEXEC); #endif #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-align" fbbcomm_builder_recvmsg_scm_rights_set_fds(&ic_msg1, (int *) CMSG_DATA(cmsg), num_fds); #pragma GCC diagnostic pop {{ grab_lock_if_needed('true') | indent(8) }} fb_fbbcomm_send_msg(&ic_msg1, fb_sv_conn); {{ release_lock_if_needed() | indent(8) }} } } } {% if 'recvmmsg' in func %} } /* looping over msgvec */ {% endif %} ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_seek.c000066400000000000000000000050121447164520700205750ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for functions seeking a file or querying the offset. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### if msg_skip_fields is not defined ### set msg_skip_fields = [] ### endif ### do msg_skip_fields.append("error_no") {% set msg = "seek_in_inherited" %} {# No locking around the seek(), to follow the pattern of tpl_{read,write}.c #} {% set global_lock = 'never' %} ### block set_fields {{ super() }} fbbcomm_builder_{{ msg }}_set_modify_offset(&ic_msg, modify_offset); ### endblock set_fields ### block send_msg bool modify_offset = {{ modify_offset }}; {# Acquire the lock if sending a message #} if (fd < 0 || fd >= IC_FD_STATES_SIZE || (modify_offset == false && ic_fd_states[fd].notify_on_tell == true) || (modify_offset == true && ic_fd_states[fd].notify_on_seek == true)) { /* Need to notify the supervisor */ {{ grab_lock_if_needed('true') | indent(2) }} {{ super() | indent(2) }} if (fd >= 0 && fd < IC_FD_STATES_SIZE) { ic_fd_states[fd].notify_on_tell = false; if (modify_offset) { ic_fd_states[fd].notify_on_seek = false; } } {{ release_lock_if_needed() | indent(2) }} } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_shm_open.c000066400000000000000000000037121447164520700214630ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2023 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for shm_open() #} {# ------------------------------------------------------------------ #} ### extends "tpl_once.c" ### block before {{ super() }} ### if vararg ### if target == "darwin" int mode = 0; ### else mode_t mode = 0; ### endif if (__OPEN_NEEDS_MODE(oflag)) { ### if target == "darwin" mode = va_arg(ap, int); ### else mode = va_arg(ap, mode_t); ### endif } ### endif ### endblock before ### block call_orig ### if vararg ret = {{ call_ic_orig_func }}({{ names_str }}, mode); ### else ret = {{ call_ic_orig_func }}({{ names_str }}); ### endif ### endblock call_orig firebuild-0.8.2/src/interceptor/tpl_signal.c000066400000000000000000000107311447164520700211270ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the signal() [ANSI C], sigset() [System V] and #} {# sigaction() [POSIX] calls. #} {# sigvec() [BSD] is not included because glibc dropped this API. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block call_orig ### if func in ['signal', 'SYS_signal', 'sigset'] if (signal_is_wrappable(signum)) { ### if func in ['signal', 'sigset'] sighandler_t old_orig_signal_handler = (sighandler_t)orig_signal_handlers[signum - 1]; ### else long int old_orig_signal_handler = (long int)orig_signal_handlers[signum - 1]; ### endif sighandler_t new_signal_handler = (handler == SIG_IGN || handler == SIG_DFL) ? handler : wrapper_signal_handler_1arg; orig_signal_handlers[signum - 1] = (void (*)(void))handler; ret = {{ call_ic_orig_func }}(signum, new_signal_handler); if ((void (*)(int))ret == wrapper_signal_handler_1arg) { ret = old_orig_signal_handler; } } else { ret = {{ call_ic_orig_func }}(signum, handler); } ### elif func in ['sigaction', 'SYS_sigaction', '__sigaction'] if (signal_is_wrappable(signum)) { struct sigaction wrapped_act; void (*old_orig_signal_handler)(void) = orig_signal_handlers[signum - 1]; if (act != NULL) { wrapped_act = *act; if (act->sa_flags & SA_SIGINFO) { /* sa_sigaction, handler called with 3 args */ orig_signal_handlers[signum - 1] = (void (*)(void))act->sa_sigaction; /* FIXME(egmont) It's unclear to me whether SIG_IGN and SIG_DFL are allowed values here in the SA_SIGINFO branch, * probably not (they're of incompatible pointer types, hence the double casting). Still, better safe than sorry. */ void (*new_signal_handler)(int, siginfo_t *, void *) = ((sighandler_t)(void *)act->sa_sigaction == SIG_IGN || (sighandler_t)(void *)act->sa_sigaction == SIG_DFL) ? act->sa_sigaction : wrapper_signal_handler_3arg; wrapped_act.sa_sigaction = new_signal_handler; } else { /* sa_handler, handler called with 1 arg */ orig_signal_handlers[signum - 1] = (void (*)(void))wrapped_act.sa_handler; void (*new_signal_handler)(int) = (act->sa_handler == SIG_IGN || act->sa_handler == SIG_DFL) ? act->sa_handler : wrapper_signal_handler_1arg; wrapped_act.sa_handler = new_signal_handler; } } ret = {{ call_ic_orig_func }}(signum, act ? &wrapped_act : NULL, oldact); if (ret == 0 && oldact != NULL) { if (oldact->sa_flags & SA_SIGINFO) { /* sa_sigaction, handler called with 3 args */ if (oldact->sa_sigaction == wrapper_signal_handler_3arg) { oldact->sa_sigaction = (void (*)(int, siginfo_t *, void *))old_orig_signal_handler; } } else { /* sa_handler, handler called with 1 arg */ if (oldact->sa_handler == wrapper_signal_handler_1arg) { oldact->sa_handler = (void (*)(int))old_orig_signal_handler; } } } } else { ret = get_ic_orig_sigaction()(signum, act, oldact); } ### endif ### endblock call_orig ### block send_msg /* Shhhhh, don't tell anything to the supervisor */ ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_skip.c000066400000000000000000000035371447164520700206260ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for methods that we don't intercept. #} {# This is a standalone template, does not extend tpl.c. #} {# ------------------------------------------------------------------ #} ### if syscall ### if gen == 'impl_syscalls.c.inc' #ifdef {{ func }} /* this is prone against typos in the syscall name, but handles older kernels */ case {{ func }}: { skip_interception = true; goto default_syscall_handling; } #endif ### endif ### else ### if gen == 'decl.h' #define get_ic_orig_{{ func }}() {{ func }} ### endif ### endif firebuild-0.8.2/src/interceptor/tpl_syscall.c000066400000000000000000000120441447164520700213230ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the syscall() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block impl_c ### if ifdef_guard {{ ifdef_guard }} ### endif /* Make the intercepting function visible */ #pragma GCC visibility push(default) #pragma GCC diagnostic push ### if target == "darwin" {{ rettype }} interposing_{{ func }} ({{ sig_str }}) { ### else {{ rettype }} {{ func }} ({{ sig_str }}) { ### endif bool skip_interception = false; switch (number) { #include "interceptor/gen_impl_syscalls.c.inc" default_syscall_handling: default: { /* Warm up */ int saved_errno = errno; if (!skip_interception && !ic_init_started) fb_ic_init(); /* use a copy, in case another thread modifies it */ bool i_am_intercepting = !skip_interception && intercepting_enabled; (void)i_am_intercepting; /* sometimes it's unused, silence warning */ #ifdef FB_EXTRA_DEBUG if (insert_trace_markers) { char debug_buf[256]; snprintf(debug_buf, sizeof(debug_buf), "%s%s{{ debug_before_fmt }}", i_am_intercepting ? "" : "[not intercepting] ", "{{ func }}"{{ debug_before_args }}); insert_begin_marker(debug_buf); } #endif /* Notify the supervisor */ bool i_locked = false; /* "i" as in "me, myself and I" */ if (!skip_interception && (number < 0 || number >= IC_CALLED_SYSCALL_SIZE || !ic_called_{{ func }}[number])) { /* Grabbing the global lock (unless it's already ours, e.g. we're in a signal handler) */ if (i_am_intercepting) { grab_global_lock(&i_locked, "{{ func }}"); } /* Global lock grabbed */ } /* Pass on several long parameters unchanged, see #178. */ va_list ap_pass; va_start(ap_pass, number); long arg1 = va_arg(ap_pass, long); long arg2 = va_arg(ap_pass, long); long arg3 = va_arg(ap_pass, long); long arg4 = va_arg(ap_pass, long); long arg5 = va_arg(ap_pass, long); long arg6 = va_arg(ap_pass, long); long arg7 = va_arg(ap_pass, long); long arg8 = va_arg(ap_pass, long); va_end(ap_pass); if (!skip_interception) { errno = saved_errno; } {{ rettype }} ret = get_ic_orig_{{ func }}()(number, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8); if (!skip_interception) { saved_errno = errno; if (number < 0 || number >= IC_CALLED_SYSCALL_SIZE || !ic_called_{{ func }}[number]) { if (number >= 0 && number < IC_CALLED_SYSCALL_SIZE) { ic_called_{{ func }}[number] = true; } FBBCOMM_Builder_gen_call ic_msg; fbbcomm_builder_gen_call_init(&ic_msg); char call[32]; ### if rettype == "long" snprintf(call, sizeof(call), "{{ func }}(%ld)", number); ### else snprintf(call, sizeof(call), "{{ func }}(%d)", number); ### endif fbbcomm_builder_gen_call_set_call(&ic_msg, call); fb_fbbcomm_send_msg(&ic_msg, fb_sv_conn); /* Releasing the global lock (if we grabbed it in this pass) */ if (i_locked) { release_global_lock(); } /* Global lock released */ } } #ifdef FB_EXTRA_DEBUG if (insert_trace_markers) { char debug_buf[256]; snprintf(debug_buf, sizeof(debug_buf), "%s%s{{ debug_after_fmt }}", i_am_intercepting ? "" : "[not intercepting] ", "{{ func }}"{{ debug_after_args }}); insert_end_marker(debug_buf); } #endif if (!skip_interception) { errno = saved_errno; } return ret; } } } #pragma GCC visibility pop ### if ifdef_guard #endif ### endif ### endblock impl_c firebuild-0.8.2/src/interceptor/tpl_system.c000066400000000000000000000046111447164520700211760ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for the system() call. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### block before { pthread_mutex_lock(&ic_system_popen_lock); /* Notify the supervisor before the call */ FBBCOMM_Builder_system ic_msg; fbbcomm_builder_system_init(&ic_msg); fbbcomm_builder_system_set_cmd(&ic_msg, cmd); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); } ### endblock before ### block call_orig ENVIRON_SAVE_AND_FIXUP(did_env_fixup, environ_saved); {{ super() }} ENVIRON_RESTORE(did_env_fixup, environ_saved); ### endblock call_orig ### block send_msg { /* Notify the supervisor after the call */ FBBCOMM_Builder_system_ret ic_msg; fbbcomm_builder_system_ret_init(&ic_msg); fbbcomm_builder_system_ret_set_cmd(&ic_msg, cmd); fbbcomm_builder_system_ret_set_ret(&ic_msg, ret); fbbcomm_builder_system_ret_set_error_no(&ic_msg, saved_errno); fb_fbbcomm_send_msg_and_check_ack(&ic_msg, fb_sv_conn); pthread_mutex_unlock(&ic_system_popen_lock); } ### endblock send_msg firebuild-0.8.2/src/interceptor/tpl_write.c000066400000000000000000000052721447164520700210100ustar00rootroot00000000000000{# ------------------------------------------------------------------ #} {# Copyright (c) 2022 Firebuild Inc. #} {# All rights reserved. #} {# Free for personal use and commercial trial. #} {# Non-trial commercial use requires licenses available from #} {# https://firebuild.com. #} {# Modification and redistribution are permitted, but commercial use #} {# of derivative works is subject to the same requirements of this #} {# license. #} {# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, #} {# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF #} {# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND #} {# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT #} {# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, #} {# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #} {# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER #} {# DEALINGS IN THE SOFTWARE. #} {# ------------------------------------------------------------------ #} {# Template for functions writing to a (regular or special) file, #} {# including #} {# - low-level [p]write*() family #} {# - high-level stdio like fwrite(), putc(), printf(), perror() etc. #} {# - low-level socket writing send*() family #} {# - ftruncate() #} {# and perhaps more. #} {# ------------------------------------------------------------------ #} ### extends "tpl.c" ### if is_pwrite is not defined ### set is_pwrite = "false" ### endif ### if msg_skip_fields is not defined ### set msg_skip_fields = [] ### endif ### do msg_skip_fields.append("error_no") {% set msg = "write_to_inherited" %} {# No locking around the write(): see issue #279 #} {% set global_lock = 'never' %} ### block set_fields {{ super() }} fbbcomm_builder_{{ msg }}_set_is_pwrite(&ic_msg, is_pwrite); ### endblock set_fields ### block send_msg bool is_pwrite = {{ is_pwrite }}; {# Acquire the lock if sending a message #} if (notify_on_write(fd, is_pwrite)) { /* Need to notify the supervisor */ {{ grab_lock_if_needed('true') | indent(2) }} {{ super() | indent(2) }} set_notify_on_write_state(fd, is_pwrite); {{ release_lock_if_needed() | indent(2) }} } ### endblock send_msg firebuild-0.8.2/test/000077500000000000000000000000001447164520700144575ustar00rootroot00000000000000firebuild-0.8.2/test/.gitignore000066400000000000000000000011201447164520700164410ustar00rootroot00000000000000Testing close_fds_exec data dirlist fbb_test fbb_test.bin fbb_test_builder_debug.txt fbb_test_serialized_debug.txt fbbtest.? fbbtest_decode.c firebuild-build-report.html firebuild-profile.* libc-*symbols.txt libc-symbols.txt parallel_make_test_main* parallel_make_test.?.txt public-symbols*.txt public-symbols.txt run-firebuild stderr test_cache_dir test_cmd_clone test_cmd_fork_exec test_cmd_popen test_cmd_posix_spawn test_cmd_system test_env_fixup test_err test_error test_exec test_file_ops test_file_ops_2 test_file_ops_3 test_helper.bash test_sendfile test_static test_system test_wait firebuild-0.8.2/test/CMakeLists.txt000066400000000000000000000062271447164520700172260ustar00rootroot00000000000000add_custom_command( OUTPUT fbbtest.cc fbbtest.h fbbtest_decode.c DEPENDS fbbtest.def ../src/common/fbb/generate_fbb ../src/common/fbb/tpl.c ../src/common/fbb/tpl.h WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ../src/common/fbb/generate_fbb fbbtest ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Generating fbbtest files") add_custom_target(fbbtest_gen_files ALL DEPENDS fbbtest.cc fbbtest.h fbbtest_decode.c) add_custom_target(check-bins) function(add_test_binary TEST_BINARY) add_executable("${TEST_BINARY}" EXCLUDE_FROM_ALL "${TEST_BINARY}.c") add_dependencies(check-bins "${TEST_BINARY}") endfunction() foreach(TESTFILE empty-config.conf integration.bats test_parallel_make.Makefile test_symbols) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${TESTFILE}" "${CMAKE_CURRENT_BINARY_DIR}/${TESTFILE}" COPYONLY) endforeach() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_helper.bash.in ${CMAKE_CURRENT_BINARY_DIR}/test_helper.bash @ONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/run-firebuild.in ${CMAKE_CURRENT_BINARY_DIR}/run-firebuild) add_test(bats-integration ./integration.bats) # firebuild's debug build would crash on the invalid entries in an assert() if (uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" OR CMAKE_C_FLAGS MATCHES "-DFB_EXTRA_DEBUG") set_property(TEST bats-integration PROPERTY ENVIRONMENT "SKIP_GC_INVALID_ENTRIES_TEST=1") endif() add_custom_target(check ctest -V) add_custom_target(valgrind-check env FIREBUILD_PREFIX_CMD='valgrind -q --leak-check=full --track-fds=yes --error-exitcode=1' ctest -V) add_test_binary(test_cmd_fork_exec) add_test_binary(test_cmd_popen) add_test_binary(test_cmd_posix_spawn) if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") add_test_binary(test_cmd_clone) endif() add_test_binary(test_cmd_system) add_test_binary(test_system) add_test_binary(test_exec) if (SANITIZE) set_source_files_properties(test_file_ops.c PROPERTIES COMPILE_DEFINITIONS "SKIP_TEST_NULL_NONNULL_PARAMS=1") endif() add_test_binary(test_file_ops) add_test_binary(test_file_ops_2) add_test_binary(test_file_ops_3) add_test_binary(test_random) set_source_files_properties(test_sendfile.c PROPERTIES COMPILE_FLAGS "-Wno-deprecated-declarations") add_test_binary(test_sendfile) add_test_binary(test_wait) add_test_binary(test_orphan) add_test_binary(close_fds_exec) add_test_binary(test_err) if(NOT APPLE) add_test_binary(test_error) endif() add_test_binary(test_env_fixup) if(NOT APPLE) add_test_binary(test_static) target_link_libraries(test_static "-static") endif() include_directories(${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src ${CMAKE_CURRENT_BINARY_DIR}) file(CREATE_LINK ${CMAKE_SOURCE_DIR}/data "${CMAKE_BINARY_DIR}/test/data" SYMBOLIC) set_source_files_properties(fbb_test.cc PROPERTIES COMPILE_FLAGS "-Wno-cast-align") add_executable(fbb_test EXCLUDE_FROM_ALL fbb_test.cc fbbtest.cc) add_dependencies(fbb_test fbbtest_gen_files) add_test(fbb ./fbb_test) add_custom_target(check-deps) add_dependencies(check-deps fbb_test) add_dependencies(check-deps firebuild) add_dependencies(check-deps firebuild-bin) add_dependencies(check check-deps check-bins) add_dependencies(valgrind-check check-deps check-bins) add_test(test-symbols ./test_symbols ${CMAKE_BINARY_DIR}) firebuild-0.8.2/test/CPPLINT.cfg000066400000000000000000000001371447164520700162520ustar00rootroot00000000000000filter=-readability/casting exclude_files=(fbb.*.[ch]|parallel_make_test_main.c|test_pch.[ch]) firebuild-0.8.2/test/close_fds_exec.c000066400000000000000000000017561447164520700176010ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include int main(int argc, char* argv[]) { (void) argc; /* unused */ int i; for (i = 3; i < 120; i++) { close(i); } execvp(argv[1], &argv[1]); } firebuild-0.8.2/test/empty-config.conf000066400000000000000000000003761447164520700177350ustar00rootroot00000000000000// (Almost) empty configuration file version = 1.0; // Those settings are changed in ./run-firebuild, thus they are needed to not break the test env_vars = { pass_through = [ "PATH", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"]; }; ignore_locations = []; firebuild-0.8.2/test/fbb_test.cc000066400000000000000000000232261447164520700165630ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* Cancel the gcc command line "-DNDEBUG", if present, * so that we can use assert() even with prod builds. */ #ifdef NDEBUG #undef NDEBUG #endif #include #include #include #include #include #include #include #include #include "./fbbtest.h" const char *my_stringarray_item_fn(int index, const void *user_data, fbb_size_t *len_out) { if (user_data != (void *) 42) { fprintf(stderr, "user_data mismatch\n"); exit(1); } if (index == 0) { if (len_out) { *len_out = 3; } return "one"; } else if (index == 1) { if (len_out) { *len_out = 3; } return "two"; } else if (index == 2) { if (len_out) { *len_out = 5; } return "three"; } else { if (len_out) { *len_out = 4; } return "four"; } } int main(int argc, char *argv[]) { (void)argc; (void)argv; /* construct the builder */ const int int_array[] = { 33 }; /* 8 bytes, so it won't get a padding i.e. a trailing '\0' */ const char char_array[] = { 'W', 'e', 'l', 'c', 'o', 'm', 'e', '!' }; const char *string_array[] = { "lorem1", /* trailing '\0' plus 1 byte padding */ "lorem02", /* trailing '\0' and no padding */ "lorem003", /* trailing '\0' plus 3 or 7 bytes of padding */ NULL }; const char *string_array2[] = { "the", "quick", "brown", "fox", NULL }; FBBTEST_Builder_testing builder; fbbtest_builder_testing_init(&builder); fbbtest_builder_testing_set_reqint(&builder, 42); fbbtest_builder_testing_set_optint(&builder, 100); fbbtest_builder_testing_set_arrint(&builder, int_array, 1 /* length */); fbbtest_builder_testing_set_reqchr(&builder, 'x'); /* leaving optchr unset */ fbbtest_builder_testing_set_arrchr(&builder, char_array, 8 /* length */); fbbtest_builder_testing_set_reqstr(&builder, "foo"); fbbtest_builder_testing_set_optstr(&builder, "quux"); fbbtest_builder_testing_set_arrstr(&builder, string_array); fbbtest_builder_testing_set_arrstr2(&builder, string_array2); FBBTEST_Builder_testing2 builder2; fbbtest_builder_testing2_init(&builder2); fbbtest_builder_testing2_set_t2(&builder2, 60); fbbtest_builder_testing_set_reqfbb(&builder, reinterpret_cast(&builder2)); FBBTEST_Builder_testing3 builder3; fbbtest_builder_testing3_init(&builder3); fbbtest_builder_testing_set_optfbb(&builder, reinterpret_cast(&builder3)); FBBTEST_Builder_testing builder4; fbbtest_builder_testing_init(&builder4); fbbtest_builder_testing_set_reqint(&builder4, 44); fbbtest_builder_testing_set_reqchr(&builder4, 'y'); fbbtest_builder_testing_set_reqstr(&builder4, "hi there"); fbbtest_builder_testing_set_arrstr_item_fn(&builder4, 4, my_stringarray_item_fn, reinterpret_cast(42)); fbbtest_builder_testing_set_reqfbb(&builder4, reinterpret_cast(&builder2)); FBBTEST_Builder_testing2 builder5; fbbtest_builder_testing2_init(&builder5); fbbtest_builder_testing2_set_t2(&builder5, 70); FBBTEST_Builder_testing2 builder6; fbbtest_builder_testing2_init(&builder6); FBBTEST_Builder **builder_array = reinterpret_cast(malloc(4 * sizeof(FBBTEST_Builder *))); builder_array[0] = reinterpret_cast(&builder4); builder_array[1] = reinterpret_cast(&builder5); builder_array[2] = reinterpret_cast(&builder6); builder_array[3] = NULL; fbbtest_builder_testing_set_arrfbb(&builder, builder_array); /* debug the builder to a file */ FILE *builder_debug_f = fopen("fbb_test_builder_debug.txt", "w"); assert(builder_debug_f != NULL); fbbtest_builder_debug(builder_debug_f, reinterpret_cast(&builder)); fclose(builder_debug_f); int builder_debug_fd = open("fbb_test_builder_debug.txt", O_RDONLY); assert(builder_debug_fd >= 0); struct stat builder_debug_st; assert(fstat(builder_debug_fd, &builder_debug_st) == 0); void *builder_debug_p = mmap(NULL, builder_debug_st.st_size, PROT_READ, MAP_SHARED, builder_debug_fd, 0); assert(builder_debug_p != MAP_FAILED); /* serialize to memory */ size_t len = fbbtest_builder_measure(reinterpret_cast(&builder)); char *p = reinterpret_cast(malloc(len)); assert(fbbtest_builder_serialize(reinterpret_cast(&builder), p) == len); /* dump to file for easier debugging */ int fd = open("fbb_test.bin", O_CREAT|O_RDWR|O_TRUNC, 0666); assert(fd >= 0); assert(write(fd, p, len) == static_cast(len)); close(fd); /* debug the serialized version to a file */ FILE *serialized_debug_f = fopen("fbb_test_serialized_debug.txt", "w"); assert(serialized_debug_f != NULL); fbbtest_serialized_debug(serialized_debug_f, reinterpret_cast(p)); fclose(serialized_debug_f); int serialized_debug_fd = open("fbb_test_serialized_debug.txt", O_RDONLY); assert(serialized_debug_fd >= 0); struct stat serialized_debug_st; assert(fstat(serialized_debug_fd, &serialized_debug_st) == 0); void *serialized_debug_p = mmap(NULL, serialized_debug_st.st_size, PROT_READ, MAP_SHARED, serialized_debug_fd, 0); assert(serialized_debug_p != MAP_FAILED); /* compare the two debug outputs */ assert(builder_debug_st.st_size == serialized_debug_st.st_size); assert(memcmp(builder_debug_p, serialized_debug_p, builder_debug_st.st_size) == 0); /* check the serialized version's fields manually */ assert(fbbtest_serialized_get_tag(reinterpret_cast(p)) == FBBTEST_TAG_testing); FBBTEST_Serialized_testing *msg = reinterpret_cast(p); /* check if optionals are set */ assert(fbbtest_serialized_testing_has_optint(msg)); assert(!fbbtest_serialized_testing_has_optchr(msg)); assert(fbbtest_serialized_testing_has_optstr(msg)); assert(fbbtest_serialized_testing_has_optfbb(msg)); /* check the values of required and optional scalars */ assert(fbbtest_serialized_testing_get_reqint(msg) == 42); assert(fbbtest_serialized_testing_get_optint(msg) == 100); assert(fbbtest_serialized_testing_get_reqchr(msg) == 'x'); assert(fbbtest_serialized_testing_get_reqstr_len(msg) == 3); assert(fbbtest_serialized_testing_get_optstr_len(msg) == 4); assert(strcmp(fbbtest_serialized_testing_get_reqstr(msg), "foo") == 0); assert(strcmp(fbbtest_serialized_testing_get_optstr(msg), "quux") == 0); /* check the values of arrays, C api */ assert(fbbtest_serialized_testing_get_arrint_count(msg) == 1); assert(fbbtest_serialized_testing_get_arrint_at(msg, 0) == 33); assert(fbbtest_serialized_testing_get_arrchr_count(msg) == 8); assert(strncmp(fbbtest_serialized_testing_get_arrchr(msg), "Welcome!", 8) == 0); assert(fbbtest_serialized_testing_get_arrstr_count(msg) == 3); assert(fbbtest_serialized_testing_get_arrstr_len_at(msg, 0) == 6); assert(fbbtest_serialized_testing_get_arrstr_len_at(msg, 1) == 7); assert(fbbtest_serialized_testing_get_arrstr_len_at(msg, 2) == 8); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(msg, 0), "lorem1") == 0); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(msg, 1), "lorem02") == 0); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(msg, 2), "lorem003") == 0); /* check the values of arrays, C++ api */ std::vector cxx_ints = fbbtest_serialized_testing_get_arrint_as_vector(msg); assert(cxx_ints.size() == 1); assert(cxx_ints[0] == 33); std::vector cxx_chars = fbbtest_serialized_testing_get_arrchr_as_vector(msg); assert(cxx_chars.size() == 8); assert(std::string(cxx_chars.begin(), cxx_chars.end()) == "Welcome!"); std::vector cxx_strings = fbbtest_serialized_testing_get_arrstr_as_vector(msg); assert(cxx_strings.size() == 3); assert(cxx_strings[0] == "lorem1"); assert(cxx_strings[1] == "lorem02"); assert(cxx_strings[2] == "lorem003"); /* check some of the embedded FBBs */ assert(fbbtest_serialized_testing_get_arrfbb_count(msg) == 3); const FBBTEST_Serialized *fbb0_generic = fbbtest_serialized_testing_get_arrfbb_at(msg, 0); assert(fbbtest_serialized_get_tag(fbb0_generic) == FBBTEST_TAG_testing); const FBBTEST_Serialized_testing *fbb0 = reinterpret_cast(fbb0_generic); assert(fbbtest_serialized_testing_get_arrstr_count(fbb0) == 4); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(fbb0, 0), "one") == 0); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(fbb0, 1), "two") == 0); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(fbb0, 2), "three") == 0); assert(strcmp(fbbtest_serialized_testing_get_arrstr_at(fbb0, 3), "four") == 0); printf("fbb testing succeeded\n"); return 0; } firebuild-0.8.2/test/fbbtest.def000066400000000000000000000031661447164520700165760ustar00rootroot00000000000000# Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # This is the definition of an FBB format used for testing FBB itself. # This is a Python dictionary, to be read and processed by "generate_fbb". { "tags": [ ("testing", [ (REQUIRED, "int", "reqint"), (OPTIONAL, "int", "optint"), (ARRAY, "int", "arrint"), (REQUIRED, "char", "reqchr"), (OPTIONAL, "char", "optchr"), (ARRAY, "char", "arrchr"), (REQUIRED, STRING, "reqstr"), (OPTIONAL, STRING, "optstr"), (ARRAY, STRING, "arrstr"), (ARRAY, STRING, "arrstr2"), (REQUIRED, FBB, "reqfbb"), (OPTIONAL, FBB, "optfbb"), (ARRAY, FBB, "arrfbb"), ]), ("testing2", [ (OPTIONAL, "int", "t2"), ]), ("testing3", [ (OPTIONAL, "int", "t3"), ]), ("testing4", [ (OPTIONAL, "int", "t4"), ]), ] } firebuild-0.8.2/test/integration.bats000077500000000000000000000453671447164520700176770ustar00rootroot00000000000000#!/usr/bin/env bats load test_helper setup() { rm -rf test_cache_dir } @test "--help" { result=$(./run-firebuild --help) echo "$result" | grep -q "in case of failure" } @test "empty-config" { result=$(./run-firebuild -c empty-config.conf -- ls integration.bats) assert_streq "$result" "integration.bats" assert_streq "$(strip_stderr stderr)" "" } @test "bash -c ls" { for i in 1 2; do result=$(./run-firebuild -o 'processes.dont_shortcut -= "ls"' -- bash -c "ls integration.bats") assert_streq "$result" "integration.bats" assert_streq "$(strip_stderr stderr)" "" done } @test "bash -c grep ok" { for i in 1 2; do result=$(echo -e "foo\nok\nbar" | ./run-firebuild -- bash -c "grep ok") assert_streq "$result" "ok" assert_streq "$(strip_stderr stderr)" "" done } @test "debugging with trace markers and report generation" { for i in 1 2; do result=$(./run-firebuild -o 'processes.dont_shortcut -= "ls"' -r -d all -i -- bash -c "ls integration.bats; bash -c ls | tee dirlist > /dev/null") assert_streq "$result" "$(printf 'integration.bats\nFIREBUILD: Generated report: firebuild-build-report.html')" done } @test "bash exec chain" { for i in 1 2; do result=$(./run-firebuild -o 'processes.dont_shortcut -= "ls"' -- bash -c "exec bash -c exec\\ bash\\ -c\\ ls\\\\\ integration.bats") assert_streq "$result" "integration.bats" assert_streq "$(strip_stderr stderr)" "" done } @test "dash bash exec chain with allow-list" { for i in 1 2; do result=$(./run-firebuild -d cache -o 'processes.dont_shortcut -= "ls"' -o 'processes.shortcut_allow_list += "bash"' -- dash -c "exec bash -c exec\\ bash\\ -c\\ ls\\\\\ integration.bats") assert_streq "$result" "integration.bats" assert_streq "$(strip_stderr stderr)" "" assert_streq "$(grep -h 'original_executed_path' test_cache_dir/objs/*/*/*/%_directory_debug.json | sed 's|/.*/||' | uniq -c)" ' 2 "original_executed_path": "bash",' done } @test "simple pipe" { for i in 1 2; do result=$(./run-firebuild -- bash -c 'seq 30000 | (sleep 0.01 && grep ^9)') assert_streq "$result" "$(seq 10000 | grep ^9)" assert_streq "$(strip_stderr stderr)" "" done } @test "yes | head" { for i in 1 2; do result=$(./run-firebuild -- bash -c 'yes 2> /dev/null | head -n 10000000 | tail -n 1') assert_streq "$result" "y" assert_streq "$(strip_stderr stderr)" "" done } @test "parallel make" { for i in 1 2; do # clean up previous run make -s -f test_parallel_make.Makefile clean result=$(./run-firebuild -o 'processes.jobserver_users += "make"' -- make -s -j8 -f test_parallel_make.Makefile) assert_streq "$result" "ok" assert_streq "$(strip_stderr stderr)" "" result=$(./run-firebuild -s) assert_streq "$(strip_stderr stderr)" "" done } @test "orphan processes" { # Orphan process detection does not work in WSL1 [ "$(uname)" = "Linux" ] && [ "$(systemd-detect-virt)" != "wsl" ] || skip for i in 1 2; do if ! with_valgrind; then result=$(./run-firebuild -o 'processes.dont_shortcut += "sleep"' -- bash -c 'for i in $(seq 10); do (sleep 0.3; ls integration.bats; false)& done; /bin/echo foo' | sort) else result=$(./run-firebuild -o 'processes.dont_shortcut += "sleep"' -- bash -c 'for i in $(seq 10); do (sleep 1; ls integration.bats; false)& done; /bin/echo foo' | sort) fi assert_streq "$result" "foo" if [ "$(uname)" = "Linux" ]; then assert_streq "$(strip_stderr stderr | uniq -c)" " 10 Orphan process has been killed by signal 15" else assert_streq "$(strip_stderr stderr)" "" fi result=$(./run-firebuild -- ./test_orphan) assert_streq "$result" "" if [ "$(uname)" = "Linux" ]; then # there may be one or two detected orphan processes assert_streq "$(strip_stderr stderr | uniq)" "Orphan process has been killed by signal 15" else assert_streq "$(strip_stderr stderr)" "" fi done } @test "system()" { for i in 1 2; do result=$(./run-firebuild -- ./test_system) assert_streq "$result" "ok" assert_streq "$(strip_stderr stderr)" "" done } @test "exec()" { for i in 1 2; do result=$(./run-firebuild -- ./test_exec) assert_streq "$result" "ok" assert_streq "$(strip_stderr stderr)" "" done } @test "closedir() inside an rm -r" { for i in 1 2; do result=$(./run-firebuild -- bash -c 'mkdir -p TeMp/FoO; rm -r TeMp') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" done } @test "file operations" { for i in 1 2; do # clean up before running the test rm -rf test_directory/ foo-dir/ result=$(./run-firebuild -- ./test_file_ops) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" # Due to the "again" parameter the 1st level cannot be shortcut in # the first iteration, but the 2nd level (./test_file_ops_2) can, # it should fetch the cached entries stored in the previous run. result=$(./run-firebuild -- ./test_file_ops again) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" # test sendfile and friends result=$(./run-firebuild -- bash -c './test_sendfile < integration.bats > foo') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" rm -f foo # The process can find a directory missing then can create a file in it due to the directory # having been created by an other parallel process result=$(./run-firebuild bash -c "(bash -c 'sh -c \"echo -x > foo-dir/bar1\" 2> /dev/null; sleep 0.2; sh -c \"echo x > foo-dir/bar2\" 2> /dev/null') & (sleep 0.1; [ $i == 2 ] || mkdir foo-dir); wait") assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" # Existing directories are not created again rm -rf foo-dir; mkdir foo-dir result=$(./run-firebuild bash -c "rm -rf foo-dir ; mkdir foo-dir") assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" # Two processes write a file in parallel # 1. One executed process writer is an ancestor of the other. result=$(./run-firebuild -- bash -c '( (sleep 0.1; echo 1) >> foo)& sh -c "(sleep 0.11; echo 1) >> foo" ; wait') # 2. The writers have a common ancestor. result=$(./run-firebuild -- bash -c 'for i in 1 2; do sh -c "(sleep 0.1$i; echo 1) >> foo" & done; wait') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" rm -f foo done } @test "randomness handling" { for i in 1 2; do result=$(./run-firebuild -o 'ignore_locations -= "/dev/urandom"' -- ./test_random) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" result=$(./run-firebuild -- ./test_random again) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" done } @test "waiting for a child" { for i in 1 2; do result=$(./run-firebuild -o 'processes.skip_cache -= "touch"' -- ./test_wait) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" # Due to the "again" parameter the outer process cannot be shortcut # in the first iteration, but the children can, they should fetch # the cached entries stored in the previous run. result=$(./run-firebuild -o 'processes.skip_cache -= "touch"' -- ./test_wait again) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" done } @test "err()" { stderr_expected=$'test_err: warn1: No such file or directory\ntest_err: warn2: Permission denied\ntest_err: err1: No such file or directory\natexit_handler' for i in 1 2; do result=$(./run-firebuild -- ./test_err || true) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "$stderr_expected" done } @test "error()" { [ -x ./test_error ] || skip stderr_expected=$'./test_error: error1: No such file or directory\n./test_error: error2: Permission denied\n./test_error: error3: No such file or directory\natexit_handler' for i in 1 2; do result=$(./run-firebuild -- ./test_error || true) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "$stderr_expected" done } @test "env fixup" { for i in 1 2; do result=$(./run-firebuild -- ./test_env_fixup) echo "$result" | grep -qx "AAA=aaa" echo "$result" | grep -qx "BBB=bbb" echo "$result" | grep -qx "LD_PRELOAD= LIBXXX.SO libfirebuild.so LIBYYY.SO " strip_stderr stderr | grep -q "ERROR: ld.so: object 'LIBXXX.SO' from LD_PRELOAD cannot be preloaded" strip_stderr stderr | grep -q "ERROR: ld.so: object 'LIBYYY.SO' from LD_PRELOAD cannot be preloaded" # Valgrind finds an error in fakeroot https://bugs.debian.org/983272 if fakeroot ls > /dev/null 2>&1; then if ! with_valgrind; then result=$(fakeroot ./run-firebuild -- id -u) assert_streq "$result" "0" result=$(./run-firebuild -- fakeroot id -u) assert_streq "$result" "0" fi result=$(./run-firebuild -- fakeroot id -u) assert_streq "$result" "0" fi done } @test "popen() a statically linked binary and a normal one" { ldd ./test_static 2>&1 | grep -Eq '(not a dynamic executable|statically linked)' for i in 1 2; do if [ -x ./test_static ]; then result=$(./run-firebuild -- ./test_cmd_popen ./test_static r) assert_streq "$result" "I am statically linked." assert_streq "$(strip_stderr stderr)" "" fi result=$(./run-firebuild -- bash -c "echo -e 'bar\nfoo\nbar' | ./test_cmd_popen 'grep foo' w") assert_streq "$result" "foo" assert_streq "$(strip_stderr stderr)" "" done } @test "fork() + exec() a statically linked binary" { [ -x ./test_static ] || skip ldd ./test_static 2>&1 | grep -Eq '(not a dynamic executable|statically linked)' for i in 1 2; do result=$(./run-firebuild -- ./test_cmd_fork_exec ./test_static) assert_streq "$result" "I am statically linked." assert_streq "$(strip_stderr stderr)" "" done # command substitution with statically linked binary for i in 1 2; do result=$(timeout 10 ./run-firebuild -- sh -c 'echo `./test_static`') assert_streq "$result" "I am statically linked." assert_streq "$(strip_stderr stderr)" "" done } @test "posix_spawn() a statically linked binary" { [ -x ./test_static ] || skip ldd ./test_static 2>&1 | grep -Eq '(not a dynamic executable|statically linked)' for i in 1 2; do result=$(./run-firebuild -- ./test_cmd_posix_spawn ./test_static) assert_streq "$result" "I am statically linked." assert_streq "$(strip_stderr stderr)" "" done } @test "clone() a statically linked binary" { [ -x ./test_cmd_clone ] || skip ldd ./test_static 2>&1 | grep -Eq '(not a dynamic executable|statically linked)' for i in 1 2; do result=$(./run-firebuild -- ./test_cmd_clone ./test_static | uniq -c) assert_streq "$result" " 2 I am statically linked." assert_streq "$(strip_stderr stderr)" "" done } @test "system() a statically linked binary" { [ -x ./test_static ] || skip ldd ./test_static 2>&1 | grep -Eq '(not a dynamic executable|statically linked)' for i in 1 2; do result=$(./run-firebuild -- ./test_cmd_system './test_static 3' | tr '\n' ' ') assert_streq "$result" "I am statically linked. end " assert_streq "$(strip_stderr stderr)" "" done } @test "pipe replaying" { result=$(./run-firebuild -o 'processes.skip_cache -= "echo"' -o 'min_cpu_time = -1.0' -- echo foo) assert_streq "$result" "foo" assert_streq "$(strip_stderr stderr)" "" # Poison the cache, pretending that the output was "quux" instead of "foo". # (Bash supports wildcard redirection to a single matching file.) echo quux > test_cache_dir/blobs/*/*/* # Replaying the "echo foo" command now needs to print "quux". result=$(./run-firebuild -o 'processes.skip_cache -= "echo"' -- echo foo) assert_streq "$result" "quux" assert_streq "$(strip_stderr stderr)" "" } @test "parallel sleeps" { for i in 1 2; do # Valgrind ignores the limit bumped internally in firebuild # See: https://bugs.kde.org/show_bug.cgi?id=432508 result=$(with_valgrind && ulimit -S -n 8000 ; ./run-firebuild -- bash -c 'for i in $(if [ "$(uname)" = "Darwin" ]; then seq 400; else seq 2000; fi); do sleep 1 & done; wait') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" done } @test "max entry size" { result=$(./run-firebuild -d caching -o 'processes.skip_cache -= "dd"' dd if=/dev/zero of=foo bs=1MB count=251) assert_streq "$result" "" assert_streq "$(strip_stderr stderr | grep max_entry)" "FIREBUILD: Could not store blob in cache because it would exceed max_entry_size" result=$(./run-firebuild -d caching -o 'processes.skip_cache -= "dd"' -o 'max_entry_size = 0.0042' dd if=/dev/zero of=foo bs=4k count=1) assert_streq "$result" "" assert_streq "$(strip_stderr stderr | grep max_entry)" "FIREBUILD: Could not store entry in cache because it would exceed max_entry_size" rm -f foo } @test "gc" { rm -f foo result=$(./run-firebuild -d cache -- bash -c 'echo foo > foo') assert_streq "$result" "" if [ "$SKIP_GC_INVALID_ENTRIES_TEST" != 1 ]; then echo foo > test_cache_dir/blobs/invalid_blob_name echo bar > test_cache_dir/objs/invalid_obj_name fi ln -s invalid_blob_name test_cache_dir/blobs/unexpected_symlink ln -s invalid_obj_name test_cache_dir/objs/unexpected_symlink mkdir test_cache_dir/blobs/to_be_removed test_cache_dir/objs/to_be_removed touch test_cache_dir/objs/to_be_removed/%_directory_debug.json mkdir test_cache_dir/objs/many-entries for i in $(seq -w 30); do cp test_cache_dir/objs/?/??/*/??????????? test_cache_dir/objs/many-entries/12345678${i}+ done # update cache size new_cache_size=$((cat test_cache_dir/size | tr '\n' ' ' ; printf '+ 30 *' ; (du --apparent-size -b test_cache_dir/objs/?/??/*/??????????? | cut -f1) ) | bc) echo $new_cache_size > test_cache_dir/size result=$(./run-firebuild -o 'shortcut_tries = 18' -d cache --gc) assert_streq "$result" "" if [ "$SKIP_GC_INVALID_ENTRIES_TEST" != 1 ]; then assert_streq "$(grep 'invalid_.*_name' stderr | wc -l)" "2" assert_streq "$(strip_stderr stderr | grep -v 'invalid_.*_name' | grep -v 'type is unexpected')" "FIREBUILD ERROR: There are 8 bytes in the cache stored in files with unexpected name." fi # debug files are kept with "-d cache" [ -f test_cache_dir/objs/*/*/*/%_directory_debug.json ] # there is a non-directory debug json file as well debug_json=$(ls test_cache_dir/objs/*/*/*/*_debug.json | grep -v %_directory) [ -f "${debug_json}" ] [ -f test_cache_dir/blobs/*/*/*_debug.txt ] # empty dirs were removed from blobs/, the one in objs/ is kept due to %_directory_debug.json result=$(find test_cache_dir/blobs -name 'to_be_removed') assert_streq "$result" "" assert_streq "$(ls test_cache_dir/objs/many-entries/ | wc -l)" "18" result=$( ./run-firebuild --gc) assert_streq "$result" "" if [ "$SKIP_GC_INVALID_ENTRIES_TEST" != 1 ]; then assert_streq "$(grep 'invalid_.*_name' stderr | wc -l)" "2" fi assert_streq "$(strip_stderr stderr | grep -v 'invalid_.*_name' | grep -v ' unexpected')" "" # debug files are deleted without "-d cache" result=$(find test_cache_dir/ -name '*debug*') assert_streq "$result" "" result=$(find test_cache_dir/ -name 'to_be_removed') assert_streq "$result" "" result=$( ./run-firebuild -o 'max_cache_size = 0.00002' --gc) cp -r test_cache_dir test_cache_dir.bak assert_streq "$result" "" assert_streq "$(strip_stderr stderr | grep -v 'invalid_.*_name' | grep -v ' unexpected')" "" rm -f foo } @test "cache-format" { result=$(./run-firebuild -d cache -- bash -c 'echo foo > foo') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" assert_streq "$(cat test_cache_dir/cache-format)" "1" # older cache versions are OK (assuming they are handled) echo 0 > test_cache_dir/cache-format result=$(./run-firebuild -d cache -- bash -c 'echo foo > foo') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" assert_streq "$(cat test_cache_dir/cache-format)" "0" # future cache versions prevent using the cache echo 2 > test_cache_dir/cache-format result=$(./run-firebuild -d cache -- bash -c 'echo foo > foo') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "FIREBUILD ERROR: Cache format version is not supported, not reading or writing the cache" # unknown cache versions prevent using the cache, too echo foo > test_cache_dir/cache-format result=$(./run-firebuild -d cache -- bash -c 'echo foo > foo') assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "FIREBUILD ERROR: Cache format version is not supported, not reading or writing the cache" } @test "stats" { # Populate the cache result=$(./run-firebuild -z -- bash -c 'ls integration.bats') result=$(./run-firebuild -o 'processes.dont_shortcut = []' -- bash -c 'ls integration.bats') # Use stats for current run result=$(./run-firebuild -s bash -c 'ls integration.bats' | sed 's/ */ /g;s/seconds/ms/;s/[0-9-][0-9\.]* ms/N ms/;s/[0-9-][0-9\.]* kB/N kB/') assert_streq "$result" "$(printf 'integration.bats\n\nStatistics of current run:\n Hits: 1 / 1 (100.00 %%)\n Misses: 0\n Uncacheable: 0\n GC runs: 0\nNewly cached: N kB\nSaved CPU time: N ms\n')" # use --show-stats together with --gc ... result=$(./run-firebuild --gc --show-stats | sed 's/ */ /g;s/seconds/ms/;s/[0-9-][0-9\.]* ms/N ms/;s/[0-9-][0-9\.]* kB/N kB/') assert_streq "$result" "$(printf 'Statistics of stored cache:\n Hits: 1 / 4 (25.00 %%)\n Misses: 3\n Uncacheable: 1\n GC runs: 1\nCache size: N kB\nSaved CPU time: N ms\n')" # ... and without --gc result=$(./run-firebuild -s | sed 's/ */ /g;s/seconds/ms/;s/[0-9-][0-9\.]* ms/N ms/;s/[0-9-][0-9\.]* kB/N kB/') assert_streq "$result" "$(printf 'Statistics of stored cache:\n Hits: 1 / 4 (25.00 %%)\n Misses: 3\n Uncacheable: 1\n GC runs: 1\nCache size: N kB\nSaved CPU time: N ms\n')" result=$(./run-firebuild -z) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" result=$(./run-firebuild -s | sed 's/ */ /g;s/seconds/ms/;s/[0-9-][0-9\.]* ms/N ms/;s/[0-9-][0-9\.]* kB/N kB/') assert_streq "$result" "$(printf 'Statistics of stored cache:\n Hits: 0 / 0 (0.00 %%)\n Misses: 0\n Uncacheable: 0\n GC runs: 0\nCache size: N kB\nSaved CPU time: N ms\n')" } @test "clang pch" { # this test is very slow under valgrind ! with_valgrind || skip for no_pch_param in "" "-fno-pch-timestamp"; do for i in 1 2; do rm -f test_pch.h.pch test_pch.*.s result=$(./run-firebuild -- clang -cc1 ${no_pch_param} $TEST_SOURCE_DIR/test_pch.h -emit-pch -o test_pch.h.pch) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" # not reproducible to check test_pch.h.pch's embedded timestamp again result=$(./run-firebuild -- clang -cc1 -include-pch test_pch.h.pch $TEST_SOURCE_DIR/test_pch.c -o test_pch.$i.s) assert_streq "$result" "" assert_streq "$(strip_stderr stderr)" "" sleep 0.01 touch test_pch.h rm -f test_pch.h.pch test_pch.*.s done done } firebuild-0.8.2/test/run-firebuild.in000077500000000000000000000011271447164520700175620ustar00rootroot00000000000000#!/bin/bash . @CMAKE_CURRENT_BINARY_DIR@/test_helper.bash if [ -z "$TEST_INSTALLED_FIREBUILD" ]; then export FIREBUILD_DATA_DIR="@CMAKE_SOURCE_DIR@/data" fi # run firebuild closing fd 3 which is opened by bats $FIREBUILD_PREFIX_CMD firebuild \ -c @CMAKE_BINARY_DIR@/etc/firebuild.conf \ -o 'env_vars.pass_through += "GCOV_PREFIX"' \ -o 'env_vars.pass_through += "GCOV_PREFIX_STRIP"' \ -o "ignore_locations += \"$GCOV_PREFIX\"" \ "$@" 2>stderr 3>&- exitcode=$? if [ $exitcode != 0 ]; then echo "firebuild exited with $exitcode, stderr is:" >&2 cat stderr >&2 fi exit $exitcode firebuild-0.8.2/test/test_cmd_clone.c000066400000000000000000000040101447164520700176000ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* Based on clone(2) man page. */ #define _GNU_SOURCE #include #include #include #include #include #include #define STACK_SIZE (1024 * 1024) /* Stack size for cloned child */ /* Performs a clone() / waitpid() pair. * The command executes and its parameter taken from the command line. * Returns 0 (unless an error occurred), not the command's exit code. */ int child(void *arg) { return execv(*(char**)arg, (char**)arg); } int main(int argc, char *argv[]) { if (argc < 2) { fprintf(stderr, "need at least 1 argument\n"); return 1; } char *stack, *stack_top; stack = malloc(STACK_SIZE); if (!stack) { perror("malloc"); return 1; } stack_top = stack + STACK_SIZE; /* This clone can be intercepted. */ int ret = clone(child, stack_top, CLONE_VFORK|SIGCHLD, &argv[1]); if (ret == -1) { errno = ret; perror("clone"); return 1; } /* This one disables interception. */ ret = clone(child, stack_top, CLONE_PTRACE|SIGCHLD, &argv[1]); if (ret == -1) { errno = ret; perror("clone"); return 1; } if (waitpid(ret, NULL, 0) < 0) { perror("waitpid"); return 1; } return 0; } firebuild-0.8.2/test/test_cmd_fork_exec.c000066400000000000000000000030341447164520700204520ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* Performs a fork(), execvp() in the child, waitpid() in the parent. * The command to execute and its parameters are taken from the command line, one by one. * Returns 0 (unless an error occurred), not the command's exit code. */ #include #include #include int main(int argc, char *argv[]) { if (argc < 2) { fprintf(stderr, "need at least 1 argument\n"); return 1; } pid_t pid = fork(); if (pid < 0) { perror("fork"); return 1; } if (pid == 0) { /* child */ execvp(argv[1], argv + 1); perror("execvp"); return 1; } else { /* parent */ if (waitpid(pid, NULL, 0) < 0) { perror("waitpid"); return 1; } return 0; } } firebuild-0.8.2/test/test_cmd_popen.c000066400000000000000000000032771447164520700176370ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* Performs a popen() / pclose() pair. * The command (to be passed to "sh -c") is taken from the first command line parameter, * the pipe type is taken from the second. * Returns 0 (unless an error occurred), not the command's exit code. */ #include #include #include int main(int argc, char *argv[]) { if (argc != 3) { fprintf(stderr, "need exactly 2 arguments\n"); return 1; } FILE *f = popen(argv[1], argv[2]); if (f == NULL) { perror("popen"); return 1; } char buf[4096]; ssize_t n; if ((fcntl(fileno(f), F_GETFL) & O_ACCMODE) == O_RDONLY) { while ((n = fread(buf, 1, sizeof(buf), f)) > 0) { fwrite(buf, 1, n, stdout); } } else { while ((n = fread(buf, 1, sizeof(buf), stdin)) > 0) { fwrite(buf, 1, n, f); } } if (pclose(f) < 0) { perror("pclose"); return 1; } return 0; } firebuild-0.8.2/test/test_cmd_posix_spawn.c000066400000000000000000000027631447164520700210670ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include /* Performs a posix_spawnp() / waitpid() pair. * The command to execute and its parameters are taken from the command line, one by one. * Returns 0 (unless an error occurred), not the command's exit code. */ extern char **environ; int main(int argc, char *argv[]) { if (argc < 2) { fprintf(stderr, "need at least 1 argument\n"); return 1; } pid_t pid; int ret = posix_spawnp(&pid, argv[1], NULL, NULL, argv + 1, environ); if (ret) { errno = ret; perror("posix_spawnp"); return 1; } if (waitpid(pid, NULL, 0) < 0) { perror("waitpid"); return 1; } return 0; } firebuild-0.8.2/test/test_cmd_system.c000066400000000000000000000022771447164520700200410ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* Performs a system() call. * The command (to be passed to "sh -c") is taken from the first command line parameter. * Returns 0 (unless an error occurred), not the command's exit code. */ #include #include int main(int argc, char *argv[]) { if (argc != 2) { fprintf(stderr, "need exactly 1 argument\n"); return 1; } return system(argv[1]); } firebuild-0.8.2/test/test_env_fixup.c000066400000000000000000000021341447164520700176650ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ extern char **environ; #include #include #include int main() { char *myenv[] = { "AAA=aaa", "LD_PRELOAD= LIBXXX.SO libfirebuild.so LIBYYY.SO ", NULL }; environ = myenv; setenv("BBB", "bbb", 0); return system("printenv"); } firebuild-0.8.2/test/test_err.c000066400000000000000000000023001447164520700164450ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include void atexit_handler() { fprintf(stderr, "atexit_handler\n"); } int main() { atexit(atexit_handler); errno = ENOENT; warn("warn%d", 1); errno = EACCES; warn("warn%d", 2); errno = ENOENT; err(1, "err%d", 1); /* should not reach here */ errno = EACCES; err(1, "err%d", 2); return 0; } firebuild-0.8.2/test/test_error.c000066400000000000000000000024551447164520700170210ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #ifdef __APPLE__ #include #include #else #include #endif #include #include void atexit_handler() { fprintf(stderr, "atexit_handler\n"); } int main() { atexit(atexit_handler); #ifdef __APPLE__ err(1, "error%d", ENOENT); #else error(0, ENOENT, "error%d", 1); error(0, EACCES, "error%d", 2); error(1, ENOENT, "error%d", 3); /* should not reach here */ error(1, EACCES, "error%d", 4); #endif return 0; } firebuild-0.8.2/test/test_exec.c000066400000000000000000000017151447164520700166120ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include int main() { char * const argv[] = {"echo", "ok", NULL}; execvp("foo", argv); execvp("echo", argv); return 0; } firebuild-0.8.2/test/test_file_ops.c000066400000000000000000000143251447164520700174670ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #ifdef __linux__ #include #endif #include #ifdef __linux__ #include #endif #include #include #ifdef __linux__ #include #endif #include #include #define TOSTR(x) TOSTR2(x) #define TOSTR2(x) #x #define LOC "[" __FILE__ ":" TOSTR(__LINE__) "]" int main() { int fd, fd_dup, fd_dup2, i, pipe_fds[2]; struct stat st_buf; /* Close invalid file descriptior. Should not affect shortcutting. */ close(-1); #ifdef __linux__ if (pipe2(pipe_fds, 0) != 0) { perror("pipe2" LOC); #else if (pipe(pipe_fds) != 0) { perror("pipe" LOC); #endif exit(1); } else { close(pipe_fds[0]); close(pipe_fds[1]); } /* Set up some files for test_file_ops_[23]. */ fd = creat("test_empty_1.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } if (fstat(fd, &st_buf) != 0) { perror("stat" LOC); exit(1); } fd_dup = dup(fd); if (fd_dup == -1) { perror("dup" LOC); exit(1); } fd_dup2 = dup2(fd, fd_dup); if (fd_dup2 == -1) { perror("dup2" LOC); exit(1); } #ifdef __linux__ int fd_dup3; fd_dup3 = dup3(fd, fd_dup2, O_CLOEXEC); if (fd_dup3 == -1) { perror("dup3" LOC); exit(1); } close(fd); #endif fd = creat("test_empty_2.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); const char *msg = "Hello World!\n"; fd = creat("test_nonempty_1.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } i = write(fd, msg, strlen(msg)); if (i != (int) strlen(msg)) { perror("write" LOC); exit(1); } close(fd); fd = creat("test_nonempty_2.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } i = write(fd, msg, strlen(msg)); if (i != (int) strlen(msg)) { perror("write" LOC); exit(1); } close(fd); /* Only create _1, and not _2. */ fd = creat("test_maybe_exists_1.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); DIR *d = opendir("./"); if (d == NULL) { perror("opendir" LOC); exit(1); } closedir(d); if (mkdir("test_directory", 0700) == -1) { perror("mkdir" LOC); exit(1); } #ifdef O_TMPFILE fd = open("test_directory", O_RDWR | O_TMPFILE, 0744); /* Error on WSL1 is EISDIR. */ if (fd == -1 && (errno != ENOTSUP && errno != EISDIR)) { perror("open(..., O_TMPFILE)" LOC); exit(1); } close(fd); #endif char tmp_file[] = "tmpprefixXXXXXX"; fd = mkstemp(tmp_file); if (fd == -1) { perror("mkstemp" LOC); exit(1); } close(fd); unlink(tmp_file); char tmp_dir[] = "./prefixXXXXXX"; char *mkdtemp_ret = mkdtemp(tmp_dir); if (!mkdtemp_ret) { perror("mkdtemp" LOC); exit(1); } rmdir(mkdtemp_ret); #ifdef __linux__ fd = memfd_create("foo", MFD_CLOEXEC); if (fd == -1 && errno != ENOSYS) { perror("memfd_create" LOC); exit(1); } close(fd); fd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); if (fd == -1 && errno != ENOSYS) { perror("timerfd_create" LOC); exit(1); } close(fd); fd = eventfd(0, EFD_CLOEXEC); if (fd == -1 && errno != ENOSYS) { perror("eventfd" LOC); exit(1); } close(fd); sigset_t mask; sigemptyset(&mask); fd = signalfd(-1, &mask, SFD_CLOEXEC); if (fd == -1) { perror("signalfd" LOC); exit(1); } close(fd); #endif fd = socket(AF_INET, SOCK_DGRAM, 0); if (fd == -1) { perror("socket" LOC); exit(1); } close(fd); if (socketpair(AF_UNIX, SOCK_STREAM #ifdef SOCK_CLOEXEC | SOCK_CLOEXEC #endif , 0, pipe_fds) != 0) { perror("socketpair" LOC); exit(1); } else { close(pipe_fds[0]); close(pipe_fds[1]); } #ifdef STATX_TYPE /* Allow skipping this test since the ASAN & UBSAN build finds out that this code is incorrect. */ #ifndef SKIP_TEST_NULL_NONNULL_PARAMS #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wnonnull" /* Call statx with invalid parameters, like cargo does. */ statx(0, NULL, 0, STATX_ALL, NULL); #pragma GCC diagnostic pop #endif #endif if (stat(".", &st_buf) != 0) { perror("stat" LOC); exit(1); } if (stat("test_file_ops", &st_buf) != 0) { perror("stat" LOC); exit(1); } if (stat("stat_nonexistent", &st_buf) == 0) { fprintf(stderr, "stat() found unexpected file/dir"); exit(1); } if (system(NULL) == 0) { exit(1); } /* Run part 2. */ fd = open("test_nonempty_2.txt", O_WRONLY); lseek(fd, -2, SEEK_END); if (system("./test_file_ops_2") != 0) { exit(1); } close(fd); /* Cleanup. */ if (unlink("test_empty_1.txt") != 0) { perror("unlink" LOC); exit(1); } if (unlink("test_empty_2.txt") != 0) { perror("unlink" LOC); exit(1); } if (unlink("test_nonempty_1.txt") != 0) { perror("unlink" LOC); exit(1); } if (unlink("test_nonempty_2.txt") != 0) { perror("unlink" LOC); exit(1); } if (unlink("test_maybe_exists_1.txt") != 0) { perror("unlink" LOC); exit(1); } if (unlink("test_maybe_exists_2.txt") != 0) { perror("unlink" LOC); exit(1); } if (unlink("test_exclusive.txt") != 0) { perror("unlink" LOC); exit(1); } if (rmdir("test_directory") != 0) { perror("unlink" LOC); exit(1); } return 0; } firebuild-0.8.2/test/test_file_ops_2.c000066400000000000000000000021411447164520700177010ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include int main() { /* * Just run test_file_ops_3. * * The point is to test that file events propagate upwards from * test_file_ops_3 to this test_file_ops_2 correctly. */ execl("./test_file_ops_3", "test_file_ops", NULL); exit(1); } firebuild-0.8.2/test/test_file_ops_3.c000066400000000000000000000052231447164520700177060ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #include #include #define TOSTR(x) TOSTR2(x) #define TOSTR2(x) #x #define LOC "[" __FILE__ ":" TOSTR(__LINE__) "]" int main() { int fd; /* Open existing file for reading. */ fd = open("/etc/passwd", O_RDONLY); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); /* Attempt to open nonexisting file for reading. */ fd = open("/no/such/file", O_RDONLY); if (fd != -1) { fprintf(stderr, "open" LOC " should have failed\n"); exit(1); } /* Attempt to write to nonexisting file, without O_CREAT. */ fd = open("wont_create_1", O_WRONLY|O_TRUNC); if (fd != -1) { fprintf(stderr, "open" LOC " should have failed\n"); exit(1); } /* Attempt to write to existing file, with O_EXCL. */ fd = open("test_empty_1.txt", O_WRONLY|O_CREAT|O_EXCL, 0600); if (fd != -1) { fprintf(stderr, "open" LOC " should have failed\n"); exit(1); } /* Open for writing, but don't modify. */ fd = open("test_nonempty_1.txt", O_WRONLY); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); /* Open for writing, truncate. */ fd = open("test_nonempty_2.txt", O_WRONLY|O_TRUNC); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); /* Open for writing existing file, with CREAT and TRUNC. */ fd = creat("test_maybe_exists_1.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); /* Open for writing nonexisting file, with CREAT and TRUNC. */ fd = creat("test_maybe_exists_2.txt", 0600); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); /* Exclusive creation. */ fd = open("test_exclusive.txt", O_WRONLY|O_CREAT|O_EXCL, 0600); if (fd == -1) { perror("open" LOC); exit(1); } close(fd); return 0; } firebuild-0.8.2/test/test_helper.bash.in000066400000000000000000000014611447164520700202430ustar00rootroot00000000000000#!/bin/bash if [ -z "$TEST_INSTALLED_FIREBUILD" ] ; then export @CMAKE_TARGET_LD_LIBRARY_PATH@=@CMAKE_BINARY_DIR@/src/interceptor:$@CMAKE_TARGET_LD_LIBRARY_PATH@ export PATH=@CMAKE_BINARY_DIR@/src/firebuild:$PATH fi export FIREBUILD_CACHE_DIR=@CMAKE_CURRENT_BINARY_DIR@/test_cache_dir export GCOV_PREFIX=@CMAKE_BINARY_DIR@/gcov export GCOV_PREFIX_STRIP=$(echo @CMAKE_BINARY_DIR@ | tr -dc / | wc -c) export TEST_SOURCE_DIR=@CMAKE_SOURCE_DIR@/test function strip_stderr () { awk '/^==[0-9]*== $/ {next} /FILE DESCRIPTORS: 0 open at exit/ {next} {print}' < $1 } function assert_streq () { # Note: bats catches and only prints this in case of failure. echo "assert_streq: [ \"$1\" = \"$2\" ]" [ "$1" = "$2" ] } function with_valgrind () { set | grep -q '^FIREBUILD_PREFIX_CMD=.*valgrind' } firebuild-0.8.2/test/test_orphan.c000066400000000000000000000024301447164520700171500ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* * Test for forking orphan children. */ #include #include int main() { /* Test not waiting, but orphaning a zombie child that quits early. */ if (fork() > 0) { usleep(100*1000); } else { return 0; } /* Test not waiting, but orphaning a child that quits later. */ if (fork() == 0) { usleep(100*1000); return 0; } /* Test forking an orphan that quits almost the same time as the parent. */ fork(); return 0; } firebuild-0.8.2/test/test_parallel_make.Makefile000066400000000000000000000011101447164520700217370ustar00rootroot00000000000000check: parallel_make_test_main ./$< parallel_make_test.%.c: printf "void _"$(subst .,_,$@)"(){}" > $@ parallel_make_test_main.c: printf "#include \nint main() {printf(\"ok\\\n\"); return 0;}" > $@ # Note the '+' prefix before the commad. This makes make keep the jobserver pipe open even when not invoking a sub-make. # See: https://savannah.gnu.org/bugs/?47392 parallel_make_test_main: parallel_make_test_main.c $(patsubst %,parallel_make_test.%.o,$(shell seq 8)) +gcc -flto=auto -o $@ $^ clean: rm -f parallel_make_test.*.? parallel_make_test_main* .PHONY: clean firebuild-0.8.2/test/test_pch.c000066400000000000000000000015611447164520700164370ustar00rootroot00000000000000/* * Copyright (c) 2023 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "test_pch.h" int main() { return 0; } firebuild-0.8.2/test/test_pch.h000066400000000000000000000016241447164520700164440ustar00rootroot00000000000000/* * Copyright (c) 2023 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #ifndef TEST_PCH_H_ #define TEST_PCH_H_ #define PCH_TEST 1 #endif // TEST_PCH_H_ firebuild-0.8.2/test/test_random.c000066400000000000000000000026541447164520700171510ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include #include #include #include #define TOSTR(x) TOSTR2(x) #define TOSTR2(x) #x #define LOC "[" __FILE__ ":" TOSTR(__LINE__) "]" int main() { #ifdef __linux__ char buf[4]; ssize_t ret_len = getrandom(buf, sizeof(buf), 0); if (ret_len == -1) { perror("getrandom" LOC); exit(1); } assert(ret_len == sizeof(buf)); ret_len = getrandom(buf, sizeof(buf), GRND_RANDOM); if (ret_len == -1) { perror("getrandom" LOC); exit(1); } assert(ret_len <= (ssize_t) sizeof(buf)); /* With GRND_RANDOM, short read is possible. */ #endif return 0; } firebuild-0.8.2/test/test_sendfile.c000066400000000000000000000055341447164520700174620ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #define _GNU_SOURCE #include #include #include #include #ifdef __APPLE__ #include #include #else #include #endif #include #include #include #include #define TOSTR(x) TOSTR2(x) #define TOSTR2(x) #x #define LOC "[" __FILE__ ":" TOSTR(__LINE__) "]" int main() { int fd1, fd2; #ifdef O_TMPFILE fd1 = open(".", O_RDWR | O_TMPFILE, 0644); if (fd1 == -1) { if (errno == ENOTSUP || errno == EISDIR) { return 0; } else { perror("open" LOC); exit(1); } } #else /* sendfile() will fail, but that still excercises most of the code. */ // TODO(rbalint) create socket and test with that fd1 = 0; #endif fd2 = open("integration.bats", O_RDWR); if (fd2 == -1) { perror("open" LOC); close(fd1); exit(1); } #ifdef __APPLE__ off_t len = 10; if (sendfile(fd2, fd1, 0, &len, NULL, 0) == -1) { #else if (sendfile(fd1, fd2, NULL, 10) == -1) { #endif perror("sendfile" LOC); close(fd1); close(fd2); exit(1); } if (syscall(SYS_sendfile, fd1, fd2, NULL, 10) == -1) { perror("SYS_sendfile" LOC); close(fd1); close(fd2); exit(1); } #ifndef __APPLE__ if (copy_file_range(fd2, NULL, fd1, NULL, 10, 0) == -1) { perror("copy_file_range" LOC); close(fd1); close(fd2); exit(1); } /* test inherited fds*/ loff_t long_offset = 0; if (copy_file_range(0, NULL, 1, &long_offset, 10, 0) == -1) { perror("copy_file_range" LOC); close(fd1); close(fd2); exit(1); } if (copy_file_range(0, &long_offset, 1, NULL, 10, 0) == -1) { perror("copy_file_range" LOC); close(fd1); close(fd2); exit(1); } if (sendfile(1, 0, NULL, 10) == -1) { perror("sendfile" LOC); close(fd1); close(fd2); exit(1); } off_t offset = 0; if (sendfile(1, 0, &offset, 10) == -1) { perror("sendfile" LOC); close(fd1); close(fd2); exit(1); } #endif close(fd1); close(fd2); return 0; } firebuild-0.8.2/test/test_static.c000066400000000000000000000025201447164520700171500ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* This binary is meant to be statically linked. */ #include #include #include #include #include int main(int argc, char *argv[]) { puts("I am statically linked."); fflush(stdout); if (argc > 1) { int recurse_level = 0; sscanf(argv[1], "%d", &recurse_level); while (recurse_level-- > 0) { pid_t child_pid = fork(); if (child_pid > 0) { wait(NULL); return 0; } } return system("echo end"); } return 0; } firebuild-0.8.2/test/test_symbols000077500000000000000000000126351447164520700171430ustar00rootroot00000000000000#!/bin/bash # Check whether our preload library has any unexpected public symbol. # Check that all our overridden methods actually exist in libc, libdl or libpthread. # Check that for each overridden method we also override its fortified # counterpart, if that exists at all. # Copyright (c) 2022 Firebuild Inc. # All rights reserved. # Free for personal use and commercial trial. # Non-trial commercial use requires licenses available from https://firebuild.com. # Modification and redistribution are permitted, but commercial use of # derivative works is subject to the same requirements of this license # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # Figure out CMAKE_BINARY_DIR if [ $# -lt 1 ]; then echo "Usage: test_symbols binary_dir" >&2 exit 1 fi binary_dir="$1" status=0 # Additional allowed public symbols that are not auto-generated into gen_list.txt. additional_allowed_symbols="" function check_function_class () { local fix=$1 local class=$2 local kind=$3 local base # Get the list of libc, libdl and libpthread symbols of this class case $kind in "prefix") grep ^${fix} < libc-symbols.txt > libc-${class}-symbols.txt ;; "postfix") grep ${fix}'$' < libc-symbols.txt > libc-${class}-symbols.txt esac # Get the list of libc symbols that we don't override, but override the # counterpart with the prefix/postfix local missing=$(for chk in $(< libc-${class}-symbols.txt); do case $kind in "prefix") base="${chk#__}" if grep -Fqx "$base" public-symbols.txt && ! grep -Fqx "$chk" public-symbols.txt; then echo "$chk" fi ;; "postfix") base="${chk%${fix}}" base="${base#__}" if grep -Fqx "$base" public-symbols.txt && ! grep -Fqx "$chk" public-symbols.txt; then echo "$chk" fi ;; esac done) # Report the non-overridden symbols if [ -z "$missing" ]; then echo "All expected ${class} methods overridden" else echo "${class@u} methods not overridden:" echo "$missing" | sed 's/^\(.*\)$/ \1/' status=1 fi } # Extract and sort the public symbols of our library, # filtering out the ones added by gcov (__gcov* and mangle_path) # and ones appearing with old toolchains(__bss_start, _edata, etc.) nm -D "$binary_dir/src/interceptor/libfirebuild.so" | \ grep -v ' [Uvw] ' | \ cut -d' ' -f3- | \ grep -v '^__gcov_' | \ grep -v '^mangle_path$' | \ egrep -v '^_(_bss_start|edata|end|fini|init)$' | \ LC_ALL=C sort | uniq > public-symbols.txt # Gather and sort the allowed public symbols { cat "$binary_dir/src/interceptor/gen_list.txt"; echo "$additional_allowed_symbols"; } | \ LC_ALL=C sort > public-symbols-allowed.txt # Get the list of unexpected ones unexpected=$(LC_ALL=C comm -23 public-symbols.txt public-symbols-allowed.txt) # Report the list of unexpected ones if [ -z "$unexpected" ]; then echo "No unexpected public symbols" else echo "Unexpected public symbols:" echo "$unexpected" | sed 's/^\(.*\)$/ \1/' status=1 fi # Get the list of libc, libdl and libpthread symbols. # Some symbols have been removed from or introduced recently to glibc known_extra=" __bss_start _edata _end _fini _init _Fork arc4random arc4random_buf arc4random_uniform execveat stime ustat stat stat64 lstat lstat64 fstat fstat64 fstatat fstatat64 mknod mknodat closefrom close_range pidfd_open posix_spawn_file_actions_addclosefrom_np shm_open shm_unlink " # TODO(rbalint) provide properly versioned symbols in libfirebuild libs=$(LD_PRELOAD=libpthread.so.0 ldd "$binary_dir/src/interceptor/libfirebuild.so" | grep -E 'lib(c|dl|pthread).so' | cut -d' ' -f3) (nm -D $libs | \ grep ' [TWi] ' | \ cut -d' ' -f3- ; \ echo "$known_extra") | \ sed 's/\(.*\)@@\(.*\)/\1@@\2\n\1/' | \ LC_ALL=C sort > libc-symbols.txt # Get the list of extra ones extra=$(LC_ALL=C comm -23 public-symbols.txt libc-symbols.txt) # Report the list of extra ones if [ -z "$extra" ]; then echo "No unexpected overridden methods" else echo "Overridden methods that do not exist in libc, libdl or libpthread:" echo "$extra" | sed 's/^\(.*\)$/ \1/' status=1 fi # Check the list of libc, libdl and libpthread fortified symbols check_function_class "_chk" "fortified" "postfix" # Check the *64 variants check_function_class "64" "64bit" "postfix" # Check the *_time64 variants check_function_class "_time64" "time64" "postfix" # Check the __* variants check_function_class "__" "underscore" "prefix" exit $status firebuild-0.8.2/test/test_system.c000066400000000000000000000016001447164520700172030ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include int main() { return system("echo ok"); } firebuild-0.8.2/test/test_wait.c000066400000000000000000000132411447164520700166270ustar00rootroot00000000000000/* * Copyright (c) 2022 Firebuild Inc. * All rights reserved. * * Free for personal use and commercial trial. * Non-trial commercial use requires licenses available from https://firebuild.com. * Modification and redistribution are permitted, but commercial use of derivative * works is subject to the same requirements of this license * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* Test for #185: a system(), pclose(), wait(), waitpid() has to * wait not only for the process to terminate, but also for the * supervisor to do the administration, check the state of the * files as produced by that child. If we allowed the parent to * continue, it might modify the files and thus the wrong actions * would be recorded for the child, causing problems when we * shortcut it the next time. */ #include #include #include #include #include #include #include #include extern char **environ; #define TOSTR(x) TOSTR2(x) #define TOSTR2(x) #x #define LOC "[" __FILE__ ":" TOSTR(__LINE__) "]" int main() { int fd; FILE *f, *f2; pid_t pid; siginfo_t info; /* Test waiting at system() */ if ((fd = creat("test_wait_system.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if (system("exec touch test_wait_system.txt") != 0) { perror("system" LOC); exit(1); } if (unlink("test_wait_system.txt") != 0) { perror("unlink" LOC); exit(1); } /* Test waiting at pclose() */ if ((fd = creat("test_wait_pclose.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if ((f = popen("exec touch test_wait_pclose.txt", "w")) == NULL) { perror("popen" LOC); exit(1); } /* Run popen again to excercise supervisor tracking f to be closed in the new child. */ if ((f2 = popen("exec touch test_wait_pclose.txt", "r")) == NULL) { perror("popen" LOC); exit(1); } if (pclose(f) != 0) { perror("pclose" LOC); exit(1); } if (pclose(f2) != 0) { perror("pclose" LOC); exit(1); } if (unlink("test_wait_pclose.txt") != 0) { perror("unlink" LOC); exit(1); } /* Test waiting at wait() */ if ((fd = creat("test_wait_wait.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if (posix_spawn(&pid, "/bin/touch", NULL, NULL, (char *const[]){ "touch", "test_wait_wait.txt", NULL }, environ) != 0) { perror("posix_spawn" LOC); exit(1); } if (wait(NULL) != pid) { perror("wait" LOC); exit(1); } if (unlink("test_wait_wait.txt") != 0) { perror("unlink" LOC); exit(1); } /* Test waiting at waitpid() */ if ((fd = creat("test_wait_waitpid.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if (posix_spawn(&pid, "/bin/touch", NULL, NULL, (char *const[]){ "touch", "test_wait_waitpid.txt", NULL }, environ) != 0) { perror("posix_spawn" LOC); exit(1); } if (waitpid(pid, NULL, 0) != pid) { perror("waitpid" LOC); exit(1); } if (unlink("test_wait_waitpid.txt") != 0) { perror("unlink" LOC); exit(1); } /* Test waiting at wait3() */ if ((fd = creat("test_wait_wait3.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if (posix_spawn(&pid, "/bin/touch", NULL, NULL, (char *const[]){ "touch", "test_wait_wait3.txt", NULL }, environ) != 0) { perror("posix_spawn" LOC); exit(1); } if (wait3(NULL, 0, NULL) != pid) { perror("wait3" LOC); exit(1); } if (unlink("test_wait_wait3.txt") != 0) { perror("unlink" LOC); exit(1); } /* Test waiting at wait4() */ if ((fd = creat("test_wait_wait4.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if (posix_spawn(&pid, "/bin/touch", NULL, NULL, (char *const[]){ "touch", "test_wait_wait4.txt", NULL }, environ) != 0) { perror("posix_spawn" LOC); exit(1); } if (wait4(pid, NULL, 0, NULL) != pid) { perror("wait4" LOC); exit(1); } if (unlink("test_wait_wait4.txt") != 0) { perror("unlink" LOC); exit(1); } /* Test waiting at waitid() */ if ((fd = creat("test_wait_waitid.txt", 0600)) < 0) { perror("open" LOC); exit(1); } if (close(fd) != 0) { perror("close" LOC); exit(1); } if (posix_spawn(&pid, "/bin/touch", NULL, NULL, (char *const[]){"touch", "test_wait_waitid.txt", NULL}, environ) != 0) { perror("posix_spawn" LOC); exit(1); } if (waitid(P_PID, pid, &info, WEXITED) != 0) { perror("waitid" LOC); exit(1); } if (info.si_pid != pid) { fprintf(stderr, "waitid returned unexpected pid" LOC); exit(1); } if (unlink("test_wait_waitid.txt") != 0) { perror("unlink" LOC); exit(1); } return 0; } firebuild-0.8.2/tools/000077500000000000000000000000001447164520700146405ustar00rootroot00000000000000firebuild-0.8.2/tools/calculate-coverage000077500000000000000000000011241447164520700203120ustar00rootroot00000000000000#!/bin/sh set -e find . -name '*.gcno' | grep -v /gcov/ | while read f; do mkdir -p gcov/$(dirname $f) ln -sf $(realpath $f) gcov/$f done lcov -q -c -d . --no-external -o gcov/tmp.info lcov -q -r gcov/tmp.info -o gcov/coverage-incl-tests.info '*_generated.h' '*/fbb*' '*/interceptor/gen_*.*' genhtml -o gcovhtml gcov/coverage-incl-tests.info lcov -q -r gcov/coverage-incl-tests.info -o gcov/coverage-excl-tests.info '*/test/*.c*' echo "Line coverage rate, excluding test files:" lcov --summary gcov/coverage-excl-tests.info 2>&1 | grep '^ lines' | sed 's/ lines.*: \([^%]*\)%.*/\1/' firebuild-0.8.2/tools/interceptor-strace-highlight000077500000000000000000000020061447164520700223460ustar00rootroot00000000000000#!/usr/bin/awk -f # Colorizer for strace logs with interceptor trace marks. # # Please don't carry attributes (colors etc.) across newlines, # "less -R" doesn't support that. # # FIXME: Make the regexps more robust to eliminate false positives, # e.g. when a string parameter happens to contain "fork(" or such. { if ($1 ~ /^[0-9]+$/) { pid = $1 # the real PID from "strace -f" } else { pid = 0 # a single fake PID for "strace" / "strace -ff" } } /FIREBUILD.*intercept-begin/ { print "\x1b[36m" $0 "\x1b[m" # cyan pid_is_intercepted[pid] = 1 next } /FIREBUILD.*intercept-end/ { print "\x1b[36m" $0 "\x1b[m" # cyan pid_is_intercepted[pid] = 0 next } /FIREBUILD/ { print "\x1b[35m" $0 "\x1b[m" # magenta next } /execve\(/ { pid_is_intercepted[pid] = 0 } /fork\(/ || /clone\(/ { pid_is_intercepted[$NF] = pid_is_intercepted[pid] } pid_is_intercepted[pid] == 1 { print "\x1b[91m" $0 "\x1b[m" # bright red } pid_is_intercepted[pid] != 1 { print $0 # default } firebuild-0.8.2/tools/rebuild-self000077500000000000000000000006611447164520700171460ustar00rootroot00000000000000#!/bin/bash set -e if [ -e CMakeCache.txt ]; then echo "Please remove generated cmake files from the top dir" >&2 exit 1 fi build_dir="${1:-build-self-dir}" mkdir ${build_dir}-first-build cd ${build_dir}-first-build cmake .. make all cd test mkdir ../../$build_dir ./run-firebuild -- env -C ../../$build_dir cmake .. ./run-firebuild -- make -C ../../$build_dir clean time ./run-firebuild -- make -C ../../$build_dir all