pax_global_header00006660000000000000000000000064144143065640014521gustar00rootroot0000000000000052 comment=afba3ba6ef272665346f3f3cab3317a1d48f6a80 zst-0.4/000077500000000000000000000000001441430656400122045ustar00rootroot00000000000000zst-0.4/.cirrus.yml000066400000000000000000000007061441430656400143170ustar00rootroot00000000000000ubuntu_amd64_task: container: image: ubuntu:jammy install_script: - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade - DEBIAN_FRONTEND=noninteractive apt-get -y install cmake libz-dev libbz2-dev liblzma-dev libzstd-dev bzip2 zstd xz-utils valgrind build_script: - cmake . - make -j$(getconf _NPROCESSORS_ONLN) test_script: USE_VALGRIND=y ctest -j$(getconf _NPROCESSORS_ONLN) --output-on-failure zst-0.4/.github/000077500000000000000000000000001441430656400135445ustar00rootroot00000000000000zst-0.4/.github/workflows/000077500000000000000000000000001441430656400156015ustar00rootroot00000000000000zst-0.4/.github/workflows/codecov.yml000066400000000000000000000015051441430656400177470ustar00rootroot00000000000000name: Codecov on: [push, pull_request] env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} jobs: ubuntu: runs-on: ubuntu-latest steps: - name: Update apt run: sudo apt-get update -qq - name: Install dependencies run: sudo apt-get install -y cmake libz-dev libbz2-dev liblzma-dev libzstd-dev bzip2 zstd xz-utils - uses: actions/checkout@v3 - name: cmake run: cmake -B ${{github.workspace}}/build -DCMAKE_C_FLAGS="-O0 -g --coverage" - name: compile run: make -C ${{github.workspace}}/build - name: run run: cd ${{github.workspace}}/build && ctest --output-on-failure - name: Codecov uses: codecov/codecov-action@v3 with: gcov: true zst-0.4/.github/workflows/codeql.yml000066400000000000000000000047001441430656400175740ustar00rootroot00000000000000name: "CodeQL" on: push: branches: [ "master" ] pull_request: # The branches below must be a subset of the branches above branches: [ "master" ] schedule: - cron: '29 11 * * 4' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'cpp' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v3 - name: Install B-Deps run: | sudo apt-get update sudo apt-get -y install cmake libz-dev libbz2-dev liblzma-dev libzstd-dev bzip2 zstd xz-utils # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: category: "/language:${{matrix.language}}" zst-0.4/.github/workflows/coverity.yml000066400000000000000000000013411441430656400201670ustar00rootroot00000000000000name: Coverity scan on: push: branches: [coverity] jobs: coverity: runs-on: ubuntu-latest steps: - name: Update apt run: sudo apt-get update -qq - name: Install dependencies run: sudo apt-get install -y cmake libz-dev libbz2-dev liblzma-dev libzstd-dev bzip2 zstd xz-utils - uses: actions/checkout@v3 - name: cmake run: cmake -B ${{github.workspace}}/build - uses: vapier/coverity-scan-action@v1 with: email: ${{ secrets.COVERITY_SCAN_EMAIL }} token: ${{ secrets.COVERITY_SCAN_TOKEN }} command: make -C ${{github.workspace}}/build zst-0.4/.github/workflows/email.yml000066400000000000000000000017761441430656400174260ustar00rootroot00000000000000on: check_suite: type: ['completed'] name: Email about Cirrus CI failures jobs: continue: name: After Cirrus CI Failure if: github.event.check_suite.app.name == 'Cirrus CI' && github.event.check_suite.conclusion != 'success' runs-on: ubuntu-latest steps: - uses: octokit/request-action@v2.x id: get_failed_check_run with: route: GET /repos/${{ github.repository }}/check-suites/${{ github.event.check_suite.id }}/check-runs?status=completed mediaType: '{"previews": ["antiope"]}' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | echo "Cirrus CI ${{ github.event.check_suite.conclusion }} on ${{ github.event.check_suite.head_branch }} branch!" echo "SHA ${{ github.event.check_suite.head_sha }}" echo $MESSAGE echo "##[error]See $CHECK_RUN_URL for details" && false env: CHECK_RUN_URL: ${{ fromJson(steps.get_failed_check_run.outputs.data).check_runs[0].html_url }} zst-0.4/.gitignore000066400000000000000000000001671441430656400142000ustar00rootroot00000000000000*~ zst *.o CMakeCache.txt CMakeFiles/ CTestTestfile.cmake Testing/ Makefile cmake_install.cmake config.h /tests/test-* zst-0.4/CMakeLists.txt000066400000000000000000000025731441430656400147530ustar00rootroot00000000000000cmake_minimum_required(VERSION 2.8.12) if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) cmake_policy(SET CMP0110 NEW) endif() project(zst C) include(CheckIncludeFiles) include(CheckLibraryExists) include(CheckFunctionExists) include(GNUInstallDirs) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") endif (NOT CMAKE_BUILD_TYPE) function(find_lib define so func) CHECK_LIBRARY_EXISTS(${so} ${func} "" HAVE_LIB${define}) if(${HAVE_LIB${define}}) set(libs ${libs} ${so} PARENT_SCOPE) endif() endfunction() find_lib(Z z gzread) find_lib(BZ2 bz2 BZ2_bzBuffToBuffCompress) find_lib(LZMA lzma lzma_code) find_lib(ZSTD zstd ZSTD_compress) CHECK_FUNCTION_EXISTS(copy_file_range HAVE_COPY_FILE_RANGE) CHECK_FUNCTION_EXISTS(stat64 HAVE_STAT64) function(add_flag flag) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flag}" PARENT_SCOPE) endfunction() add_flag(-Wall) add_flag(-Werror=format-security) add_flag(-Wmissing-declarations) add_flag(-Wredundant-decls) add_flag(-Wno-parentheses) configure_file(config.h.cmake ${CMAKE_BINARY_DIR}/config.h) include_directories(${CMAKE_BINARY_DIR}) set(zst_sources compress.c zst.c ) add_executable(zst ${zst_sources}) target_link_libraries(zst ${libs}) install(TARGETS zst DESTINATION bin) install(FILES zst.1 zst.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) install(FILES ${CMAKE_BINARY_DIR}/zst DESTINATION bin) enable_testing() add_subdirectory(tests) zst-0.4/LICENSE000066400000000000000000000011701441430656400132100ustar00rootroot00000000000000Š2022 Adam Borowski This project is available to anyone who is not a {Net,Open,Free}BSD developer under the terms of GNU GPL version 2 or higher, as published by the FSF, full text available from approximately 4294967296 places. Developers of the aforemented three BSDs are instead granted rights under the BSD-1-Clause license -- ie, you have all the usual BSD freedoms as long as my original authorship is not misrepresented within the source. You are also authorized to replace this file by something less clumsy. (Yes, I'm aware of a potential exploit within these license terms, that's intentional.) zst-0.4/README.md000066400000000000000000000027621441430656400134720ustar00rootroot00000000000000zst --- As compressors go, `zstd` is awesome. It's very fast while beating any commonly used alternative except `xz` in compression ratio. But alas, its command-line tool leaves much to be wished for: * its package takes over 2MB, 1.2MB for the main executable * it fails to remove files it's done with * its compression levels use a different scale than the rest of the world (1..19 sometimes 22 instead of 1..9) * turning off its chattiness makes you lose warnings The first bit is a blocker towards making that tool a guaranteed (“*essential*”) part of a Linux distribution — it'd bloat the size of minimal systems (eg. containers) too much. The other bits sometimes interfere with making zstd a drop-in replacement: in most places you can insert arbitrary arguments but it's not always the case. On the other hand, `libzstd` is **already** essential! And while we're here, there's three other essential compressor libraries: `zlib` `libbz2` `liblzma` (xz). Why not provide all four from a single small binary? Thus, let's unify popular compressors and avoid weirdness of their command-line tools. Supported formats ----------------- * zst * xz * gz * bz2 Status ------ This tool should be considered of beta quality. * [x] guess algorithm via header * [ ] threaded [de]compression * [ ] threading when multiple files * [ ] sparse files * [ ] io optimizations * [x] decent test coverage * [ ] dlopen non-essential compressors? * [x] behave according to argv[0] * [ ] --rsyncable zst-0.4/compress.c000066400000000000000000000456501441430656400142150ustar00rootroot00000000000000#define _GNU_SOURCE #include "config.h" #include #include #include #include #include #include #include #ifdef HAVE_LIBBZ2 # include #endif #ifdef HAVE_LIBZ # include #endif #ifdef HAVE_LIBLZMA # include #endif #ifdef HAVE_LIBZSTD # include #endif #include "compress.h" #include "zst.h" #define U64(x) (*((uint64_t*)(x))) #define BUFFER_SIZE 32768 #define GRIPE(f,msg,...) fprintf(stderr, "%s: %s%s: " msg, exe, fi->path, fi->name_##f?: "std" #f,##__VA_ARGS__) #define ERR(l,f,msg,...) do {GRIPE(f, msg "\n",##__VA_ARGS__);goto l;} while(0) #define ERRoom(l,f) ERR(l,f,"Out of memory.") #define ERRueof(l,f) ERR(l,f,"unexpected end of file") #define ERRlibc(l,f) ERR(l,f,"%m") static int rewrite(int fd, const void *buf, size_t len) { if (fd == -1) return 0; while (len) { size_t done = write(fd, buf, len); if (done == -1) if (errno == EINTR) continue; else return -1; buf += done; len -= done; } return 0; } #ifdef HAVE_LIBBZ2 static const char *bzerr(int e) { switch (e) { case BZ_MEM_ERROR: return "out of memory"; case BZ_DATA_ERROR: return "compressed data corrupted"; case BZ_DATA_ERROR_MAGIC: return "not a bzip2 file"; case BZ_IO_ERROR: return strerror(errno); case BZ_UNEXPECTED_EOF: return "unexpected end of file"; # ifndef NDEBUG // these can happen only if our code is bogus case BZ_SEQUENCE_ERROR: return "internal error: bad call sequence"; case BZ_PARAM_ERROR: return "internal error: param error"; case BZ_OUTBUFF_FULL: return "internal error: buffer full"; case BZ_CONFIG_ERROR: return "internal error: bad config"; case BZ_OK: return "internal_error: OK when not obviously not OK"; case BZ_RUN_OK: return "internal error: unexpected RUN_OK"; case BZ_FLUSH_OK: return "internal error: unexpected FLUSH_OK"; case BZ_FINISH_OK: return "internal error: unexpected FINISH_OK"; case BZ_STREAM_END: return "internal error: unexpected STREAM_END"; # endif default: return "invalid error?!?"; } } #define ERRbz2(l,f) ERR(l,f,"%s", bzerr(ret)) static int read_bz2(int in, int out, file_info *restrict fi, char *head) { bz_stream st; int ret; char inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; bzero(&st, sizeof st); if (ret = BZ2_bzDecompressInit(&st, 0, 0)) ERRbz2(end, in); if (head) { if ((st.avail_in = read(in, inbuf + MLEN, BUFFER_SIZE - MLEN)) == -1) ERRlibc(fail, in); st.avail_in += MLEN; st.next_in = inbuf; U64(inbuf) = U64(head); goto work; } while ((st.avail_in = read(in, st.next_in = inbuf, BUFFER_SIZE)) > 0) { work: fi->sz += st.avail_in; do { if (ret == BZ_STREAM_END) if ((ret = BZ2_bzDecompressEnd(&st)) || (ret = BZ2_bzDecompressInit(&st, 0, 0))) { ERRbz2(end, in); } st.next_out = outbuf; st.avail_out = sizeof outbuf; if ((ret = BZ2_bzDecompress(&st)) && ret != BZ_STREAM_END) ERRbz2(fail, in); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sd += st.next_out - outbuf; } while (st.avail_in); } if (st.avail_in) ERRlibc(fail, in); if (ret == BZ_STREAM_END) goto ok; // Flush the stream do { st.next_out = outbuf; st.avail_out = sizeof outbuf; ret = BZ2_bzDecompress(&st); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sd += st.next_out - outbuf; } while (!ret && !st.avail_out); if (ret == BZ_OK) ret = BZ_UNEXPECTED_EOF; // happens on very short files if (ret != BZ_STREAM_END) ERRbz2(fail, in); ok: BZ2_bzDecompressEnd(&st); return 0; fail: BZ2_bzDecompressEnd(&st); end: return 1; } static int write_bz2(int in, int out, file_info *restrict fi, char *head) { bz_stream st; int ret; char inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; bzero(&st, sizeof st); if ((ret = BZ2_bzCompressInit(&st, level?:9, 0, 0))) ERRbz2(end, in); while ((st.avail_in = read(in, st.next_in = inbuf, BUFFER_SIZE)) > 0) { fi->sd += st.avail_in; do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); if ((ret = BZ2_bzCompress(&st, BZ_RUN)) && ret != BZ_RUN_OK) ERRbz2(fail, in); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sz += st.next_out - outbuf; } while (st.avail_in); } if (st.avail_in) ERRlibc(fail, in); // Flush the stream do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); ret = BZ2_bzCompress(&st, BZ_FINISH); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sz += st.next_out - outbuf; } while (ret == BZ_FINISH_OK); if (ret != BZ_STREAM_END) ERRbz2(fail, in); BZ2_bzCompressEnd(&st); return 0; fail: BZ2_bzCompressEnd(&st); end: return 1; } # undef ERRbz2 #endif #ifdef HAVE_LIBZ static const char *gzerr(int e) { switch (e) { case Z_NEED_DICT: return "file compressed with a private dictionary"; case Z_ERRNO: return strerror(errno); case Z_STREAM_ERROR: return "internal error"; case Z_DATA_ERROR: return "file is corrupted"; case Z_MEM_ERROR: return "out of memory"; case Z_BUF_ERROR: return "unexpected end of file"; case Z_VERSION_ERROR: return "unsupported version of gzip"; default: return "invalid error?!?"; } } #define ERRgz(l,f) ERR(l,f,"%s\n", gzerr(ret)) static int read_gz(int in, int out, file_info *restrict fi, char *head) { z_stream st; int ret = 0; ssize_t len; Bytef inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; bzero(&st, sizeof st); if (inflateInit2(&st, 32)) ERRoom(end, in); if (head) { if ((len = read(in, inbuf + MLEN, BUFFER_SIZE - MLEN)) == -1) ERRlibc(fail, in); st.avail_in = len + MLEN; st.next_in = inbuf; U64(inbuf) = U64(head); goto work; } while ((len = read(in, st.next_in = inbuf, BUFFER_SIZE)) > 0) { st.avail_in = len; work: fi->sz += st.avail_in; do { // concatenated stream => reset stream if (ret == Z_STREAM_END) if (ret = inflateReset(&st)) ERRgz(fail, in); st.next_out = outbuf; st.avail_out = sizeof outbuf; if ((ret = inflate(&st, Z_NO_FLUSH)) && ret != Z_STREAM_END) ERRgz(fail, in); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sd += st.next_out - outbuf; } while (st.avail_in); } if (st.avail_in) ERRlibc(fail, in); // Flush the stream do { st.next_out = outbuf; st.avail_out = sizeof outbuf; ret = inflate(&st, Z_FINISH); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sd += st.next_out - outbuf; } while (!ret); if (ret != Z_STREAM_END) ERRgz(fail, in); inflateEnd(&st); return 0; fail: inflateEnd(&st); end: return 1; } static int write_gz(int in, int out, file_info *restrict fi, char *head) { z_stream st; int ret; ssize_t len; Bytef inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; bzero(&st, sizeof st); if ((ret = deflateInit2(&st, level?:6, Z_DEFLATED, 31, 9, 0))) ERRgz(end, in); while ((len = read(in, st.next_in = inbuf, BUFFER_SIZE)) > 0) { st.avail_in = len; fi->sd += len; do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); if ((ret = deflate(&st, Z_NO_FLUSH))) ERRgz(fail, in); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sz += st.next_out - outbuf; } while (st.avail_in); } if (len) ERRlibc(fail, in); // Flush the stream do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); ret = deflate(&st, Z_FINISH); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sz += st.next_out - outbuf; } while (!ret); if (ret != Z_STREAM_END) ERRgz(fail, in); deflateEnd(&st); return 0; fail: deflateEnd(&st); end: return 1; } # undef ERRgz #endif #ifdef HAVE_LIBLZMA static const char *xzerr(lzma_ret e) { switch (e) { case LZMA_MEM_ERROR: return "out of memory"; case LZMA_MEMLIMIT_ERROR: return "memory usage limit was reached"; case LZMA_FORMAT_ERROR: return "file format not recognized"; case LZMA_OPTIONS_ERROR: return "invalid or unsupported options"; case LZMA_DATA_ERROR: return "file is corrupted"; case LZMA_BUF_ERROR: return "unexpected end of file"; case LZMA_PROG_ERROR: return "internal error"; default: return "invalid error?!?"; } } #define ERRxz(l,f) ERR(l,f, "%s\n", xzerr(ret)) static int read_xz(int in, int out, file_info *restrict fi, char *head) { uint8_t inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; lzma_stream st = LZMA_STREAM_INIT; lzma_ret ret = 0; if (lzma_stream_decoder(&st, UINT64_MAX, LZMA_CONCATENATED)) ERRoom(end, in); if (head) { if ((st.avail_in = read(in, inbuf + MLEN, BUFFER_SIZE - MLEN)) == -1) ERRlibc(fail, in); st.avail_in += MLEN; st.next_in = inbuf; U64(inbuf) = U64(head); goto work; } while ((st.avail_in = read(in, (uint8_t*)(st.next_in = inbuf), BUFFER_SIZE)) > 0) { work: fi->sz += st.avail_in; do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); if ((ret = lzma_code(&st, LZMA_RUN))) ERRxz(fail, in); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sd += st.next_out - outbuf; } while (st.avail_in); } if (st.avail_in) ERRlibc(fail, in); // Flush the stream do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); ret = lzma_code(&st, LZMA_FINISH); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sd += st.next_out - outbuf; } while (!ret); if (ret != LZMA_STREAM_END) ERRxz(fail, in); lzma_end(&st); return 0; fail: lzma_end(&st); end: return 1; } static int write_xz(int in, int out, file_info *restrict fi, char *head) { uint8_t inbuf[BUFFER_SIZE], outbuf[BUFFER_SIZE]; lzma_stream st = LZMA_STREAM_INIT; lzma_ret ret = 0; int xzlevel = level?:6; if (xzlevel == 1) // xz level 1 is boring, 0 stands out xzlevel = 0; if (lzma_easy_encoder(&st, xzlevel, LZMA_CHECK_CRC64)) ERRoom(end, in); while ((st.avail_in = read(in, (uint8_t*)(st.next_in = inbuf), BUFFER_SIZE)) > 0) { fi->sd += st.avail_in; do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); if ((ret = lzma_code(&st, LZMA_RUN))) ERRxz(fail, in); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sz += st.next_out - outbuf; } while (st.avail_in); } if (st.avail_in) ERRlibc(fail, in); // Flush the stream do { st.next_out = outbuf; st.avail_out = sizeof(outbuf); ret = lzma_code(&st, LZMA_FINISH); if (rewrite(out, outbuf, st.next_out - outbuf)) ERRlibc(fail, out); fi->sz += st.next_out - outbuf; } while (!ret); if (ret != LZMA_STREAM_END) ERRxz(fail, in); lzma_end(&st); return 0; fail: lzma_end(&st); end: return 1; } # undef ERRxz #endif #ifdef HAVE_LIBZSTD #define ERRzstd(l,f) ERR(l,f, "%s\n", ZSTD_getErrorName(r)) static int read_zstd(int in, int out, file_info *restrict fi, char *head) { int err = 1; ZSTD_inBuffer zin; ZSTD_outBuffer zout; size_t const inbufsz = ZSTD_DStreamInSize(); size_t r; zin.src = malloc(inbufsz); zout.size = ZSTD_DStreamOutSize(); zout.dst = malloc(zout.size); if (!zin.src || !zout.dst) ERRoom(end, in); ZSTD_DStream* const stream = ZSTD_createDStream(); if (!stream) ERRoom(end, in); if (ZSTD_isError(r = ZSTD_initDStream(stream))) ERRzstd(fail, in); int end_of_frame = 0; // empty file is an error if (head) { if ((r = read(in, (char*)zin.src + MLEN, inbufsz - MLEN)) == -1) ERRlibc(fail, in); r += MLEN; U64(zin.src) = U64(head); goto work; } while ((r = read(in, (void*)zin.src, inbufsz))) { if (r == -1) ERRlibc(fail, in); work: fi->sz += r; zin.size = r; zin.pos = 0; while (zin.pos < zin.size) { zout.pos = 0; if (ZSTD_isError(r = ZSTD_decompressStream(stream, &zout, &zin))) ERRzstd(fail, in); end_of_frame = !r; fi->sd += zout.pos; if (rewrite(out, zout.dst, zout.pos)) ERRlibc(fail, out); } } if (!fi->sz) // old versions of libzstd give a weird error for empty files ERRueof(fail, in); // flush if (!end_of_frame) { zout.pos = 0; if (ZSTD_isError(r = ZSTD_decompressStream(stream, &zout, &zin))) ERRzstd(fail, in); fi->sd += zout.pos; if (rewrite(out, zout.dst, zout.pos)) ERRlibc(fail, out); // write first, fail later -- hopefully salvaging some data if (r) ERRueof(fail, in); } err = 0; fail: ZSTD_freeDStream(stream); end: free((void*)zin.src); free(zout.dst); return err; } static int write_zstd(int in, int out, file_info *restrict fi, char *head) { int err = 1; ZSTD_inBuffer zin; ZSTD_outBuffer zout; size_t const inbufsz = ZSTD_CStreamInSize(); size_t r; zin.src = malloc(inbufsz); zout.size = ZSTD_CStreamOutSize(); zout.dst = malloc(zout.size); if (!zin.src || !zout.dst) ERRoom(end, in); ZSTD_CStream* const stream = ZSTD_createCStream(); if (!stream) ERRoom(end, in); // unlike all other compressors, zstd levels go 1..19 (..22 as "extreme") int zlevel = ((level?:2) - 1) * 18 / 8 + 1; assert(zlevel <= 19); if (ZSTD_isError(r = ZSTD_initCStream(stream, zlevel))) ERRzstd(fail, in); ZSTD_CCtx_setParameter(stream, ZSTD_c_checksumFlag, 1); while ((r = read(in, (void*)zin.src, inbufsz))) { if (r == -1) ERRlibc(fail, in); fi->sd += r; zin.size = r; zin.pos = 0; while (zin.pos < zin.size) { zout.pos = 0; if (ZSTD_isError(r = ZSTD_compressStream(stream, &zout, &zin))) ERRzstd(fail, in); if (rewrite(out, zout.dst, zout.pos)) ERRlibc(fail, out); fi->sz += zout.pos; } } zout.pos = 0; if (ZSTD_isError(r = ZSTD_endStream(stream, &zout))) ERRzstd(fail, in); if (rewrite(out, zout.dst, zout.pos)) ERRlibc(fail, out); fi->sz += zout.pos; err = 0; fail: ZSTD_freeCStream(stream); end: free((void*)zin.src); free(zout.dst); return err; } # undef ERRzstd #endif static int cat(int in, int out, file_info *restrict fi, char *head) { if (out == -1) return 0; if (head) { if (rewrite(out, head, MLEN)) ERRlibc(end, out); fi->sz = fi->sd = MLEN; } ssize_t r; #ifdef HAVE_COPY_FILE_RANGE while ((r = copy_file_range(in, 0, out, 0, PTRDIFF_MAX, 0)) > 0) { fi->sz += r; fi->sd += r; } if (!r) return 0; if (errno != EINVAL && errno != EXDEV) // EXDEV regressed in 5.19 ERRlibc(end, in); #endif char buf[BUFFER_SIZE]; while ((r = read(in, buf, sizeof buf)) > 0) { if (rewrite(out, buf, r)) ERRlibc(end, out); fi->sz += r; fi->sd += r; } if (r) ERRlibc(end, in); return 0; end: return 1; } compress_info compressors[]={ #ifdef HAVE_LIBZSTD {"zstd", ".zst", write_zstd}, #endif #ifdef HAVE_LIBLZMA {"xz", ".xz", write_xz}, #endif #ifdef HAVE_LIBZ {"gzip", ".gz", write_gz}, #endif #ifdef HAVE_LIBBZ2 {"bzip2", ".bz2", write_bz2}, #endif {0, 0, 0}, }; compress_info decompressors[]={ #ifdef HAVE_LIBZSTD {"zstd", ".zst", read_zstd, {0x28,0xb5,0x2f,0xfd}, {0xff,0xff,0xff,0xff}}, #endif #ifdef HAVE_LIBLZMA {"xz", ".xz", read_xz, {0xfd,0x37,0x7a,0x58,0x5a}, {0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xf0}}, #endif #ifdef HAVE_LIBZ {"gzip", ".gz", read_gz, {0x1f,0x8b,8}, {0xff,0xff,0xff,0xe0}}, #endif #ifdef HAVE_LIBBZ2 {"bzip2", ".bz2", read_bz2, "BZh01AY&", {0xff,0xff,0xff,0xf0,0xff,0xff,0xff,0xff}}, // empty file has no BlockHeader {"", "/", read_bz2, "BZh0\x17rE8", {0xff,0xff,0xff,0xf0,0xff,0xff,0xff,0xff}}, #endif {0, 0, 0}, }; int match_suffix(const char *txt, const char *ext) { int tl,el; tl=strlen(txt); el=strlen(ext); if (tl<=el) return 0; txt+=tl-el; return !strncmp(txt, ext, el); } compress_info *comp_by_ext(const char *name, compress_info *ci) { for (;ci->name;ci++) if (match_suffix(name, ci->ext)) return ci; return 0; } compress_info *comp_by_name(const char *name, compress_info *ci) { for (;ci->name;ci++) if (!strcmp(name, ci->name) || !strcmp(name, ci->ext+1)) return ci; return 0; } static bool verify_magic(const char *bytes, const compress_info *comp) { return (U64(bytes) & U64(comp->magicmask)) == U64(comp->magic); } bool decomp(bool can_cat, int in, int out, file_info*restrict fi) { char buf[MLEN]; ssize_t r = read(in, buf, MLEN); if (r == -1) ERRlibc(err, in); if (r < MLEN) // shortest legal file is 9 bytes (zstd w/o checksum) { if (!can_cat) ERR(err, in, "not a compressed file"); if (rewrite(out, buf, r)) ERRlibc(err, out); fi->sd = fi->sz = r; return 0; } for (const compress_info *ci = decompressors; ci->comp; ci++) if (verify_magic(buf, ci)) return ci->comp(in, out, fi, buf); if (can_cat) return cat(in, out, fi, buf); ERR(err, in, "not a compressed file"); err: return 1; } zst-0.4/compress.h000066400000000000000000000012731441430656400142130ustar00rootroot00000000000000#include typedef struct { const char *path, *name_in, *name_out; unsigned long long sz, sd; } file_info; typedef int(compress_func)(int,int,file_info*restrict,char*); #define MLEN 8 typedef struct { const char *name; const char *ext; compress_func *comp; char magic[MLEN], magicmask[MLEN]; } compress_info; extern compress_info compressors[]; extern compress_info decompressors[]; compress_info *comp_by_ext(const char *name, compress_info *ci); compress_info *comp_by_name(const char *name, compress_info *ci); bool decomp(bool can_cat, int in, int out, file_info*restrict fi); int match_suffix(const char *txt, const char *ext); zst-0.4/config.h.cmake000066400000000000000000000002371441430656400147030ustar00rootroot00000000000000#cmakedefine HAVE_LIBZ #cmakedefine HAVE_LIBBZ2 #cmakedefine HAVE_LIBLZMA #cmakedefine HAVE_LIBZSTD #cmakedefine HAVE_COPY_FILE_RANGE #cmakedefine HAVE_STAT64 zst-0.4/tests/000077500000000000000000000000001441430656400133465ustar00rootroot00000000000000zst-0.4/tests/CMakeLists.txt000066400000000000000000000027201441430656400161070ustar00rootroot00000000000000file(GLOB tests LIST_DIRECTORIES false CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.t) function(compressor TOOL EXT) foreach(t ${tests}) string(REGEX REPLACE ".*/([^/]*)\\.t$" "\\1" t ${t}) set(tn "${t}-${TOOL}") if(CMAKE_VERSION VERSION_LESS 3.19) string(REPLACE " " "_" tn "${tn}") string(REPLACE "#" "♯" tn "${tn}") string(REPLACE "(" "âĻ…" tn "${tn}") string(REPLACE ")" "âĻ†" tn "${tn}") endif() add_test(NAME ${tn} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/runtest.sh "${t}") set_tests_properties(${tn} PROPERTIES SKIP_RETURN_CODE 42 ENVIRONMENT "TOOL=${TOOL};EXT=${EXT};SRC=${CMAKE_SOURCE_DIR};BIN=${CMAKE_BINARY_DIR}") endforeach() endfunction() if(HAVE_LIBZ) compressor(gzip .gz) endif() if(HAVE_LIBBZ2) compressor(bzip2 .bz2) endif() if(HAVE_LIBLZMA) compressor(xz .xz) endif() if(HAVE_LIBZSTD) compressor(zstd .zst) endif() file(GLOB Tests LIST_DIRECTORIES false CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/*.T) foreach(t ${Tests}) string(REGEX REPLACE ".*/([^/]*)\\.T$" "\\1" t ${t}) set(tn ${t}) if(CMAKE_VERSION VERSION_LESS 3.19) string(REPLACE " " "_" tn "${tn}") string(REPLACE "#" "♯" tn "${tn}") string(REPLACE "(" "âĻ…" tn "${tn}") string(REPLACE ")" "âĻ†" tn "${tn}") endif() add_test(NAME ${tn} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/runtest.sh "${t}") set_tests_properties(${tn} PROPERTIES SKIP_RETURN_CODE 42 ENVIRONMENT "SRC=${CMAKE_SOURCE_DIR};BIN=${CMAKE_BINARY_DIR}") endforeach() zst-0.4/tests/cd-argv0-NOVG.t000066400000000000000000000004421441430656400157050ustar00rootroot00000000000000case $TOOL in gzip) TOOLCAT=gzcat ;; bzip2) TOOLCAT=bzcat ;; bzip3) TOOLCAT=bz3cat ;; *) TOOLCAT=${TOOL}cat ;; esac echo TOOLCAT = $TOOLCAT cp "$F" file mkdir mybin # in case . is in PATH ln -s "$Z" mybin/"$TOOLCAT" $TOOL file mybin/$TOOLCAT file$EXT >out cmp -b "$F" out zst-0.4/tests/cdf-file-suffix.t000066400000000000000000000002331441430656400165040ustar00rootroot00000000000000echo meow >meow.txt $Z -cdf meow.txt >stdout cmp -b meow.txt stdout $TOOL meow.txt $Z -cdf meow.txt$EXT >stdout echo meow >meow.txt cmp -b meow.txt stdout zst-0.4/tests/d-argv0-NOVG.t000066400000000000000000000004631441430656400155450ustar00rootroot00000000000000case $TOOL in gzip) UNTOOL=gunzip ;; bzip2) UNTOOL=bunzip2 ;; bzip3) UNTOOL=bunzip3 ;; *) UNTOOL=un$TOOL ;; esac echo UNTOOL = $UNTOOL cp "$F" file mkdir mybin # in case . is in PATH ln -s "$Z" mybin/"$UNTOOL" $TOOL file rm -f file # zstd leaves it mybin/$UNTOOL file$EXT cmp -b "$F" file zst-0.4/tests/d-bigfile.t000066400000000000000000000002071441430656400153540ustar00rootroot00000000000000dd if=/dev/urandom bs=65536 count=16 status=none|od >in cp in file $TOOL file rm -f file # zstd keeps it $Z -d file$EXT cmp -b in file zst-0.4/tests/d-bigfile0.t000066400000000000000000000002011441430656400154260ustar00rootroot00000000000000dd if=/dev/zero bs=65536 count=16 status=none >in cp in file $TOOL file rm -f file # zstd keeps it $Z -d file$EXT cmp -b in file zst-0.4/tests/d-bigfileR.t000066400000000000000000000002041441430656400154730ustar00rootroot00000000000000dd if=/dev/urandom bs=65536 count=16 status=none >in cp in file $TOOL file rm -f file # zstd keeps it $Z -d file$EXT cmp -b in file zst-0.4/tests/d-dir-nosuffix.T000066400000000000000000000002221441430656400163250ustar00rootroot00000000000000mkdir foo cd foo touch a touch b gzip b cd .. $Z -dr foo 2>stderr || ret=$? cat stderr test $ret -eq 2 grep -q 'unknown suffix -- ignored' stderr zst-0.4/tests/d-dir.t000066400000000000000000000001501441430656400145260ustar00rootroot00000000000000mkdir dir cp -p $F dir/file $TOOL dir/file rm -f dir/file # zstd keeps it $Z -rd dir cmp -b $F dir/file zst-0.4/tests/d-emptyfile.t000066400000000000000000000001401441430656400157450ustar00rootroot00000000000000cat file $TOOL file rm -f file # zstd keeps it $Z -d file$EXT cmp -b /dev/null file zst-0.4/tests/d-file-corrupted.t000066400000000000000000000003521441430656400167000ustar00rootroot00000000000000dd if=/dev/urandom bs=65536 count=1 status=none|od >file $Z -F$TOOL file dd if=/dev/zero of=file$EXT bs=4096 seek=1 count=1 conv=notrunc status=none $Z -d file$EXT 2>stderr || ret=$? cat stderr test $ret -eq 1 grep -qi corrupt stderr zst-0.4/tests/d-file-exist.t000066400000000000000000000000551441430656400160250ustar00rootroot00000000000000cp -p $F file $TOOL -k file ! $Z -d file$EXT zst-0.4/tests/d-file-noperm.t000066400000000000000000000002371441430656400161730ustar00rootroot00000000000000[ `whoami` != root ] || exit 42 cp -p $F file $TOOL file rm -f file chmod u-r file$EXT ! $Z -d file$EXT 2>stderr cat stderr grep -q 'Permission denied' stderr zst-0.4/tests/d-file-notcomp.T000066400000000000000000000001361441430656400163100ustar00rootroot00000000000000cp -p $F file.zst ! $Z -d file.zst 2>stderr cat stderr grep -q 'not a compressed file' stderr zst-0.4/tests/d-file-truncated.t000066400000000000000000000001371441430656400166630ustar00rootroot00000000000000cp -p $F file $TOOL file rm -f file # zstd keeps it truncate -s 1024 file$EXT ! $Z -d file$EXT zst-0.4/tests/d-file.t000066400000000000000000000001221441430656400146660ustar00rootroot00000000000000cp -p $F file $TOOL file rm -f file # zstd keeps it $Z -d file$EXT cmp -b $F file zst-0.4/tests/d-pipe-truncations.t000066400000000000000000000004571441430656400172660ustar00rootroot00000000000000echo meow|$TOOL >file$EXT LEN=`stat -c %s file$EXT` echo LEN=$LEN i=0 while [ $i -lt $LEN ]; do echo len=$i ! dd status=none if=file$EXT bs=1 count=$i|$Z -d >/dev/null 2>stderr cat stderr grep -q ': stdin: \(not a compressed file\|unexpected end of file\)$' stderr i=$(( i+1 )) done zst-0.4/tests/dc-file.t000066400000000000000000000001351441430656400150350ustar00rootroot00000000000000cp -p $F file $TOOL file rm -f file # zstd keeps it $Z -cd file$EXT >stdout cmp -b $F stdout zst-0.4/tests/df-file-notcomp.T000066400000000000000000000000411441430656400164510ustar00rootroot00000000000000$Z -cdf <$F >file cmp -b $F file zst-0.4/tests/dq-warn.T000066400000000000000000000000261441430656400150420ustar00rootroot00000000000000touch foo $Z -dq foo zst-0.4/tests/runtest.sh000077500000000000000000000010131441430656400154040ustar00rootroot00000000000000#!/bin/sh set -e if [ -z "$SRC" ] || [ -z "$BIN" ]; then echo >&2 "Required ENV vars not set." exit 1 fi if [ -n "$USE_VALGRIND" ] && [ "${1%NOVG}" = "$1" ] && which >/dev/null 2>/dev/null valgrind; then VG="valgrind --error-exitcode=43 --leak-check=full --show-leak-kinds=all --errors-for-leak-kinds=all " fi TESTDIR="$BIN/tests/test-$1-$TOOL" export Z="$VG$BIN/zst" F="$SRC/zst.c" rm -rf "$TESTDIR" mkdir "$TESTDIR" cd "$TESTDIR" if [ -n "$TOOL" ]; then T=t; else T=T; fi sh -e "$SRC/tests/$1.$T" rm -rf "$TESTDIR" zst-0.4/tests/t-dir.t000066400000000000000000000001251441430656400145500ustar00rootroot00000000000000mkdir dir cp -p $F dir/file $TOOL dir/file rm -f dir/file # zstd keeps it $Z -rt dir zst-0.4/tests/t-file.t000066400000000000000000000001031441430656400147050ustar00rootroot00000000000000cp -p $F file $TOOL file rm -f file # zstd keeps it $Z -t file$EXT zst-0.4/tests/t-pipe-empty.t000066400000000000000000000001331441430656400160620ustar00rootroot00000000000000! $Z -tF$TOOL stderr cat stderr grep ': stdin: not a compressed file$' stderr zst-0.4/tests/z-argv0-NOVG.t000066400000000000000000000001541441430656400155700ustar00rootroot00000000000000cp "$F" file mkdir mybin # in case . is in PATH ln -s "$Z" mybin/"$TOOL" mybin/$TOOL file $TOOL -t file$EXT zst-0.4/tests/z-badfmt.T000066400000000000000000000000411441430656400151720ustar00rootroot00000000000000! $Z -Fbad /dev/null zst-0.4/tests/z-bigfile.t000066400000000000000000000001651441430656400154050ustar00rootroot00000000000000dd if=/dev/urandom bs=65536 count=16 status=none|od >in cp in file $Z -F $TOOL file $TOOL -d file$EXT cmp -b in file zst-0.4/tests/z-bigfile0.t000066400000000000000000000001571441430656400154660ustar00rootroot00000000000000dd if=/dev/zero bs=65536 count=16 status=none >in cp in file $Z -F $TOOL file $TOOL -d file$EXT cmp -b in file zst-0.4/tests/z-bigfileR.t000066400000000000000000000001621441430656400155240ustar00rootroot00000000000000dd if=/dev/urandom bs=65536 count=16 status=none >in cp in file $Z -F $TOOL file $TOOL -d file$EXT cmp -b in file zst-0.4/tests/z-dir-alreadysuffix.t000066400000000000000000000002661441430656400174300ustar00rootroot00000000000000mkdir foo cd foo touch a touch b.gz touch c.bz2 touch d.xz touch e.zst cd .. $Z -F$TOOL -r foo 2>stderr || ret=$? cat stderr test $ret -eq 2 test `grep unchanged stderr|wc -l` -eq 4 zst-0.4/tests/z-dir-nonreg.T000066400000000000000000000002661441430656400160120ustar00rootroot00000000000000mkdir foo cd foo ln -s /dev/null a mkfifo b cd .. $Z -r foo 2>stderr || ret=$? ls -al foo cat stderr test $ret -eq 2 grep -q 'is not a directory or a regular file -- ignored' stderr zst-0.4/tests/z-emptyfile.t000066400000000000000000000001161441430656400157760ustar00rootroot00000000000000cat file $Z -F $TOOL file $TOOL -d file$EXT cmp -b /dev/null file zst-0.4/tests/z-file-alreadysuffix.t000066400000000000000000000002321441430656400175620ustar00rootroot00000000000000touch a touch b.gz touch c.bz2 touch d.xz touch e.zst $Z -F$TOOL * 2>stderr || ret=$? cat stderr test $ret -eq 2 test `grep unchanged stderr|wc -l` -eq 4 zst-0.4/tests/z-file-exist.t000066400000000000000000000000571441430656400160550ustar00rootroot00000000000000cp -p $F file $TOOL -k file ! $Z -F $TOOL file zst-0.4/tests/z-file-isdir.T000066400000000000000000000001611441430656400157670ustar00rootroot00000000000000mkdir foo $Z foo 2>stderr || ret=$? cat stderr test $ret -eq 2 grep -q 'is not a regular file -- ignored' stderr zst-0.4/tests/z-file-nonreg.T000066400000000000000000000002521441430656400161460ustar00rootroot00000000000000mkdir foo cd foo ln -s /dev/null a mkfifo b $Z -r * 2>stderr || ret=$? ls -al cat stderr test $ret -eq 2 grep -q 'is not a directory or a regular file -- ignored' stderr zst-0.4/tests/z-file-noperm.t000066400000000000000000000001761441430656400162230ustar00rootroot00000000000000[ `whoami` != root ] || exit 42 cp -p $F file chmod u-r file ! $Z file 2>stderr cat stderr grep -q 'Permission denied' stderr zst-0.4/tests/z-file.t000066400000000000000000000001001441430656400147100ustar00rootroot00000000000000cp -p $F file $Z -F $TOOL file $TOOL -d file$EXT cmp -b $F file zst-0.4/tests/z-level.t000066400000000000000000000003241441430656400151100ustar00rootroot00000000000000# Need enough data for compression level to matter. dd if=/dev/urandom bs=65536 count=1|od >file $Z -F$TOOL -z1 1$EXT $Z -F$TOOL -z9 9$EXT find . -name '*'$EXT -printf '%s\t%P\n'|sort -nr|sort -ck2 zst-0.4/tests/z-unknownformat.T000066400000000000000000000000521441430656400166470ustar00rootroot00000000000000! $Z -T doublespace /dev/null zst-0.4/tests/zc-file.t000066400000000000000000000001161441430656400150620ustar00rootroot00000000000000cp -p $F file $Z -cF $TOOL file >stdout $TOOL -cd stdout >file cmp -b $F file zst-0.4/tests/zd-dir-many.t000066400000000000000000000005331441430656400156670ustar00rootroot00000000000000HD="0 1 2 3 4 5 6 7 8 9 A B C D E F" for a in $HD; do for b in $HD; do for c in $HD; do x=$a$b$c mkdir $x echo meow >$x/foo echo bark >$x/bar done done done $Z -rF$TOOL * $Z -dr * grep -lr '^meow$' .|wc -l|tee meow grep -lr '^bark$' .|wc -l|tee bark diff -u meow - <$a$b$c done done done $Z -F$TOOL * $Z -d *$EXT ! grep -lv '^meow$' * zst-0.4/tests/zd-full.t000066400000000000000000000005151441430656400151110ustar00rootroot00000000000000! $Z -zF$TOOL /dev/full 2>stderr cat stderr grep -q ': stdout: No space left on device' stderr echo meow|$TOOL >file$EXT ! $Z -d /dev/full 2>stderr cat stderr grep -q ': stdout: No space left on device' stderr ! $Z -cd file$EXT >/dev/full 2>stderr cat stderr grep -q ': stdout: No space left on device' stderr zst-0.4/tests/zd-keep.t000066400000000000000000000001241441430656400150670ustar00rootroot00000000000000touch file $Z -k -F$TOOL file test -f file rm file $Z -kd file$EXT test -f file$EXT zst-0.4/tests/zdf-file-exist.t000066400000000000000000000001061441430656400163620ustar00rootroot00000000000000cp -p $F file $TOOL -k file $Z -f file $Z -df file$EXT cmp -b $F file zst-0.4/tests/zf-file-alreadysuffix.t000066400000000000000000000001171441430656400177320ustar00rootroot00000000000000touch a touch b.gz touch c.bz2 touch d.xz touch e.zst $Z -f -F$TOOL * 2>stderr zst-0.4/tests/ztd-pieces.t000066400000000000000000000002631441430656400156030ustar00rootroot00000000000000printf foo|$Z -F$TOOL >f printf bar|$Z -F$TOOL >>f printf baz|$Z -F$TOOL >>f if which >/dev/null 2>/dev/null hd;then hd f;fi $Z -t exp $Z -cd /dev/null 2>stderr cat stderr grep -q '^stdin: 4 → [0-9][0-9] ([0-9]\{3,4\}%)$' stderr $Z /dev/null 2>stderr cat stderr grep -q '^stdin: 0 → [0-9][0-9] (header)$' stderr zst-0.4/zst.1000066400000000000000000000052501441430656400131100ustar00rootroot00000000000000.TH zst 1 2022-10-18 .SH NAME zst \- compress or decompress .zst/.bz2/.gz/.xz files .SH SYNOPSIS .B zst .RI [ options ] " files" ... .SH DESCRIPTION The .B zst command can reduce the size of files by using a number of popular compression algorithms; in this version these are .IR zstd ", " bzip2 ", " xz ", " gzip -- and decompress them back. .P While these compressors' native tools may expose more options specific to the algorithm in question, .B zst unifies them with common functionality. Eg, only .B gzip and .B zstd can recurse with .BR -r , .B zstd defaults to .I -k and its levels go up to 19 rather than 9, etc. .P The default compressor is .I zstd as it's fastest while also compressing well; you may want to use .I xz instead when disk space / network bandwidth is at premium. On the other hand, neither .I gzip nor .I bzip2 are a superior choice in any case but especially .I gzip is entrenched for historical reasons. .SH OPTIONS .B Mode of operation .TP .B -z Compress (default). The file will be replaced by a compressed copy with an appropriate suffix added: .IR .zst / .bz2 / .xz / .gz according to the algorithm used. .TP .B -d Decompress. Files without a known suffix will be left untouched. .TP .B -t Test the integrity of compressed files; this is functionally same as decompression redirected to .IR /dev/null . .PP .B Modifiers .TP .B -c Write compressed or decompressed data to standard output. This implies .BR -k . For compression, if the stdout is a terminal, .B -f must be also specified. .TP .B -k The source file won't be removed after [de]compression. .TP .B -f Will overwrite existing files. Allows writing compressed data to a terminal. When .B -c is given and the data is not in the expected format, it will be passed through unmodified. Allows compressing a file that's already compressed. .TP .B -r If a directory is among file names specified on the command line, all files inside will be processed, possibly recursing into directories deeper in. .TP .BR -1 .. -9 Compression level: .B -1 is the weakest but fastest level the algorithm knows, .B -9 is strongest and slowest. Note that unlike the .B zstd tool, the scale is 1..9 for all algorithms -- level 9 corresponds to what .B zstd knows as 19. The defaults are: zst 2, bz2 9, gz 6, xz 6. .TP .B -v List all processed files. When compressing, the old, new, and percentage of required size is given. .TP .B -q Suppress all warnings. Unrelated to .BR -v . .TP .B -n Ignored; for compat with .BR gzip . .TP .B -F Specify compression algorithm to use. .SH RETURN VALUE .B 1 if any errors happened, .B 2 if there's a warning but no errors, .B 0 if all went ok. .SH SEE ALSO .BR zstd (1), .BR bzip2 (1), .BR gzip (1), .BR xz (1). zst-0.4/zst.c000066400000000000000000000212411441430656400131700ustar00rootroot00000000000000#define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include "compress.h" #include "zst.h" #ifndef HAVE_STAT64 # define stat64 stat # define fstat64 fstat #endif #ifndef O_LARGEFILE # define O_LARGEFILE 0 #endif #define die(...) do {fprintf(stderr, __VA_ARGS__); exit(1);} while(0) #define ARRAYSZ(x) (sizeof(x) / sizeof((x)[0])) #define warn(msg, ...) do {if (!quiet) {fprintf(stderr, "%s: " msg, exe, __VA_ARGS__); if (!err) err=2;}} while(0) const char *exe; static bool cat; bool force; static bool keep; static bool quiet; static bool verbose; static bool recurse; int level; static int op; static int err; static compress_info *comp; static int flink(int dir, int fd, const char *newname) { char proclink[26]; sprintf(proclink, "/proc/self/fd/%d", fd); int ret = linkat(AT_FDCWD, proclink, dir, newname, AT_SYMLINK_FOLLOW); #ifdef AT_EMPTY_PATH if (ret && errno==ENOENT) ret = linkat(fd, "", dir, newname, AT_EMPTY_PATH); #endif return ret; } #define FAIL(msg, ...) do {fprintf(stderr, "%s: " msg, exe, __VA_ARGS__); err=1; goto closure;} while(0) static void do_file(int dir, const char *name, const char *path, int fd, struct stat64 *restrict st) { int out = -1; bool notmp = 0; char *name2 = 0; compress_info *fcomp = comp; if (!op && fd>0 && comp_by_ext(name, compressors) && !force) { warn("%s: already has a compression suffix -- unchanged\n", name); close(fd); return; } if (op && fd>0 && !(fcomp = comp_by_ext(name, decompressors)) && !(cat && force)) { warn("%s: unknown suffix -- ignored\n", name); close(fd); return; } if (op == 't') out = -1; else if (fd <= 0 || cat) { out = 1; if (!op && !force & isatty(1)) die("%s: refusing to write compressed data to a terminal, use -f to force.\n", exe); } else { if (op) name2 = strndup(name, strlen(name) - strlen(fcomp->ext)); else asprintf(&name2, "%s%s", name, comp->ext); if (!force) { if (!faccessat(dir, name2, F_OK, AT_EACCESS)) FAIL("%s%s already exists.\n", path, name2); if (errno != ENOENT) FAIL("%s%s: %m\n", path, name2); } #ifdef O_TMPFILE out = openat(dir, ".", O_TMPFILE|O_WRONLY|O_CLOEXEC|O_LARGEFILE, 0666); #endif if (out == -1) { notmp = 1; out = openat(dir, name2, O_CREAT|O_TRUNC|O_WRONLY|O_CLOEXEC|O_LARGEFILE|O_NOFOLLOW, 0600); if (out == -1) FAIL("can't create %s%s: %m\n", path, name2); } } file_info fi; fi.path = path; fi.name_in = name; fi.name_out = name2; fi.sz = fi.sd = 0; if (op? decomp(cat && force, fd, out, &fi) : fcomp->comp(fd, out, &fi, 0)) { err = 1; goto closure; } if (out > 2) { if (st) { struct timespec ts[2]; ts[0] = st->st_atim; ts[1] = st->st_mtim; futimens(out, ts); // ignore errors int perms = st->st_mode & 07777; if (fchown(out, st->st_uid, st->st_gid)) perms &= 0777; fchmod(out, perms); } if (!notmp && flink(dir, out, name2)) { if (errno==EEXIST && force) { if (unlinkat(dir, name2, 0)) FAIL("can't remove %s%s: %m\n", path, name2); if (!flink(dir, out, name2)) goto flink_ok; } FAIL("can't link %s%s: %m\n", path, name2); } notmp = 0; flink_ok: if (!keep && unlinkat(dir, name, 0)) FAIL("can't remove %s%s: %m\n", path, name); } if (verbose) { if (fi.sd) fprintf(stderr, "%s%s: %llu %s %llu (%llu%%)\n", path, name, fi.sd, op? "←" : "→", fi.sz, (200 * fi.sz / fi.sd + 1) / 2); // round to closest percent else fprintf(stderr, "%s%s: 0 %s %llu (header)\n", path, name, op? "←" : "→", fi.sz); } closure: if (notmp) if (unlinkat(dir, name2, 0)) fprintf(stderr, "%s: can't remove temporary file %s%s: %m\n", exe, path, name2); if (fd > 2) close(fd); if (out > 2) close(out); if (name2) free(name2); } // may be actually a file static void do_dir(int dir, const char *name, const char *path) { int dirfd = openat(dir, name, O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_LARGEFILE); if (dirfd == -1) FAIL("can't read %s%s: %m\n", path, name); struct stat64 sb; if (fstat64(dirfd, &sb)) FAIL("can't stat %s%s: %m\n", path, name); if (S_ISREG(sb.st_mode)) return do_file(dir, name, path, dirfd, &sb); if (!recurse) { warn("%s%s is not a regular file -- ignored\n", path, name); close(dirfd); return; } if (!S_ISDIR(sb.st_mode)) { warn("%s%s is not a directory or a regular file -- ignored\n", path, name); close(dirfd); return; } char *newpath = alloca(strlen(path) + strlen(name) + 2); if (!newpath) FAIL("out of stack in %s\n", path); sprintf(newpath, "%s%s/", path, name); DIR *d = fdopendir(dirfd); if (!d) FAIL("can't list %s%s: %m\n", path, name); struct dirent *de; while ((de = readdir(d))) { // "." or ".." if (de->d_name[0]=='.' && (!de->d_name[1] || de->d_name[1]=='.' && !de->d_name[2])) continue; if (de->d_type!=DT_DIR && de->d_type!=DT_REG && de->d_type!=DT_UNKNOWN) { warn("%s%s is not a directory or a regular file -- ignored\n", path, de->d_name); continue; } do_dir(dirfd, de->d_name, newpath); } closedir(d); return; closure: close(dirfd); } static const char* guess_prog(void) { const char *progs[][3] = { {"gzip", "gunzip", "gzcat"}, {"bzip2", "bunzip2", "bzcat"}, {"xz", "unxz", "xzcat"}, {"zstd", "unzstd", "zstdcat"}, {"zst", "unzst", "zstcat"}, {"bzip3", "bunzip3", "bz3cat"}, {"lz4", "unlz4", "lz4cat"}, {"lzop", 0, 0}, {"brotli","unbrotli",0}, {"lzip", 0, 0}, {"pack", 0, 0}, {"compress", "uncompress", 0}, }; for (int i=0; i= argc) do_file(-1, "stdin", "", 0, 0); else for (; optind < argc; optind++) do_dir(AT_FDCWD, argv[optind], ""); return err; } zst-0.4/zst.h000066400000000000000000000000521441430656400131720ustar00rootroot00000000000000extern const char *exe; extern int level;