lz4-1.28.1/.cargo_vcs_info.json0000644000000001360000000000100116360ustar { "git": { "sha1": "287dbd063a53738917f1078192185839215d10a0" }, "path_in_vcs": "" }lz4-1.28.1/.gitattributes000064400000000000000000000000231046102023000133140ustar 00000000000000*.rs text eol=lf lz4-1.28.1/.github/workflows/test.yaml000064400000000000000000000043411046102023000156700ustar 00000000000000name: Test on: pull_request: push: branches: - master env: CARGO_INCREMENTAL: 0 jobs: test-nix: strategy: matrix: platform: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.platform }} steps: - name: Setup Rust uses: actions-rs/toolchain@v1 with: toolchain: 1.65.0 override: true components: rustfmt - name: Add wasm32 target run: rustup target add wasm32-unknown-unknown - name: Checkout uses: actions/checkout@v2 with: submodules: true - name: Check Rust formatting run: cargo fmt -- --check - name: build run: cargo build --release - name: unit tests run: cargo test -- --nocapture - name: (MacOS) install LLVM uses: KyleMayes/install-llvm-action@v2 if: "${{ matrix.platform == 'macos-latest' }}" with: version: "17" - name: (MacOS) set LLVM as CC if: "${{ matrix.platform == 'macos-latest' }}" run: echo "CC=$(pwd)/llvm/bin/clang-17" >> $GITHUB_ENV - name: build (wasm32) run: cargo build --target wasm32-unknown-unknown - name: check (wasm32) run: cargo check --target wasm32-unknown-unknown test-windows: runs-on: windows-latest steps: - name: Setup Rust uses: actions-rs/toolchain@v1 with: toolchain: 1.65.0 target: i686-pc-windows-msvc override: true components: rustfmt - name: Checkout uses: actions/checkout@v2 with: submodules: true - name: Check Rust formatting run: cargo fmt -- --check - name: build 64-bit run: cargo build --release - name: build 32-bit run: cargo build --release --target i686-pc-windows-msvc - name: build 64-bit static env: CRT_STATIC: "true" RUSTFLAGS: "-C target-feature=+crt-static" run: cargo build --release - name: build 32-bit static env: CRT_STATIC: "true" RUSTFLAGS: "-C target-feature=+crt-static" run: cargo build --release --target i686-pc-windows-msvc - name: unit tests run: cargo test -- --nocapture lz4-1.28.1/.gitignore000064400000000000000000000002341046102023000124150ustar 00000000000000# Compiled files *.o *.so *.rlib *.dll # Executables *.exe # Generated by Cargo /target/ /*.lock *.rs.bk /lz4-sys/target/ /lz4-sys/*.lock # Idea /.idea/ lz4-1.28.1/.gitmodules000064400000000000000000000001221046102023000125760ustar 00000000000000[submodule "liblz4"] path = lz4-sys/liblz4 url = https://github.com/lz4/lz4.git lz4-1.28.1/CHANGELOG.md000064400000000000000000000047751046102023000122540ustar 00000000000000 1.26.0: * Update to lz4 1.10.0 1.25.0: * Add content_size setting to Lz4FrameInfo * Add LZ4_setStreamDecode * Docs updates 1.24.0: * Update to lz4 1.9.4 (lz4-sys 1.9.4) - this fixes CVE-2021-3520, which was a security vulnerability in the core lz4 library * export the include directory of lz4 from build.rs 1.23.3 (March 5, 2022): * Update lz4 to 1.9.3 * Add `[de]compress_to_buffer` to block API to allow reusing buffers (#16) * Windows static lib support * Support favor_dec_speed * Misc small fixes 1.23.2: * Update lz4 to 1.9.2 * Remove dependency on skeptic (replace with build-dependency docmatic for README testing) * Move to Rust 2018 edition 1.23.0: * Update lz4 to v1.8.2 * Add lz4 block mode api 1.22.0: * Update lz4 to v1.8.0 * Remove lz4 redundant dependency to gcc #22 (thanks to Xidorn Quan) 1.21.1: * Fix always rebuild issue #21 1.21.0: * Fix smallest 11-byte stream decoding (thanks to Niklas Hambüchen) * Update lz4 to v1.7.5 1.20.0: * Split out separate sys package #16 (thanks to Thijs Cadier) 1.19.173: * Update lz4 to v1.7.3 1.19.131: * Update dependencies for correct work with change build environmet via `rustup override` 1.18.131: * Implemented Send for Encoder/Decoder #15 (thanks to Maxime Lenoir) 1.17.131: * Don't leave Decoder in invalid state if read fails #14 (thanks to bvinc83) 1.16.131: * Don't use -ftree-vectorize optimization on i686-pc-windows-gnu for prevent crash 1.15.131: * Add Encoder.writer() and Decoder.reader() methods (thanks to Paul Grandperrin) 1.14.131: * Modified build script to *always* compile the C code with -O3 optimization #11 (thanks to TQ Hirsch) * Import libc crate in libc.rs to fix warnings on rust-nightly #10 (thanks to TQ Hirsch) 1.13.131: * Remove wildcard dependencies for rust 1.6 1.12.131: * Fix pointer invalidation on Decoder move #8 (thanks to Maxime Lenoir) 1.11.131: * Add missing method Decoder::finish for unwrapping original Read stream 1.10.131: * Fix conflicting import on Rust nightly (thanks to maximelenoir) * Don't export the modules in the public API (thanks to Corey "See More" Richardson) 1.9.131: * Do not wait for fill whole buffer on read. It's usefull for read network stream (thanks to Brian Vincent) 1.8.131: * Update lz4 to v131 * Fix incorrect break that could cause reading after a frame ends (thanks to Brian Vincent) * Fix typo in Cargo.toml 1.7.129: * Autopublish rustdoc * Remove libc type publishing 1.6.129: * Update lz4 to r129 * Add tests * Rustup: 1.0.0-beta lz4-1.28.1/Cargo.lock0000644000000050170000000000100076140ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 3 [[package]] name = "cc" version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ "shlex", ] [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "getrandom" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", "wasi", ] [[package]] name = "libc" version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "lz4" version = "1.28.1" dependencies = [ "lz4-sys", "rand", ] [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", ] [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", ] [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" lz4-1.28.1/Cargo.toml0000644000000017420000000000100076400ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "lz4" version = "1.28.1" authors = [ "Jens Heyens ", "Artem V. Navrotskiy ", "Patrick Marks ", ] description = "Rust LZ4 bindings library." documentation = "https://docs.rs/lz4" readme = "README.md" license = "MIT" repository = "https://github.com/10xGenomics/lz4-rs" [[bin]] name = "lz4" test = false doc = false [dependencies.lz4-sys] version = "1.11.1+lz4-1.10.0" [dev-dependencies.rand] version = ">=0.7, <=0.8" lz4-1.28.1/Cargo.toml.orig000064400000000000000000000010211046102023000133070ustar 00000000000000[package] name = "lz4" license = "MIT" version = "1.28.1" readme = "README.md" authors = [ "Jens Heyens ", "Artem V. Navrotskiy ", "Patrick Marks "] description = "Rust LZ4 bindings library." repository = "https://github.com/10xGenomics/lz4-rs" documentation = "https://docs.rs/lz4" edition = "2018" [[bin]] name = "lz4" test = false doc = false [dependencies] lz4-sys = { path = "lz4-sys", version = "1.11.1+lz4-1.10.0" } [dev-dependencies] rand = ">=0.7, <=0.8" lz4-1.28.1/LICENSE000064400000000000000000000020771046102023000114410ustar 00000000000000The MIT License (MIT) Copyright (c) 2015 Artem V. Navrotskiy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. lz4-1.28.1/README.md000064400000000000000000000047431046102023000117150ustar 00000000000000lz4 ==== [![Build Status](https://github.com/10XGenomics/lz4-rs/workflows/Test/badge.svg?branch=master)](https://github.com/10XGenomics/lz4-rs/actions?query=branch%3Amaster) [![Crates.io](https://img.shields.io/crates/v/lz4.svg)](https://crates.io/crates/lz4) [![Join the chat at https://gitter.im/bozaro/lz4-rs](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/bozaro/lz4-rs?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Rustdoc](https://img.shields.io/badge/doc-rustdoc-green.svg)](https://docs.rs/lz4/) NOTE: 10xGenomics is the new official home of lz4-rs, replacing [https://github.com/bozaro/lz4-rs](https://github.com/bozaro/lz4-rs) This repository contains binding for lz4 compression library (https://github.com/Cyan4973/lz4). LZ4 is a very fast lossless compression algorithm, providing compression speed at 400 MB/s per core, with near-linear scalability for multi-threaded applications. It also features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. ## Usage Put this in your `Cargo.toml`: ```toml [dependencies] lz4 = "1.23.1" ``` Sample code for compression/decompression: ```rust extern crate lz4; use std::env; use std::fs::File; use std::io::{self, Result}; use std::path::{Path, PathBuf}; use lz4::{Decoder, EncoderBuilder}; fn main() { println!("LZ4 version: {}", lz4::version()); for path in env::args().skip(1).map(PathBuf::from) { if let Some("lz4") = path.extension().and_then(|e| e.to_str()) { decompress(&path, &path.with_extension("")).unwrap(); } else { compress(&path, &path.with_extension("lz4")).unwrap(); } } } fn compress(source: &Path, destination: &Path) -> Result<()> { println!("Compressing: {} -> {}", source.display(), destination.display()); let mut input_file = File::open(source)?; let output_file = File::create(destination)?; let mut encoder = EncoderBuilder::new() .level(4) .build(output_file)?; io::copy(&mut input_file, &mut encoder)?; let (_output, result) = encoder.finish(); result } fn decompress(source: &Path, destination: &Path) -> Result<()> { println!("Decompressing: {} -> {}", source.display(), destination.display()); let input_file = File::open(source)?; let mut decoder = Decoder::new(input_file)?; let mut output_file = File::create(destination)?; io::copy(&mut decoder, &mut output_file)?; Ok(()) } ``` lz4-1.28.1/src/bin/lz4.rs000064400000000000000000000026301046102023000130450ustar 00000000000000extern crate lz4; use std::env; use std::fs::File; use std::io::Read; use std::io::Result; use std::io::Write; use std::iter::FromIterator; use std::path::Path; fn main() { println!("LZ4 version: {}", lz4::version()); let suffix = ".lz4"; for arg in Vec::from_iter(env::args())[1..].iter() { if arg.ends_with(suffix) { decompress( &Path::new(arg), &Path::new(&arg[0..arg.len() - suffix.len()]), ) .unwrap(); } else { compress(&Path::new(arg), &Path::new(&(arg.to_string() + suffix))).unwrap(); } } } fn compress(src: &Path, dst: &Path) -> Result<()> { println!("Compressing: {:?} -> {:?}", src, dst); let mut fi = File::open(src)?; let mut fo = lz4::EncoderBuilder::new().build(File::create(dst)?)?; copy(&mut fi, &mut fo)?; match fo.finish() { (_, result) => result, } } fn decompress(src: &Path, dst: &Path) -> Result<()> { println!("Decompressing: {:?} -> {:?}", src, dst); let mut fi = lz4::Decoder::new(File::open(src)?)?; let mut fo = File::create(dst)?; copy(&mut fi, &mut fo) } fn copy(src: &mut dyn Read, dst: &mut dyn Write) -> Result<()> { let mut buffer: [u8; 1024] = [0; 1024]; loop { let len = src.read(&mut buffer)?; if len == 0 { break; } dst.write_all(&buffer[0..len])?; } Ok(()) } lz4-1.28.1/src/block/mod.rs000064400000000000000000000343471046102023000134470ustar 00000000000000//! This module provides access to the block mode functions of the lz4 C library. //! It somehow resembles the [Python-lz4](http://python-lz4.readthedocs.io/en/stable/lz4.block.html) api, //! but using Rust's Option type, the function parameters have been a little simplified. //! As does python-lz4, this module supports prepending the compressed buffer with a u32 value //! representing the size of the original, uncompressed data. //! //! # Examples //! ``` //! //! use lz4::block::{compress,decompress}; //! //! let v = vec![0u8; 1024]; //! //! let comp_with_prefix = compress(&v, None, true).unwrap(); //! let comp_wo_prefix = compress(&v, None, false).unwrap(); //! //! assert_eq!(v, decompress(&comp_with_prefix, None).unwrap()); //! assert_eq!(v, decompress(&comp_wo_prefix, Some(1024)).unwrap()); //! ``` use super::c_char; use super::liblz4::*; use std::io::{Error, ErrorKind, Result}; /// Represents the compression mode do be used. #[derive(Clone, Copy, Debug)] pub enum CompressionMode { /// High compression with compression parameter HIGHCOMPRESSION(i32), /// Fast compression with acceleration paramet FAST(i32), /// Default compression DEFAULT, } impl Default for CompressionMode { fn default() -> Self { CompressionMode::DEFAULT } } /// Returns the size of the buffer that is guaranteed to hold the result of /// compressing `uncompressed_size` bytes of in data. Returns std::io::Error /// with ErrorKind::InvalidInput if input data is too long to be compressed by lz4. pub fn compress_bound(uncompressed_size: usize) -> Result { // 0 iff src too large let compress_bound: i32 = unsafe { LZ4_compressBound(uncompressed_size as i32) }; if uncompressed_size > (i32::max_value() as usize) || compress_bound <= 0 { return Err(Error::new( ErrorKind::InvalidInput, "Compression input too long.", )); } Ok(compress_bound as usize) } /// Compresses the full src buffer using the specified CompressionMode, where None and Some(Default) /// are treated equally. If prepend_size is set, the source length will be prepended to the output /// buffer. /// /// /// # Errors /// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too long. /// Returns std::io::Error with ErrorKind::Other if the compression failed inside the C library. If /// this happens, the C api was not able to provide more information about the cause. pub fn compress(src: &[u8], mode: Option, prepend_size: bool) -> Result> { // 0 iff src too large let compress_bound: i32 = unsafe { LZ4_compressBound(src.len() as i32) }; if src.len() > (i32::max_value() as usize) || compress_bound <= 0 { return Err(Error::new( ErrorKind::InvalidInput, "Compression input too long.", )); } let mut compressed: Vec = vec![ 0; (if prepend_size { compress_bound + 4 } else { compress_bound }) as usize ]; let dec_size = compress_to_buffer(src, mode, prepend_size, &mut compressed)?; compressed.truncate(dec_size as usize); Ok(compressed) } /// Compresses the full `src` buffer using the specified CompressionMode, where None and Some(Default) /// are treated equally, writing compressed bytes to `buffer`. /// /// # Errors /// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too long. /// The buffer cannot be larger than `i32::MAX`. /// Returns std::io::Error with ErrorKind::Other if the compression data does not fit in `buffer`. pub fn compress_to_buffer( src: &[u8], mode: Option, prepend_size: bool, buffer: &mut [u8], ) -> Result { // check that src isn't too big for lz4 let max_len: i32 = unsafe { LZ4_compressBound(src.len() as i32) }; if src.len() > (i32::max_value() as usize) || max_len <= 0 { return Err(Error::new( ErrorKind::InvalidInput, "Compression input too long.", )); } let dec_size; { let dst_buf = if prepend_size { let size = src.len() as u32; buffer[0] = size as u8; buffer[1] = (size >> 8) as u8; buffer[2] = (size >> 16) as u8; buffer[3] = (size >> 24) as u8; &mut buffer[4..] } else { buffer }; let buf_len = dst_buf.len() as i32; dec_size = match mode { Some(CompressionMode::HIGHCOMPRESSION(level)) => unsafe { LZ4_compress_HC( src.as_ptr() as *const c_char, dst_buf.as_mut_ptr() as *mut c_char, src.len() as i32, buf_len, level, ) }, Some(CompressionMode::FAST(accel)) => unsafe { LZ4_compress_fast( src.as_ptr() as *const c_char, dst_buf.as_mut_ptr() as *mut c_char, src.len() as i32, buf_len, accel, ) }, _ => unsafe { LZ4_compress_default( src.as_ptr() as *const c_char, dst_buf.as_mut_ptr() as *mut c_char, src.len() as i32, buf_len, ) }, }; } if dec_size <= 0 { return Err(Error::new(ErrorKind::Other, "Compression failed")); } let written_size = if prepend_size { dec_size + 4 } else { dec_size }; Ok(written_size as usize) } fn get_decompressed_size(src: &[u8], uncompressed_size: Option) -> Result { let size; if let Some(s) = uncompressed_size { size = s; } else { if src.len() < 4 { return Err(Error::new( ErrorKind::InvalidInput, "Source buffer must at least contain size prefix.", )); } size = (src[0] as i32) | (src[1] as i32) << 8 | (src[2] as i32) << 16 | (src[3] as i32) << 24; } if size < 0 { return Err(Error::new( ErrorKind::InvalidInput, if uncompressed_size.is_some() { "Size parameter must not be negative." } else { "Parsed size prefix in buffer must not be negative." }, )); } if unsafe { LZ4_compressBound(size) } <= 0 { return Err(Error::new( ErrorKind::InvalidInput, "Given size parameter is too big", )); } Ok(size as usize) } /// Decompresses the src buffer. If uncompressed_size is None, the source length will be read from /// the start of the input buffer. /// /// /// # Errors /// Returns std::io::Error with ErrorKind::InvalidInput if the src buffer is too short, the /// provided (or parsed) uncompressed_size is to large or negative. /// Returns std::io::Error with ErrorKind::InvalidData if the decompression failed inside the C /// library. This is most likely due to malformed input. /// pub fn decompress(src: &[u8], uncompressed_size: Option) -> Result> { let size = get_decompressed_size(src, uncompressed_size)?; let mut buffer = vec![0u8; size]; let sz = decompress_to_buffer(src, uncompressed_size, &mut buffer)?; buffer.truncate(sz); Ok(buffer) } pub fn decompress_to_buffer( mut src: &[u8], uncompressed_size: Option, buffer: &mut [u8], ) -> Result { let size; if let Some(s) = uncompressed_size { size = s; } else { if src.len() < 4 { return Err(Error::new( ErrorKind::InvalidInput, "Source buffer must at least contain size prefix.", )); } size = (src[0] as i32) | (src[1] as i32) << 8 | (src[2] as i32) << 16 | (src[3] as i32) << 24; src = &src[4..]; } if size < 0 { return Err(Error::new( ErrorKind::InvalidInput, if uncompressed_size.is_some() { "Size parameter must not be negative." } else { "Parsed size prefix in buffer must not be negative." }, )); } if unsafe { LZ4_compressBound(size) } <= 0 { return Err(Error::new( ErrorKind::InvalidInput, "Given size parameter is too big", )); } if size as usize > buffer.len() { return Err(Error::new( ErrorKind::InvalidInput, "buffer isn't large enough to hold decompressed data", )); } let dec_bytes = unsafe { LZ4_decompress_safe( src.as_ptr() as *const c_char, buffer.as_mut_ptr() as *mut c_char, src.len() as i32, size, ) }; if dec_bytes < 0 { return Err(Error::new( ErrorKind::InvalidData, "Decompression failed. Input invalid or too long?", )); } Ok(dec_bytes as usize) } #[cfg(test)] mod test { use crate::block::{compress, decompress, decompress_to_buffer, CompressionMode}; use super::compress_to_buffer; /// This test will fail unless the buffer created by decompress is correctly truncated #[test] fn decompress_truncate_test() { let src = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111".as_bytes(); let rs_compressed = compress(src, None, false).unwrap(); let rs_compressed_rs_uncompressed = decompress(&rs_compressed, Some((src.len() as i32) * 256)).unwrap(); // compare the uncompressed result from rust assert_eq!(rs_compressed_rs_uncompressed, src,); } #[test] fn test_compression_without_prefix() { let size = 65536; let mut to_compress = Vec::with_capacity(size); for i in 0..size { to_compress.push(i as u8); } let mut v: Vec> = vec![]; for i in 1..100 { v.push(compress(&to_compress, Some(CompressionMode::FAST(i)), false).unwrap()); } // 12 is max high compression parameter for i in 1..12 { v.push( compress( &to_compress, Some(CompressionMode::HIGHCOMPRESSION(i)), false, ) .unwrap(), ); } v.push(compress(&to_compress, None, false).unwrap()); for val in v { assert_eq!( decompress(&val, Some(to_compress.len() as i32)).unwrap(), to_compress ); } } #[test] fn test_compression_with_prefix() { let size = 65536; let mut to_compress = Vec::with_capacity(size); for i in 0..size { to_compress.push(i as u8); } let mut v: Vec> = vec![]; for i in 1..100 { v.push(compress(&to_compress, Some(CompressionMode::FAST(i)), true).unwrap()); } // 12 is max high compression parameter for i in 1..12 { v.push( compress( &to_compress, Some(CompressionMode::HIGHCOMPRESSION(i)), true, ) .unwrap(), ); } v.push(compress(&to_compress, None, true).unwrap()); for val in v { assert_eq!(decompress(&val, None).unwrap(), to_compress); } } #[test] fn test_compress_to_buffer() { let data = b"qfn3489fqASFqvegrwe$%344thI,..kmTMN3 g{P}wefwf2fv2443ef3443RT[]rete$7-80956GRWbefvw@fVGrwGB24tggrm%&*I@!"; // test what happens when not enough space is available. let mut small_buf = vec![0; 5]; let r = compress_to_buffer(&data[..], None, true, &mut small_buf); assert!(r.is_err()); let mut big_buf = vec![0; 1000]; let r = compress_to_buffer(&data[..], None, true, &mut big_buf).unwrap(); assert_eq!(big_buf[r], 0); let mut dec_buf = vec![0u8; data.len() + 1]; let dec_bytes = decompress_to_buffer(&big_buf[..r], None, &mut dec_buf).unwrap(); assert_eq!(&dec_buf[..dec_bytes], &data[..]); } #[test] fn test_decompression_with_prefix() { let compressed: [u8; 250] = [ 0, 188, 0, 0, 255, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 116, 101, 115, 116, 32, 115, 116, 114, 105, 110, 103, 32, 99, 111, 109, 112, 114, 101, 115, 115, 101, 100, 32, 98, 121, 32, 112, 121, 116, 104, 111, 110, 45, 108, 122, 52, 32, 47, 0, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 117, 80, 45, 108, 122, 52, 32, ]; let mut reference: String = String::new(); for _ in 0..1024 { reference += "this is a test string compressed by python-lz4 "; } assert_eq!(decompress(&compressed, None).unwrap(), reference.as_bytes()) } #[test] fn test_empty_compress() { use crate::block::{compress, decompress}; let v = vec![0u8; 0]; let comp_with_prefix = compress(&v, None, true).unwrap(); dbg!(&comp_with_prefix); assert_eq!(v, decompress(&comp_with_prefix, None).unwrap()); } } lz4-1.28.1/src/decoder.rs000064400000000000000000000252151046102023000131750ustar 00000000000000use super::liblz4::*; use super::size_t; use std::io::{Error, ErrorKind, Read, Result}; use std::ptr; const BUFFER_SIZE: usize = 32 * 1024; // NOTE: unsafe to device Clone or Copy, otherwise // there can be multiple copies of the same inner LZ4 pointer #[derive(Debug)] struct DecoderContext { c: LZ4FDecompressionContext, } // NOTE: unsafe to derive Clone or Copy #[derive(Debug)] pub struct Decoder { c: DecoderContext, r: R, buf: Box<[u8]>, pos: usize, len: usize, next: usize, } // No interior mutability, so Decoder is Sync as long as R is Sync. unsafe impl Sync for Decoder {} impl Decoder { /// Creates a new decoder which reads its input from the given /// input stream. The input stream can be re-acquired by calling /// `finish()` pub fn new(r: R) -> Result> { Ok(Decoder { r, c: DecoderContext::new()?, buf: vec![0; BUFFER_SIZE].into_boxed_slice(), pos: BUFFER_SIZE, len: BUFFER_SIZE, // Minimal LZ4 stream size next: 11, }) } /// Immutable reader reference. pub fn reader(&self) -> &R { &self.r } pub fn finish(self) -> (R, Result<()>) { ( self.r, match self.next { 0 => Ok(()), _ => Err(Error::new( ErrorKind::Interrupted, "Finish runned before read end of compressed stream", )), }, ) } } impl Read for Decoder { fn read(&mut self, buf: &mut [u8]) -> Result { if self.next == 0 || buf.is_empty() { return Ok(0); } let mut dst_offset: usize = 0; while dst_offset == 0 { if self.pos >= self.len { let need = if self.buf.len() < self.next { self.buf.len() } else { self.next }; self.len = self.r.read(&mut self.buf[0..need])?; // NOTE: we do not exit here if there was nothing read // The lz4 context may still have more bytes to emit. self.pos = 0; self.next -= self.len; } while (dst_offset < buf.len()) && ((self.pos < self.len) || self.len == 0) { let mut src_size = (self.len - self.pos) as size_t; let mut dst_size = (buf.len() - dst_offset) as size_t; let len = check_error(unsafe { LZ4F_decompress( self.c.c, buf[dst_offset..].as_mut_ptr(), &mut dst_size, self.buf[self.pos..].as_ptr(), &mut src_size, ptr::null(), ) })?; self.pos += src_size as usize; dst_offset += dst_size as usize; // We need to keep trying to read bytes from the decompressor // until it is no longer emitting them, even after it // has finished reading bytes. if dst_size == 0 && src_size == 0 { return Ok(dst_offset); } if len == 0 { self.next = 0; return Ok(dst_offset); } else if self.next < len { self.next = len; } } } Ok(dst_offset) } } impl DecoderContext { fn new() -> Result { let mut context = LZ4FDecompressionContext(ptr::null_mut()); check_error(unsafe { LZ4F_createDecompressionContext(&mut context, LZ4F_VERSION) })?; Ok(DecoderContext { c: context }) } } impl Drop for DecoderContext { fn drop(&mut self) { unsafe { LZ4F_freeDecompressionContext(self.c) }; } } #[cfg(test)] mod test { extern crate rand; use self::rand::rngs::StdRng; use self::rand::Rng; use super::super::encoder::{Encoder, EncoderBuilder}; use super::Decoder; use std::io::{Cursor, Error, ErrorKind, Read, Result, Write}; const BUFFER_SIZE: usize = 64 * 1024; const END_MARK: [u8; 4] = [0x9f, 0x77, 0x22, 0x71]; struct ErrorWrapper { r: R, rng: Rn, } impl ErrorWrapper { fn new(rng: Rn, read: R) -> Self { ErrorWrapper { r: read, rng } } } impl Read for ErrorWrapper { fn read(&mut self, buf: &mut [u8]) -> Result { if self.rng.next_u32() & 0x03 == 0 { self.r.read(buf) } else { Err(Error::new(ErrorKind::Other, "Opss...")) } } } struct RetryWrapper { r: R, } impl RetryWrapper { fn new(read: R) -> Self { RetryWrapper { r: read } } } impl Read for RetryWrapper { fn read(&mut self, buf: &mut [u8]) -> Result { loop { match self.r.read(buf) { Ok(v) => { return Ok(v); } Err(e) => { if e.kind() == ErrorKind::Other { continue; } return Err(e); } } } } } fn finish_encode(encoder: Encoder) -> W { let (mut buffer, result) = encoder.finish(); result.unwrap(); buffer.write(&END_MARK).unwrap(); buffer } fn finish_decode(decoder: Decoder) { let (buffer, result) = decoder.finish(); result.unwrap(); let mut mark = Vec::new(); let mut data = Vec::new(); mark.write(&END_MARK).unwrap(); RetryWrapper::new(buffer).read_to_end(&mut data).unwrap(); assert_eq!(mark, data); } #[test] fn test_decoder_empty() { let expected: Vec = Vec::new(); let buffer = finish_encode(EncoderBuilder::new().level(1).build(Vec::new()).unwrap()); let mut decoder = Decoder::new(Cursor::new(buffer)).unwrap(); let mut actual = Vec::new(); decoder.read_to_end(&mut actual).unwrap(); assert_eq!(expected, actual); finish_decode(decoder); } #[test] fn test_decoder_smallest() { let expected: Vec = Vec::new(); let mut buffer = b"\x04\x22\x4d\x18\x40\x40\xc0\x00\x00\x00\x00".to_vec(); buffer.write(&END_MARK).unwrap(); let mut decoder = Decoder::new(Cursor::new(buffer)).unwrap(); let mut actual = Vec::new(); decoder.read_to_end(&mut actual).unwrap(); assert_eq!(expected, actual); finish_decode(decoder); } #[test] fn test_decoder_smoke() { let mut encoder = EncoderBuilder::new().level(1).build(Vec::new()).unwrap(); let mut expected = Vec::new(); expected.write(b"Some data").unwrap(); encoder.write(&expected[..4]).unwrap(); encoder.write(&expected[4..]).unwrap(); let buffer = finish_encode(encoder); let mut decoder = Decoder::new(Cursor::new(buffer)).unwrap(); let mut actual = Vec::new(); decoder.read_to_end(&mut actual).unwrap(); assert_eq!(expected, actual); finish_decode(decoder); } #[test] fn test_decoder_random() { let mut rnd = random(); let expected = random_stream(&mut rnd, 1027 * 1023 * 7); let mut encoder = EncoderBuilder::new().level(1).build(Vec::new()).unwrap(); encoder.write(&expected).unwrap(); let encoded = finish_encode(encoder); let mut decoder = Decoder::new(Cursor::new(encoded)).unwrap(); let mut actual = Vec::new(); loop { let mut buffer = [0; BUFFER_SIZE]; let size = decoder.read(&mut buffer).unwrap(); if size == 0 { break; } actual.write(&buffer[0..size]).unwrap(); } assert_eq!(expected, actual); finish_decode(decoder); } #[test] fn test_retry_read() { let mut rnd = random(); let expected = random_stream(&mut rnd, 1027 * 1023 * 7); let mut encoder = EncoderBuilder::new().level(1).build(Vec::new()).unwrap(); encoder.write(&expected).unwrap(); let encoded = finish_encode(encoder); let mut decoder = Decoder::new(ErrorWrapper::new(rnd.clone(), Cursor::new(encoded))).unwrap(); let mut actual = Vec::new(); loop { let mut buffer = [0; BUFFER_SIZE]; match decoder.read(&mut buffer) { Ok(size) => { if size == 0 { break; } actual.write(&buffer[0..size]).unwrap(); } Err(_) => {} } } assert_eq!(expected, actual); finish_decode(decoder); } /// Ensure that we emit the full decompressed stream even if we're /// using a very small output buffer. #[test] fn issue_45() { // create an encoder let mut enc = crate::EncoderBuilder::new().build(Vec::new()).unwrap(); // write 'a' 100 times to the encoder let text: Vec = vec!['a' as u8; 100]; enc.write_all(&text[..]).unwrap(); // flush the encoder enc.flush().unwrap(); // read from the decoder, buf_size bytes at a time for buf_size in [5, 10, 15, 20, 25] { let mut buf = vec![0; buf_size]; let mut total_bytes_read = 0; // create a decoder wrapping the backing buffer let mut dec = crate::Decoder::new(&enc.writer()[..]).unwrap(); while let Ok(n) = dec.read(&mut buf[..]) { if n == 0 { break; } total_bytes_read += n; } assert_eq!(total_bytes_read, text.len()); } } fn random() -> StdRng { let seed: [u8; 32] = [ 157, 164, 190, 237, 231, 103, 60, 22, 197, 108, 51, 176, 30, 170, 155, 21, 163, 249, 56, 192, 57, 112, 142, 240, 233, 46, 51, 122, 222, 137, 225, 243, ]; rand::SeedableRng::from_seed(seed) } fn random_stream(rng: &mut R, size: usize) -> Vec { (0..size).map(|_| rng.gen()).collect() } #[test] fn test_decoder_send() { fn check_send(_: &S) {} let dec = Decoder::new(Cursor::new(Vec::new())).unwrap(); check_send(&dec); } } lz4-1.28.1/src/encoder.rs000064400000000000000000000217251046102023000132110ustar 00000000000000use super::liblz4::*; use super::size_t; use std::cmp; use std::io::Result; use std::io::Write; use std::ptr; #[derive(Debug)] struct EncoderContext { c: LZ4FCompressionContext, } #[derive(Clone, Debug)] pub struct EncoderBuilder { block_size: BlockSize, block_mode: BlockMode, // 1: each block followed by a checksum of block's compressed data; 0: disabled (default) block_checksum: BlockChecksum, checksum: ContentChecksum, // 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 level: u32, // 1 == always flush (reduce need for tmp buffer) auto_flush: bool, favor_dec_speed: bool, content_size: u64, } #[derive(Debug)] pub struct Encoder { c: EncoderContext, w: W, limit: usize, buffer: Vec, } impl EncoderBuilder { pub fn new() -> Self { EncoderBuilder { block_size: BlockSize::Default, block_mode: BlockMode::Linked, checksum: ContentChecksum::ChecksumEnabled, block_checksum: BlockChecksum::BlockChecksumEnabled, level: 0, auto_flush: false, favor_dec_speed: false, content_size: 0, } } pub fn block_size(&mut self, block_size: BlockSize) -> &mut Self { self.block_size = block_size; self } pub fn block_mode(&mut self, block_mode: BlockMode) -> &mut Self { self.block_mode = block_mode; self } pub fn block_checksum(&mut self, block_checksum: BlockChecksum) -> &mut Self { self.block_checksum = block_checksum; self } pub fn checksum(&mut self, checksum: ContentChecksum) -> &mut Self { self.checksum = checksum; self } pub fn level(&mut self, level: u32) -> &mut Self { self.level = level; self } pub fn auto_flush(&mut self, auto_flush: bool) -> &mut Self { self.auto_flush = auto_flush; self } /// Favor decompression speed over compression ratio. Requires compression /// level >=10. pub fn favor_dec_speed(&mut self, favor_dec_speed: bool) -> &mut Self { self.favor_dec_speed = favor_dec_speed; self } pub fn content_size(&mut self, content_size: u64) -> &mut Self { self.content_size = content_size; self } pub fn build(&self, w: W) -> Result> { let block_size = self.block_size.get_size(); let preferences = LZ4FPreferences { frame_info: LZ4FFrameInfo { block_size_id: self.block_size.clone(), block_mode: self.block_mode.clone(), content_checksum_flag: self.checksum.clone(), content_size: self.content_size.clone(), frame_type: FrameType::Frame, dict_id: 0, block_checksum_flag: self.block_checksum.clone(), }, compression_level: self.level, auto_flush: if self.auto_flush { 1 } else { 0 }, favor_dec_speed: if self.favor_dec_speed { 1 } else { 0 }, reserved: [0; 3], }; let mut encoder = Encoder { w, c: EncoderContext::new()?, limit: block_size, buffer: Vec::with_capacity(check_error(unsafe { LZ4F_compressBound(block_size as size_t, &preferences) })?), }; encoder.write_header(&preferences)?; Ok(encoder) } } impl Encoder { fn write_header(&mut self, preferences: &LZ4FPreferences) -> Result<()> { unsafe { let len = check_error(LZ4F_compressBegin( self.c.c, self.buffer.as_mut_ptr(), self.buffer.capacity() as size_t, preferences, ))?; self.buffer.set_len(len); } self.w.write_all(&self.buffer) } fn write_end(&mut self) -> Result<()> { unsafe { let len = check_error(LZ4F_compressEnd( self.c.c, self.buffer.as_mut_ptr(), self.buffer.capacity() as size_t, ptr::null(), ))?; self.buffer.set_len(len); }; self.w.write_all(&self.buffer) } /// Immutable writer reference. pub fn writer(&self) -> &W { &self.w } /// This function is used to flag that this session of compression is done /// with. The stream is finished up (final bytes are written), and then the /// wrapped writer is returned. pub fn finish(mut self) -> (W, Result<()>) { let result = self.write_end(); (self.w, result) } } impl Write for Encoder { fn write(&mut self, buffer: &[u8]) -> Result { let mut offset = 0; while offset < buffer.len() { let size = cmp::min(buffer.len() - offset, self.limit); unsafe { let len = check_error(LZ4F_compressUpdate( self.c.c, self.buffer.as_mut_ptr(), self.buffer.capacity() as size_t, buffer[offset..].as_ptr(), size as size_t, ptr::null(), ))?; self.buffer.set_len(len); self.w.write_all(&self.buffer)?; } offset += size; } Ok(buffer.len()) } fn flush(&mut self) -> Result<()> { loop { unsafe { let len = check_error(LZ4F_flush( self.c.c, self.buffer.as_mut_ptr(), self.buffer.capacity() as size_t, ptr::null(), ))?; if len == 0 { break; } self.buffer.set_len(len); }; self.w.write_all(&self.buffer)?; } self.w.flush() } } impl EncoderContext { fn new() -> Result { let mut context = LZ4FCompressionContext(ptr::null_mut()); check_error(unsafe { LZ4F_createCompressionContext(&mut context, LZ4F_VERSION) })?; Ok(EncoderContext { c: context }) } } impl Drop for EncoderContext { fn drop(&mut self) { unsafe { LZ4F_freeCompressionContext(self.c) }; } } #[cfg(test)] mod test { use super::EncoderBuilder; use std::io::{Read, Write}; #[test] fn test_encoder_smoke() { let mut encoder = EncoderBuilder::new().level(1).build(Vec::new()).unwrap(); encoder.write(b"Some ").unwrap(); encoder.write(b"data").unwrap(); let (_, result) = encoder.finish(); result.unwrap(); } #[test] fn test_encoder_random() { let mut encoder = EncoderBuilder::new().level(1).build(Vec::new()).unwrap(); let mut input = Vec::new(); let mut rnd: u32 = 42; for _ in 0..1024 * 1024 { input.push((rnd & 0xFF) as u8); rnd = ((1664525 as u64) * (rnd as u64) + (1013904223 as u64)) as u32; } encoder.write(&input).unwrap(); let (compressed, result) = encoder.finish(); result.unwrap(); let mut dec = crate::decoder::Decoder::new(&compressed[..]).unwrap(); let mut output = Vec::new(); dec.read_to_end(&mut output).unwrap(); assert_eq!(input, output); } #[test] fn test_encoder_content_size() { let mut encoder = EncoderBuilder::new() .level(1) .content_size(1024 * 1024) .build(Vec::new()) .unwrap(); let mut input = Vec::new(); let mut rnd: u32 = 42; for _ in 0..1024 * 1024 { input.push((rnd & 0xFF) as u8); rnd = ((1664525 as u64) * (rnd as u64) + (1013904223 as u64)) as u32; } encoder.write(&input).unwrap(); let (compressed, result) = encoder.finish(); result.unwrap(); let mut dec = crate::decoder::Decoder::new(&compressed[..]).unwrap(); let mut output = Vec::new(); dec.read_to_end(&mut output).unwrap(); assert_eq!(input, output); } #[test] fn test_encoder_send() { fn check_send(_: &S) {} let enc = EncoderBuilder::new().build(Vec::new()); check_send(&enc); } #[test] fn test_favor_dec_speed() { let mut encoder = EncoderBuilder::new() .level(11) .favor_dec_speed(true) .build(Vec::new()) .unwrap(); let mut input = Vec::new(); let mut rnd: u32 = 42; for _ in 0..1024 * 1024 { input.push((rnd & 0xFF) as u8); rnd = ((1664525 as u64) * (rnd as u64) + (1013904223 as u64)) as u32; } encoder.write(&input).unwrap(); let (compressed, result) = encoder.finish(); result.unwrap(); let mut dec = crate::decoder::Decoder::new(&compressed[..]).unwrap(); let mut output = Vec::new(); dec.read_to_end(&mut output).unwrap(); assert_eq!(input, output); } } lz4-1.28.1/src/lib.rs000064400000000000000000000006211046102023000123300ustar 00000000000000#![doc = include_str!("../README.md")] extern crate lz4_sys; pub mod liblz4; mod decoder; mod encoder; pub mod block; pub use crate::decoder::Decoder; pub use crate::encoder::Encoder; pub use crate::encoder::EncoderBuilder; pub use crate::liblz4::version; pub use crate::liblz4::BlockMode; pub use crate::liblz4::BlockSize; pub use crate::liblz4::ContentChecksum; use lz4_sys::{c_char, size_t}; lz4-1.28.1/src/liblz4.rs000064400000000000000000000021541046102023000127650ustar 00000000000000use std::ffi::CStr; use std::fmt::Display; use std::fmt::Formatter; use std::io::Error; use std::io::ErrorKind; use std::str; pub use lz4_sys::*; #[derive(Debug)] pub struct LZ4Error(String); impl Display for LZ4Error { fn fmt(&self, f: &mut Formatter) -> Result<(), ::std::fmt::Error> { write!(f, "LZ4 error: {}", &self.0) } } impl ::std::error::Error for LZ4Error { fn description(&self) -> &str { &self.0 } fn cause(&self) -> Option<&dyn (::std::error::Error)> { None } } pub fn check_error(code: LZ4FErrorCode) -> Result { unsafe { if LZ4F_isError(code) != 0 { let error_name = LZ4F_getErrorName(code); return Err(Error::new( ErrorKind::Other, LZ4Error( str::from_utf8(CStr::from_ptr(error_name).to_bytes()) .unwrap() .to_string(), ), )); } } Ok(code as usize) } pub fn version() -> i32 { unsafe { LZ4_versionNumber() } } #[test] fn test_version_number() { version(); }