pax_global_header 0000666 0000000 0000000 00000000064 15255167342 0014524 g ustar 00root root 0000000 0000000 52 comment=ef9267838d70fe7ab9b8863f12cfe36d7ca6228f
Blosc-c-blosc2-d3bee7c/ 0000775 0000000 0000000 00000000000 15255167342 0014746 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/.editorconfig 0000664 0000000 0000000 00000000423 15255167342 0017422 0 ustar 00root root 0000000 0000000 ; Top-most EditorConfig file
root = true
; Global settings
[*]
end_of_line = LF
indent_style = space
trim_trailing_whitespace = true
insert_final_newline = true
; C source files
[*.{h,c}]
indent_size = 2
; CMake
[CMakeLists.txt]
indent_size = 4
[*.cmake]
indent_size = 4
Blosc-c-blosc2-d3bee7c/.gitattributes 0000664 0000000 0000000 00000000035 15255167342 0017637 0 ustar 00root root 0000000 0000000 * text=auto
*.md text eol=lf
Blosc-c-blosc2-d3bee7c/.github/ 0000775 0000000 0000000 00000000000 15255167342 0016306 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/.github/FUNDING.yml 0000664 0000000 0000000 00000000026 15255167342 0020121 0 ustar 00root root 0000000 0000000 github: FrancescAlted
Blosc-c-blosc2-d3bee7c/.github/ISSUE_TEMPLATE/ 0000775 0000000 0000000 00000000000 15255167342 0020471 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/.github/ISSUE_TEMPLATE/bug_report.md 0000664 0000000 0000000 00000001062 15255167342 0023162 0 ustar 00root root 0000000 0000000 ---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Provide a minimal program reproducing the error.
**Expected behavior**
A clear and concise description of what you expected to happen.
**Logs**
If applicable, add logs to help explain your problem.
**System information:**
- OS: [e.g. OSX]
- Compiler [e.g. gcc, clang]
- Version [e.g. 2.0.1]
**Additional context**
Add any other context about the problem here.
Blosc-c-blosc2-d3bee7c/.github/dependabot.yml 0000664 0000000 0000000 00000000371 15255167342 0021137 0 ustar 00root root 0000000 0000000 # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
Blosc-c-blosc2-d3bee7c/.github/workflows/ 0000775 0000000 0000000 00000000000 15255167342 0020343 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/.github/workflows/cmake.yml 0000664 0000000 0000000 00000011777 15255167342 0022163 0 ustar 00root root 0000000 0000000 name: CI CMake
on: [push, pull_request]
jobs:
ci-cmake:
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- name: Ubuntu GCC
os: ubuntu-latest
compiler: gcc
- name: Ubuntu GCC No Plugins
os: ubuntu-latest
compiler: gcc
cmake-args: -D BUILD_PLUGINS=OFF
# Out of source build
- name: Ubuntu GCC OSB
os: ubuntu-latest
compiler: gcc
build-dir: ../build
build-src-dir: ../c-blosc2
- name: Ubuntu GCC External LZ4
os: ubuntu-latest
compiler: gcc
packages: liblz4-1 liblz4-dev
cmake-args: -D PREFER_EXTERNAL_LZ4=ON
- name: Ubuntu GCC External ZLIB
os: ubuntu-latest
compiler: gcc
packages: zlib1g-dev
cmake-args: -D PREFER_EXTERNAL_ZLIB=ON
# Not too recent Zstd libs (< 1.4.4) in distros perform pretty bad on compression
# ratios when using dictionaries, making some tests not passing.
# Commenting this out for the time being.
# - name: Ubuntu GCC External ZSTD
# os: ubuntu-latest
# compiler: gcc
# packages: zstd libzstd-dev
# cmake-args: -D PREFER_EXTERNAL_ZSTD=ON
# For some reason, some tests do not pass on ARM SF and HF. Not sure what's going on,
# but having ARM AARCH64 working is good enough for now, so commenting the former out.
# - name: Ubuntu GCC ARM SF
# os: ubuntu-latest
# packages: qemu qemu-user gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libc-dev-armel-cross
# cmake-args: -D CMAKE_TOOLCHAIN_FILE=cmake/toolchain-armsf.cmake
# - name: Ubuntu GCC ARM HF
# os: ubuntu-latest
# packages: qemu qemu-user gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc-dev-armel-cross
# cmake-args: -D CMAKE_TOOLCHAIN_FILE=cmake/toolchain-armhf.cmake
# - name: Ubuntu GCC AARCH64
# os: ubuntu-latest
# packages: qemu-system-aarch64 qemu-user gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc-dev-arm64-cross
# cmake-args: -D CMAKE_TOOLCHAIN_FILE=cmake/toolchain-aarch64.cmake
- name: Ubuntu GCC AARCH64 (native)
os: ubuntu-latest
arch: arm64
compiler: gcc
- name: Ubuntu Clang
os: ubuntu-latest
compiler: clang
- name: Ubuntu Clang No AVX2
os: ubuntu-latest
compiler: clang
cmake-args: -D DEACTIVATE_AVX2=ON
- name: Ubuntu Clang No AVX512
os: ubuntu-latest
compiler: clang
cmake-args: -D DEACTIVATE_AVX512=ON
- name: Ubuntu Clang No ZLIB
os: ubuntu-latest
compiler: clang
cmake-args: -D DEACTIVATE_ZLIB=ON
- name: Ubuntu Clang No ZSTD
os: ubuntu-latest
compiler: clang
cmake-args: -D DEACTIVATE_ZSTD=ON
- name: Windows Clang Win64
os: windows-latest
compiler: clang-cl
cmake-args: -G Ninja
- name: Windows MSVC Win64
os: windows-latest
compiler: cl
cmake-args: -A x64
- name: Windows GCC Ninja
os: windows-latest
compiler: gcc
cmake-args: -G Ninja
- name: Windows GCC MinGW
os: windows-latest
compiler: gcc
cmake-args: -G "MinGW Makefiles"
- name: macOS Clang
os: macOS-latest
compiler: clang
- name: macOS GCC
os: macOS-latest
compiler: gcc
steps:
- uses: actions/checkout@v7
- name: Install packages (Ubuntu)
if: runner.os == 'Linux' && matrix.packages
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.packages }}
- name: Install packages (Windows)
if: runner.os == 'Windows'
run: choco install ninja ${{ matrix.packages }}
- name: Install packages (macOS)
if: runner.os == 'macOS'
run: brew install ninja ${{ matrix.packages }}
- name: Generate project files
run: cmake -S ${{ matrix.build-src-dir || '.' }} -B ${{ matrix.build-dir || '.' }} ${{ matrix.cmake-args }} -D CMAKE_BUILD_TYPE=${{ matrix.build-config || 'Release' }} -D BUILD_SHARED_LIBS=OFF -D CMAKE_C_COMPILER=${{ matrix.compiler }} -D CMAKE_CXX_COMPILER=${{ matrix.compiler }}
env:
CC: ${{ matrix.compiler }}
CFLAGS: ${{ matrix.cflags }}
LDFLAGS: ${{ matrix.ldflags }}
CI: true
- name: Compile source code
run: cmake --build ${{ matrix.build-dir || '.' }} --config ${{ matrix.build-config || 'Release' }} --parallel 4
- name: Run test cases
run: ctest -C Release -j 4 --output-on-failure --max-width 120
working-directory: ${{ matrix.build-dir || '.' }}
Blosc-c-blosc2-d3bee7c/.github/workflows/fuzz.yml 0000664 0000000 0000000 00000001146 15255167342 0022066 0 ustar 00root root 0000000 0000000 name: CI Fuzz
on: [push, pull_request]
jobs:
Fuzzing:
runs-on: ubuntu-latest
steps:
- name: Build Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: 'c-blosc2'
dry-run: false
- name: Run Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
oss-fuzz-project-name: 'c-blosc2'
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@v7
if: failure()
with:
name: artifacts
path: ./out/artifacts
Blosc-c-blosc2-d3bee7c/.gitignore 0000664 0000000 0000000 00000000116 15255167342 0016734 0 ustar 00root root 0000000 0000000 bench/bench
build*
.idea
.*.swp
cmake-build-*
/doc/doxygen/xml/
/doc/xml
.vs/
Blosc-c-blosc2-d3bee7c/.guix-channel 0000664 0000000 0000000 00000000066 15255167342 0017333 0 ustar 00root root 0000000 0000000 (channel
(version 0)
(directory ".guix/modules"))
Blosc-c-blosc2-d3bee7c/.guix/ 0000775 0000000 0000000 00000000000 15255167342 0016000 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/.guix/modules/ 0000775 0000000 0000000 00000000000 15255167342 0017450 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/.guix/modules/c-blosc2-package.scm 0000664 0000000 0000000 00000010263 15255167342 0023153 0 ustar 00root root 0000000 0000000 ;;; This file follows the suggestions in the article "From development
;;; environments to continuous integration—the ultimate guide to software
;;; development with Guix" by Ludovic Courtès at the Guix blog:
;;; .
(define-module (c-blosc2-package)
#:use-module (guix)
#:use-module (guix build-system cmake)
#:use-module (guix git-download)
#:use-module ((guix licenses)
#:prefix license:)
#:use-module (gnu packages compression)
#:use-module (ice-9 regex)
#:use-module (ice-9 textual-ports))
(define (current-source-root)
(dirname (dirname (current-source-directory))))
(define (get-c-blosc2-version)
(let ((version-path (string-append (current-source-root) "/include/blosc2.h"))
(version-rx (make-regexp
"^\\s*#\\s*define\\s*BLOSC2_VERSION_STRING\\s*\"([^\"]*)\".*"
regexp/newline)))
(call-with-input-file version-path
(lambda (port)
(let* ((version-body (get-string-all port))
(version-match (regexp-exec version-rx version-body)))
(and version-match
(match:substring version-match 1)))))))
(define vcs-file?
;; Return true if the given file is under version control.
(or (git-predicate (current-source-root))
(const #t)))
(define-public c-blosc2
(package
(name "c-blosc2")
(version (get-c-blosc2-version))
(source (local-file "../.."
"c-blosc2-checkout"
#:recursive? #t
#:select? (lambda (path stat)
(and (vcs-file? path stat)
(not (string-contains path
"/internal-complibs"))))))
(build-system cmake-build-system)
(arguments
;; Disable AVX2 by default as in Guix' c-blosc package.
`(#:configure-flags '("-DBUILD_STATIC=OFF"
"-DDEACTIVATE_AVX2=ON"
"-DDEACTIVATE_AVX512=ON"
"-DPREFER_EXTERNAL_LZ4=ON"
"-DPREFER_EXTERNAL_ZLIB=ON"
"-DPREFER_EXTERNAL_ZSTD=ON")))
(inputs (list lz4 zlib
;; The only input with a separate libs-only output.
`(,zstd "lib")))
(home-page "https://blosc.org")
(synopsis "Blocking, shuffling and lossless compression library")
(description
"Blosc is a high performance compressor optimized for binary
data (i.e. floating point numbers, integers and booleans, although it can
handle string data too). It has been designed to transmit data to the
processor cache faster than the traditional, non-compressed, direct memory
fetch approach via a @code{memcpy()} system call. Blosc main goal is not just
to reduce the size of large datasets on-disk or in-memory, but also to
accelerate memory-bound computations.
C-Blosc2 is the new major version of C-Blosc, and is backward compatible with
both the C-Blosc1 API and its in-memory format. However, the reverse thing is
generally not true for the format; buffers generated with C-Blosc2 are not
format-compatible with C-Blosc1 (i.e. forward compatibility is not
supported).")
(license license:bsd-3)))
(define (package-with-configure-flags p flags)
"Return P with FLAGS as additional 'configure' flags."
(package/inherit p
(arguments (substitute-keyword-arguments (package-arguments p)
((#:configure-flags original-flags
#~(list))
#~(append #$original-flags
#$flags))))))
(define-public c-blosc2-with-avx2
(package
(inherit (package-with-configure-flags c-blosc2
#~(list "-DDEACTIVATE_AVX2=OFF")))
(name "c-blosc2-with-avx2")))
(define-public c-blosc2-with-avx512
(package
(inherit (package-with-configure-flags c-blosc2
#~(list "-DDEACTIVATE_AVX2=OFF"
"-DDEACTIVATE_AVX512=OFF")))
(name "c-blosc2-with-avx512")))
c-blosc2
Blosc-c-blosc2-d3bee7c/.mailmap 0000664 0000000 0000000 00000000435 15255167342 0016371 0 ustar 00root root 0000000 0000000 Francesc Alted FrancescAlted
Francesc Alted FrancescAlted
Francesc Alted FrancescAlted
Francesc Alted FrancescAlted
Blosc-c-blosc2-d3bee7c/.readthedocs.yaml 0000664 0000000 0000000 00000001217 15255167342 0020176 0 ustar 00root root 0000000 0000000 # .readthedocs.yaml
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
# Required
version: 2
# Set the version of Python and other tools you might need
build:
os: ubuntu-20.04
tools:
python: "mambaforge-4.10"
conda:
environment: doc/environment.yml
# Build documentation in the docs/ directory with Sphinx
sphinx:
configuration: doc/conf.py
# If using Sphinx, optionally build your docs in additional formats such as PDF
# formats:
# - pdf
# Optionally declare the Python requirements required to build your docs
python:
install:
- requirements: doc/requirements.txt
Blosc-c-blosc2-d3bee7c/ANNOUNCE.md 0000664 0000000 0000000 00000004122 15255167342 0016475 0 ustar 00root root 0000000 0000000 # Announcing C-Blosc2 3.3.5
A fast, compressed, and persistent binary data store library for C.
## What is new?
This is a bugfix and maintenance release fixing robustness and security issues
discovered via OSS-Fuzz.
First, an integer overflow in the BloscLZ decompressor's long-match length
accumulation loop was fixed. A crafted input with excessive continuation bytes
could wrap a 32-bit length past `INT32_MAX` into negative values, bypassing bounds
checks and leading to a heap buffer overflow. Length bounds are now verified
prior to addition.
Second, input buffer size validation was added to the ZFP codec decompressors
(`zfp_acc_decompress`, `zfp_prec_decompress`, `zfp_rate_decompress`). Previously,
crafted frames with high metadata values could cause ZFP's bitstream reader to
read beyond buffer bounds. The required bitstream size is now validated against
the input length prior to decompression.
Third, a potential division-by-zero (floating-point exception) in
`frame_get_chunk()` and `frame_get_lazychunk()` when encountering crafted frames
with `chunksize <= 0` during special-value chunk retrieval was guarded.
There are no API or format changes in this release.
For more info, see the release notes in:
https://github.com/Blosc/c-blosc2/blob/main/RELEASE_NOTES.md
## What is it?
Blosc2 is a high-performance data container optimized for binary data.
Blosc2 is the next generation of Blosc, an
[award-winning library](https://www.blosc.org/posts/prize-push-Blosc2)
that has been around for more than a decade.
Blosc2 expands the capabilities of Blosc by providing a higher level
container that is able to store many chunks on it (hence the super-chunk name).
It supports storing data on both memory and disk using the same API.
Also, it adds more compressors and filters.
## Download sources
The github repository is over here:
https://github.com/Blosc/c-blosc2
Blosc is distributed using the BSD license, see LICENSE.txt
for details.
## Mailing list
There is an official Blosc mailing list at:
blosc@googlegroups.com
https://groups.google.com/g/blosc
Enjoy Data!
- The Blosc Development Team
Blosc-c-blosc2-d3bee7c/Blosc2Config.cmake.in 0000664 0000000 0000000 00000013020 15255167342 0020623 0 ustar 00root root 0000000 0000000 # only add PUBLIC dependencies as well
# https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#creating-a-package-configuration-file
include(CMakeFindDependencyMacro)
# Search in _ROOT:
# https://cmake.org/cmake/help/v3.12/policy/CMP0074.html
if(POLICY CMP0074)
cmake_policy(SET CMP0074 NEW)
endif()
# locate the installed FindABC.cmake modules
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/Modules")
# this section stores which configuration options were set
set(HAVE_THREADS @HAVE_THREADS@)
set(HAVE_IPP @HAVE_IPP@)
set(HAVE_LZ4_CONFIG @HAVE_LZ4_CONFIG@)
set(HAVE_ZLIB_NG @HAVE_ZLIB_NG@)
set(HAVE_ZLIB_NG_CONFIG @HAVE_ZLIB_NG_CONFIG@)
set(HAVE_ZSTD_CONFIG @HAVE_ZSTD_CONFIG@)
set(DEACTIVATE_IPP @DEACTIVATE_IPP@)
set(DEACTIVATE_ZLIB @DEACTIVATE_ZLIB@)
set(DEACTIVATE_ZSTD @DEACTIVATE_ZSTD@)
set(PREFER_EXTERNAL_LZ4 @PREFER_EXTERNAL_LZ4@)
set(PREFER_EXTERNAL_ZLIB @PREFER_EXTERNAL_ZLIB@)
set(PREFER_EXTERNAL_ZSTD @PREFER_EXTERNAL_ZSTD@)
set(BLOSC_DEPENDENCY_MODE "@BLOSC_DEPENDENCY_MODE@")
set(HAVE_ZFP @HAVE_ZFP@)
set(BLOSC_ZFP_PROVIDER "@BLOSC_ZFP_PROVIDER@")
set(LZ4_FETCHED @LZ4_FETCHED@)
set(ZLIB_NG_FETCHED @ZLIB_NG_FETCHED@)
set(ZSTD_FETCHED @ZSTD_FETCHED@)
# find dependencies and their targets, which are used in our Blosc2Targets.cmake
# additionally, the Blosc2_..._FOUND variables are used to support
# find_package(Blosc2 ... COMPONENTS ... ...)
# this enables downstream projects to express the need for specific features.
set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1
set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+
if(HAVE_THREADS)
find_dependency(Threads)
set(Blosc2_THREADS_FOUND TRUE)
else()
set(Blosc2_THREADS_FOUND FALSE)
endif()
if(NOT DEACTIVATE_IPP AND HAVE_IPP)
find_dependency(IPP)
set(Blosc2_IPP_FOUND FALSE)
else()
set(Blosc2_IPP_FOUND TRUE)
endif()
if(NOT LZ4_FETCHED)
if(HAVE_LZ4_CONFIG)
find_dependency(lz4 CONFIG)
else()
find_dependency(LZ4 MODULE)
endif()
endif()
set(Blosc2_LZ4_FOUND TRUE)
if(DEACTIVATE_ZLIB)
set(Blosc2_ZLIB_FOUND FALSE)
elseif(ZLIB_NG_FETCHED)
set(Blosc2_ZLIB_FOUND TRUE)
else()
if(HAVE_ZLIB_NG)
if(HAVE_ZLIB_NG_CONFIG)
find_dependency(zlib-ng CONFIG)
else()
find_dependency(ZLIB_NG MODULE)
endif()
else()
find_dependency(ZLIB)
endif()
set(Blosc2_ZLIB_FOUND TRUE)
endif()
if(DEACTIVATE_ZSTD)
set(Blosc2_ZSTD_FOUND FALSE)
elseif(ZSTD_FETCHED)
set(Blosc2_ZSTD_FOUND TRUE)
else()
if(HAVE_ZSTD_CONFIG)
find_dependency(zstd CONFIG)
else()
find_dependency(ZSTD MODULE)
endif()
set(Blosc2_ZSTD_FOUND TRUE)
endif()
if(HAVE_ZFP)
set(Blosc2_ZFP_FOUND TRUE)
if(BLOSC_ZFP_PROVIDER STREQUAL "EXTERNAL")
find_package(zfp CONFIG QUIET)
if(NOT zfp_FOUND)
find_package(ZFP CONFIG QUIET)
endif()
if(NOT zfp_FOUND AND NOT ZFP_FOUND)
find_dependency(ZFP MODULE)
endif()
set(BLOSC_ZFP_TARGET "")
foreach(BLOSC_ZFP_CANDIDATE zfp ZFP::zfp zfp::zfp ZFP::ZFP)
if(TARGET ${BLOSC_ZFP_CANDIDATE} AND NOT BLOSC_ZFP_TARGET)
set(BLOSC_ZFP_TARGET ${BLOSC_ZFP_CANDIDATE})
endif()
endforeach()
if(BLOSC_ZFP_TARGET AND NOT TARGET zfp)
add_library(zfp INTERFACE IMPORTED)
target_link_libraries(zfp INTERFACE ${BLOSC_ZFP_TARGET})
endif()
endif()
else()
set(Blosc2_ZFP_FOUND FALSE)
endif()
# define central Blosc2::blosc2_shared/static targets
include("${CMAKE_CURRENT_LIST_DIR}/Blosc2Targets.cmake")
# check if components are fulfilled and set Blosc2__FOUND vars
# Blosc2_FIND_COMPONENTS is a list set by find_package(... COMPONENTS ... ...)
# likewise Blosc2_FIND_REQUIRED_... per component specified
foreach(comp ${Blosc2_FIND_COMPONENTS})
if(NOT Blosc2_${comp}_FOUND)
if(Blosc2_FIND_REQUIRED_${comp})
set(Blosc2_FOUND FALSE)
endif()
endif()
endforeach()
# Defines imported targets for Blosc2 inside a Python wheel
# ------------------------------
# Shared library target
# ------------------------------
if(NOT TARGET Blosc2::blosc2_shared)
add_library(Blosc2::blosc2_shared SHARED IMPORTED GLOBAL)
if(WIN32)
# MSVC: import library (.lib) + runtime DLL (.dll)
set_target_properties(Blosc2::blosc2_shared PROPERTIES
IMPORTED_IMPLIB "${CMAKE_CURRENT_LIST_DIR}/../blosc2_shared.lib"
IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../blosc2_shared.dll"
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/../../include"
)
else()
# Linux/macOS
set_target_properties(Blosc2::blosc2_shared PROPERTIES
IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../blosc2_shared.so"
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/../../include"
)
endif()
endif()
# ------------------------------
# Static library target
# ------------------------------
if(NOT TARGET Blosc2::blosc2_static)
add_library(Blosc2::blosc2_static STATIC IMPORTED GLOBAL)
if(MSVC)
# Windows static library uses .lib
set_target_properties(Blosc2::blosc2_static PROPERTIES
IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../blosc2_static.lib"
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/../../include"
)
else()
# Linux/macOS static library uses .a
set_target_properties(Blosc2::blosc2_static PROPERTIES
IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../blosc2_static.a"
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/../../include"
)
endif()
endif()
Blosc-c-blosc2-d3bee7c/CMakeLists.txt 0000664 0000000 0000000 00000111114 15255167342 0017505 0 ustar 00root root 0000000 0000000 # Blosc - Blocked Shuffling and Compression Library
#
# Copyright (c) 2021 Blosc Development Team
# https://blosc.org
# License: BSD 3-Clause (see LICENSE.txt)
#
# See LICENSE.txt for details about copyright and rights to use.
# CMake build system for Blosc
# ============================
#
# Available options:
#
# BUILD_STATIC: default ON
# build the static version of the Blosc library
# BUILD_SHARED: default ON
# build the shared library version of the Blosc library
# BUILD_TESTS: default ON
# build test programs and generates the "test" target
# BUILD_FUZZERS: default ON
# build fuzz test programs and generates the "fuzz" target
# BUILD_BENCHMARKS: default ON
# build the benchmark programs
# BUILD_EXAMPLES: default ON
# build the examples programs
# BUILD_PLUGINS: default ON
# build the plugins programs
# BUILD_LITE: default OFF
# build a lite version (only with BloscLZ and LZ4/LZ4HC) of the blosc library
# DEACTIVATE_AVX2: default OFF
# do not attempt to build with AVX2 instructions
# DEACTIVATE_AVX512: default OFF
# do not attempt to build with AVX512 instructions
# DEACTIVATE_ZLIB: default OFF
# do not include support for the Zlib library
# DEACTIVATE_ZSTD: default OFF
# do not include support for the Zstd library
# WITH_ZLIB_OPTIM: default ON
# set WITH_OPTIM when building the fetched zlib-ng library; setting OFF is useful for wasm32 targets
# PREFER_EXTERNAL_LZ4: default OFF
# when found, use the installed LZ4 libs instead of fetching upstream sources
# PREFER_EXTERNAL_ZLIB: default OFF
# when found, use the installed Zlib/Zlib-ng libs instead of fetching upstream sources
# PREFER_EXTERNAL_ZSTD: default OFF
# when found, use the installed Zstd libs instead of fetching upstream sources
# TEST_INCLUDE_BENCH_SHUFFLE_1: default ON
# add a test that runs the benchmark program passing "shuffle" with 1
# thread as second parameter
# TEST_INCLUDE_BENCH_SHUFFLE_N: default ON
# add a test that runs the benchmark program passing "shuffle" with all
# threads as second parameter
# TEST_INCLUDE_BENCH_BITSHUFFLE_1: default ON
# add a test that runs the benchmark program passing "bitshuffle" with 1
# thread as second parameter
# TEST_INCLUDE_BENCH_BITSHUFFLE_N: default ON
# add a test that runs the benchmark program passing "bitshuffle" with
# all threads as second parameter
# TEST_INCLUDE_BENCH_SUITE: default OFF
# add a test that runs the benchmark program passing "suite"
# as first parameter
# TEST_INCLUDE_BENCH_HARDSUITE: default OFF
# add a test that runs the benchmark program passing "hardsuite"
# as first parameter
# TEST_INCLUDE_BENCH_EXTREMESUITE: default OFF
# add a test that runs the benchmark program passing "extremesuite"
# as first parameter
# TEST_INCLUDE_BENCH_DEBUGSUITE: default OFF
# add a test that runs the benchmark program passing "debugsuite"
# as first parameter
#
# Components:
#
# LIB: includes blosc2.so
# DEV: static includes blosc2.a and blosc2.h
if(NOT WIN32)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
endif()
cmake_minimum_required(VERSION 3.16.3)
if(${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION} VERSION_GREATER 3.4)
cmake_policy(SET CMP0063 NEW)
endif()
# The project can have ASM (zstd 1.5.2 is starting to use it)
project(blosc LANGUAGES C ASM)
# parse the full version numbers from blosc2.h
file(READ include/blosc2.h _blosc2_h_contents)
string(REGEX REPLACE ".*#define[ \t]+BLOSC2_VERSION_MAJOR[ \t]+([0-9]+).*"
"\\1" BLOSC2_VERSION_MAJOR ${_blosc2_h_contents})
string(REGEX REPLACE ".*#define[ \t]+BLOSC2_VERSION_MINOR[ \t]+([0-9]+).*"
"\\1" BLOSC2_VERSION_MINOR ${_blosc2_h_contents})
string(REGEX REPLACE ".*#define[ \t]+BLOSC2_VERSION_RELEASE[ \t]+([0-9]+).*"
"\\1" BLOSC2_VERSION_PATCH ${_blosc2_h_contents})
string(REGEX REPLACE ".*#define[ \t]+BLOSC2_VERSION_STRING[ \t]+\"([-0-9A-Za-z.]+)\".*"
"\\1" BLOSC2_VERSION_STRING ${_blosc2_h_contents})
message("Configuring for Blosc version: " ${BLOSC2_VERSION_STRING})
# options
option(BUILD_STATIC
"Build a static version of the blosc library." ON)
option(BUILD_SHARED
"Build a shared library version of the blosc library." ON)
option(BUILD_TESTS
"Build test programs from the blosc compression library" ON)
option(BUILD_FUZZERS
"Build fuzzer programs from the blosc compression library" ${BUILD_STATIC})
option(BUILD_BENCHMARKS
"Build benchmark programs from the blosc compression library" ON)
option(BUILD_EXAMPLES
"Build example programs from the blosc compression library" ON)
option(BUILD_PLUGINS
"Build plugins programs from the blosc compression library" ON)
option(BUILD_LITE
"Build a lite version (only with BloscLZ and LZ4/LZ4HC) of the blosc library." OFF)
option(DEACTIVATE_AVX2
"Do not attempt to build with AVX2 instructions" OFF)
option(DEACTIVATE_AVX512
"Do not attempt to build with AVX512 instructions" OFF)
option(DEACTIVATE_ZLIB
"Do not include support for the Zlib library." OFF)
option(DEACTIVATE_ZSTD
"Do not include support for the Zstd library." OFF)
option(PREFER_EXTERNAL_LZ4
"Find and use external LZ4 library instead of fetching sources." OFF)
option(PREFER_EXTERNAL_ZLIB
"Find and use external ZLIB library instead of fetching sources." OFF)
option(PREFER_EXTERNAL_ZSTD
"Find and use external ZSTD library instead of fetching sources." OFF)
option(WITH_ZLIB_OPTIM
"Set WITH_OPTIM for ZLIB, turning off is useful for compiling for wasm32 targets." ON)
set(BLOSC_DEPENDENCY_MODE "BUNDLED" CACHE STRING
"Dependency resolution mode: BUNDLED uses fetched dependencies, EXTERNAL requires system dependencies, AUTO tries system dependencies before fetching.")
set_property(CACHE BLOSC_DEPENDENCY_MODE PROPERTY STRINGS BUNDLED EXTERNAL AUTO)
set(BLOSC_ENABLE_ZFP "AUTO" CACHE STRING
"ZFP codec support: AUTO enables it when available, ON requires it, OFF disables it.")
set_property(CACHE BLOSC_ENABLE_ZFP PROPERTY STRINGS AUTO ON OFF)
set(BLOSC_LZ4_VERSION "1.10.0" CACHE STRING
"lz4 version to use when fetching internal sources.")
set(BLOSC_LZ4_SOURCE_DIR "" CACHE PATH
"Optional local lz4 checkout to use instead of downloading sources.")
set(BLOSC_ZLIBNG_VERSION "2.3.3" CACHE STRING
"zlib-ng version to use when fetching internal sources.")
set(BLOSC_ZLIBNG_SOURCE_DIR "" CACHE PATH
"Optional local zlib-ng checkout to use instead of downloading sources.")
set(BLOSC_ZSTD_VERSION "1.5.7" CACHE STRING
"zstd version to use when fetching internal sources.")
set(BLOSC_ZSTD_SOURCE_DIR "" CACHE PATH
"Optional local zstd checkout to use instead of downloading sources.")
set(BLOSC_ZFP_VERSION "1.0.1" CACHE STRING
"zfp version to use when fetching plugin sources.")
set(BLOSC_ZFP_SOURCE_DIR "" CACHE PATH
"Optional local zfp checkout to use instead of downloading plugin sources.")
set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
include(GNUInstallDirs)
if(MINGW)
# This gets us DLL resource information when compiling on MinGW.
if(NOT CMAKE_RC_COMPILER)
set(CMAKE_RC_COMPILER windres.exe)
endif()
endif()
if(ENABLE_ASAN)
message(STATUS "Enabling ASAN")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Os -fno-omit-frame-pointer -fsanitize=address")
endif()
if(BUILD_PLUGINS AND BUILD_LITE)
message(FATAL_ERROR "Both plugins and lite Blosc versions can not be built at the same time.")
endif()
if(BUILD_LITE)
set(DEACTIVATE_ZLIB ON)
set(DEACTIVATE_ZSTD ON)
endif()
string(TOUPPER "${BLOSC_DEPENDENCY_MODE}" BLOSC_DEPENDENCY_MODE)
set(BLOSC_DEPENDENCY_MODE "${BLOSC_DEPENDENCY_MODE}" CACHE STRING
"Dependency resolution mode: BUNDLED uses fetched dependencies, EXTERNAL requires system dependencies, AUTO tries system dependencies before fetching." FORCE)
if(NOT BLOSC_DEPENDENCY_MODE STREQUAL "BUNDLED" AND
NOT BLOSC_DEPENDENCY_MODE STREQUAL "EXTERNAL" AND
NOT BLOSC_DEPENDENCY_MODE STREQUAL "AUTO")
message(FATAL_ERROR "BLOSC_DEPENDENCY_MODE must be one of BUNDLED, EXTERNAL, or AUTO")
endif()
string(TOUPPER "${BLOSC_ENABLE_ZFP}" BLOSC_ENABLE_ZFP)
set(BLOSC_ENABLE_ZFP "${BLOSC_ENABLE_ZFP}" CACHE STRING
"ZFP codec support: AUTO enables it when available, ON requires it, OFF disables it." FORCE)
if(NOT BLOSC_ENABLE_ZFP STREQUAL "AUTO" AND
NOT BLOSC_ENABLE_ZFP STREQUAL "ON" AND
NOT BLOSC_ENABLE_ZFP STREQUAL "OFF")
message(FATAL_ERROR "BLOSC_ENABLE_ZFP must be one of AUTO, ON, or OFF")
endif()
# Threads
set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1
set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+
if(WIN32)
# try to use the system library
find_package(Threads)
else()
find_package(Threads REQUIRED)
endif()
if(Threads_FOUND)
set(HAVE_THREADS ON)
else()
set(HAVE_THREADS OFF)
endif()
set(BLOSC_FIND_EXTERNAL_LZ4 FALSE)
set(BLOSC_REQUIRE_EXTERNAL_LZ4 FALSE)
if(BLOSC_DEPENDENCY_MODE STREQUAL "EXTERNAL")
set(BLOSC_FIND_EXTERNAL_LZ4 TRUE)
set(BLOSC_REQUIRE_EXTERNAL_LZ4 TRUE)
elseif(BLOSC_DEPENDENCY_MODE STREQUAL "AUTO" OR PREFER_EXTERNAL_LZ4)
set(BLOSC_FIND_EXTERNAL_LZ4 TRUE)
endif()
if(BLOSC_FIND_EXTERNAL_LZ4)
find_package(LZ4 CONFIG NAMES lz4)
if(LZ4_FOUND)
set(HAVE_LZ4_CONFIG TRUE)
else()
find_package(LZ4 MODULE)
endif()
endif()
if(NOT LZ4_FOUND AND BLOSC_REQUIRE_EXTERNAL_LZ4)
message(FATAL_ERROR "BLOSC_DEPENDENCY_MODE=EXTERNAL requires an external LZ4 package. Install LZ4 or use BLOSC_DEPENDENCY_MODE=BUNDLED/AUTO.")
endif()
if(NOT LZ4_FOUND)
include(FetchContent)
set(LZ4_BUILD_CLI OFF CACHE BOOL "" FORCE)
set(LZ4_BUILD_LEGACY_LZ4C OFF CACHE BOOL "" FORCE)
set(LZ4_BUNDLED_MODE ON CACHE BOOL "" FORCE)
set(LZ4_POSITION_INDEPENDENT_LIB ON CACHE BOOL "" FORCE)
if(BLOSC_LZ4_SOURCE_DIR)
message(STATUS "Using lz4 via FetchContent from local checkout: ${BLOSC_LZ4_SOURCE_DIR}")
FetchContent_Declare(
lz4
SOURCE_DIR "${BLOSC_LZ4_SOURCE_DIR}"
SOURCE_SUBDIR build/cmake)
else()
message(STATUS "Using lz4 ${BLOSC_LZ4_VERSION} via FetchContent download")
FetchContent_Declare(
lz4
URL "https://github.com/lz4/lz4/archive/refs/tags/v${BLOSC_LZ4_VERSION}.tar.gz"
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
SOURCE_SUBDIR build/cmake)
endif()
FetchContent_GetProperties(lz4)
if(NOT lz4_POPULATED)
if(POLICY CMP0169)
cmake_policy(PUSH)
cmake_policy(SET CMP0169 OLD)
endif()
FetchContent_Populate(lz4)
if(POLICY CMP0169)
cmake_policy(POP)
endif()
add_subdirectory("${lz4_SOURCE_DIR}/build/cmake" "${lz4_BINARY_DIR}" EXCLUDE_FROM_ALL)
endif()
set(LZ4_FETCHED TRUE)
set(LZ4_FOUND TRUE)
set(LZ4_SOURCE_DIR "${lz4_SOURCE_DIR}")
set(LZ4_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}/blosc2/thirdparty/lz4")
if(TARGET lz4_static)
set_target_properties(lz4_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
set_property(TARGET lz4_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES
"$"
"$")
endif()
endif()
if(NOT DEACTIVATE_ZLIB)
set(BLOSC_FIND_EXTERNAL_ZLIB FALSE)
set(BLOSC_REQUIRE_EXTERNAL_ZLIB FALSE)
if(BLOSC_DEPENDENCY_MODE STREQUAL "EXTERNAL")
set(BLOSC_FIND_EXTERNAL_ZLIB TRUE)
set(BLOSC_REQUIRE_EXTERNAL_ZLIB TRUE)
elseif(BLOSC_DEPENDENCY_MODE STREQUAL "AUTO" OR PREFER_EXTERNAL_ZLIB)
set(BLOSC_FIND_EXTERNAL_ZLIB TRUE)
endif()
if(BLOSC_FIND_EXTERNAL_ZLIB)
find_package(ZLIB_NG CONFIG NAMES zlib-ng)
if(ZLIB_NG_FOUND)
set(HAVE_ZLIB_NG TRUE)
set(HAVE_ZLIB_NG_CONFIG TRUE)
else()
find_package(ZLIB_NG MODULE)
if(ZLIB_NG_FOUND)
set(HAVE_ZLIB_NG TRUE)
else()
find_package(ZLIB)
endif()
endif()
endif()
if(NOT (ZLIB_NG_FOUND OR ZLIB_FOUND) AND BLOSC_REQUIRE_EXTERNAL_ZLIB)
message(FATAL_ERROR "BLOSC_DEPENDENCY_MODE=EXTERNAL requires an external zlib-ng or zlib package. Install one or use BLOSC_DEPENDENCY_MODE=BUNDLED/AUTO.")
endif()
if(NOT (ZLIB_NG_FOUND OR ZLIB_FOUND))
include(FetchContent)
if (NOT WITH_ZLIB_OPTIM)
set(WITH_OPTIM OFF CACHE BOOL "Disable zlib-ng architecture optimizations when fetched via FetchContent" FORCE)
set(WITH_RUNTIME_CPU_DETECTION OFF CACHE BOOL "Disable zlib-ng runtime CPU detection when fetched via FetchContent" FORCE)
endif()
set(SKIP_INSTALL_ALL ON CACHE BOOL "Skip zlib-ng install targets when fetched via FetchContent" FORCE)
set(BUILD_TESTING OFF CACHE BOOL "Disable zlib-ng tests when fetched via FetchContent" FORCE)
set(WITH_GTEST OFF CACHE BOOL "Disable zlib-ng GTest when fetched via FetchContent" FORCE)
set(WITH_FUZZERS OFF CACHE BOOL "Disable zlib-ng fuzzers when fetched via FetchContent" FORCE)
set(WITH_BENCHMARKS OFF CACHE BOOL "Disable zlib-ng benchmarks when fetched via FetchContent" FORCE)
set(WITH_BENCHMARK_APPS OFF CACHE BOOL "Disable zlib-ng benchmark apps when fetched via FetchContent" FORCE)
if(BLOSC_ZLIBNG_SOURCE_DIR)
message(STATUS "Using zlib-ng via FetchContent from local checkout: ${BLOSC_ZLIBNG_SOURCE_DIR}")
FetchContent_Declare(
zlib_ng
SOURCE_DIR "${BLOSC_ZLIBNG_SOURCE_DIR}")
else()
message(STATUS "Using zlib-ng ${BLOSC_ZLIBNG_VERSION} via FetchContent download")
FetchContent_Declare(
zlib_ng
URL "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/${BLOSC_ZLIBNG_VERSION}.tar.gz"
DOWNLOAD_EXTRACT_TIMESTAMP TRUE)
endif()
FetchContent_MakeAvailable(zlib_ng)
set(HAVE_ZLIB_NG TRUE)
set(ZLIB_NG_FETCHED TRUE)
set(ZLIB_NG_FOUND TRUE)
set(ZLIB_NG_BUILD_DIR "${zlib_ng_BINARY_DIR}")
set(ZLIB_NG_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}/blosc2/thirdparty/zlib-ng")
set(ZLIB_NG_INSTALL_TARGET "")
if(TARGET zlib-ng-static)
get_target_property(ZLIB_NG_INSTALL_TARGET zlib-ng-static ALIASED_TARGET)
if(NOT ZLIB_NG_INSTALL_TARGET)
set(ZLIB_NG_INSTALL_TARGET zlib-ng-static)
endif()
elseif(TARGET zlib-ng)
get_target_property(ZLIB_NG_INSTALL_TARGET zlib-ng ALIASED_TARGET)
if(NOT ZLIB_NG_INSTALL_TARGET)
set(ZLIB_NG_INSTALL_TARGET zlib-ng)
endif()
endif()
if(ZLIB_NG_INSTALL_TARGET)
set_target_properties(${ZLIB_NG_INSTALL_TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
set_property(TARGET ${ZLIB_NG_INSTALL_TARGET} PROPERTY INTERFACE_INCLUDE_DIRECTORIES
"$"
"$"
"$")
endif()
endif()
set(HAVE_ZLIB TRUE)
endif()
if(NOT DEACTIVATE_ZSTD)
set(BLOSC_FIND_EXTERNAL_ZSTD FALSE)
set(BLOSC_REQUIRE_EXTERNAL_ZSTD FALSE)
if(BLOSC_DEPENDENCY_MODE STREQUAL "EXTERNAL")
set(BLOSC_FIND_EXTERNAL_ZSTD TRUE)
set(BLOSC_REQUIRE_EXTERNAL_ZSTD TRUE)
elseif(BLOSC_DEPENDENCY_MODE STREQUAL "AUTO" OR PREFER_EXTERNAL_ZSTD)
set(BLOSC_FIND_EXTERNAL_ZSTD TRUE)
endif()
if(BLOSC_FIND_EXTERNAL_ZSTD)
find_package(ZSTD CONFIG NAMES zstd)
if(ZSTD_FOUND)
set(HAVE_ZSTD_CONFIG TRUE)
else()
find_package(ZSTD MODULE)
endif()
endif()
if(NOT ZSTD_FOUND AND BLOSC_REQUIRE_EXTERNAL_ZSTD)
message(FATAL_ERROR "BLOSC_DEPENDENCY_MODE=EXTERNAL requires an external Zstd package. Install Zstd or use BLOSC_DEPENDENCY_MODE=BUNDLED/AUTO.")
endif()
if(NOT ZSTD_FOUND)
include(FetchContent)
set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "Disable zstd programs when fetched via FetchContent" FORCE)
set(ZSTD_BUILD_SHARED OFF CACHE BOOL "Disable shared zstd library when fetched via FetchContent" FORCE)
set(ZSTD_BUILD_STATIC ON CACHE BOOL "Enable static zstd library when fetched via FetchContent" FORCE)
set(ZSTD_BUILD_TESTS OFF CACHE BOOL "Disable zstd tests when fetched via FetchContent" FORCE)
set(ZSTD_BUILD_CONTRIB OFF CACHE BOOL "Disable zstd contrib targets when fetched via FetchContent" FORCE)
set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "Disable zstd multithread support when fetched via FetchContent" FORCE)
if(BLOSC_ZSTD_SOURCE_DIR)
message(STATUS "Using zstd via FetchContent from local checkout: ${BLOSC_ZSTD_SOURCE_DIR}")
FetchContent_Declare(
zstd
SOURCE_DIR "${BLOSC_ZSTD_SOURCE_DIR}"
SOURCE_SUBDIR build/cmake)
else()
message(STATUS "Using zstd ${BLOSC_ZSTD_VERSION} via FetchContent download")
FetchContent_Declare(
zstd
URL "https://github.com/facebook/zstd/archive/refs/tags/v${BLOSC_ZSTD_VERSION}.tar.gz"
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
SOURCE_SUBDIR build/cmake)
endif()
FetchContent_GetProperties(zstd)
if(NOT zstd_POPULATED)
if(POLICY CMP0169)
cmake_policy(PUSH)
cmake_policy(SET CMP0169 OLD)
endif()
FetchContent_Populate(zstd)
if(POLICY CMP0169)
cmake_policy(POP)
endif()
add_subdirectory("${zstd_SOURCE_DIR}/build/cmake" "${zstd_BINARY_DIR}" EXCLUDE_FROM_ALL)
endif()
set(HAVE_ZSTD TRUE)
set(ZSTD_FETCHED TRUE)
set(ZSTD_FOUND TRUE)
set(ZSTD_BUILD_DIR "${zstd_BINARY_DIR}")
set(ZSTD_SOURCE_DIR "${zstd_SOURCE_DIR}")
set(ZSTD_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}/blosc2/thirdparty/zstd")
if(TARGET libzstd_static)
set_target_properties(libzstd_static PROPERTIES POSITION_INDEPENDENT_CODE ON)
set_property(TARGET libzstd_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES
"$"
"$")
# Silence GCC false positives in fetched legacy zstd sources on some Linux/amd64 builds.
target_compile_options(libzstd_static PRIVATE
$<$:-Wno-maybe-uninitialized>)
endif()
else()
set(HAVE_ZSTD TRUE)
endif()
endif()
set(HAVE_ZFP FALSE)
set(BLOSC_ZFP_PROVIDER "NONE")
set(BLOSC_ZFP_TARGET "")
set(ZFP_FOUND FALSE)
if(BUILD_PLUGINS AND NOT BLOSC_ENABLE_ZFP STREQUAL "OFF")
set(BLOSC_FIND_EXTERNAL_ZFP FALSE)
set(BLOSC_ALLOW_BUNDLED_ZFP FALSE)
if(BLOSC_DEPENDENCY_MODE STREQUAL "EXTERNAL")
set(BLOSC_FIND_EXTERNAL_ZFP TRUE)
elseif(BLOSC_DEPENDENCY_MODE STREQUAL "AUTO")
set(BLOSC_FIND_EXTERNAL_ZFP TRUE)
set(BLOSC_ALLOW_BUNDLED_ZFP TRUE)
else()
set(BLOSC_ALLOW_BUNDLED_ZFP TRUE)
endif()
if(BLOSC_ENABLE_ZFP STREQUAL "ON" AND NOT BLOSC_DEPENDENCY_MODE STREQUAL "EXTERNAL")
set(BLOSC_ALLOW_BUNDLED_ZFP TRUE)
endif()
if(BLOSC_FIND_EXTERNAL_ZFP)
find_package(zfp CONFIG QUIET)
if(NOT zfp_FOUND)
find_package(ZFP CONFIG QUIET)
endif()
if(NOT zfp_FOUND AND NOT ZFP_FOUND)
find_package(ZFP MODULE QUIET)
endif()
if(zfp_FOUND OR ZFP_FOUND)
set(ZFP_FOUND TRUE)
set(BLOSC_ZFP_PROVIDER "EXTERNAL")
foreach(BLOSC_ZFP_CANDIDATE zfp ZFP::zfp zfp::zfp ZFP::ZFP)
if(TARGET ${BLOSC_ZFP_CANDIDATE} AND NOT BLOSC_ZFP_TARGET)
set(BLOSC_ZFP_TARGET ${BLOSC_ZFP_CANDIDATE})
endif()
endforeach()
if(BLOSC_ZFP_TARGET AND NOT TARGET zfp)
add_library(zfp INTERFACE IMPORTED)
target_link_libraries(zfp INTERFACE ${BLOSC_ZFP_TARGET})
endif()
endif()
endif()
if(NOT ZFP_FOUND AND BLOSC_ALLOW_BUNDLED_ZFP)
set(BLOSC_ZFP_PROVIDER "BUNDLED")
endif()
if(BLOSC_ZFP_PROVIDER STREQUAL "NONE")
if(BLOSC_ENABLE_ZFP STREQUAL "ON")
message(FATAL_ERROR "BLOSC_ENABLE_ZFP=ON requires ZFP support, but no external ZFP package was found and bundled ZFP is disabled by BLOSC_DEPENDENCY_MODE=EXTERNAL.")
else()
message(STATUS "ZFP codec support disabled: no external ZFP package found in BLOSC_DEPENDENCY_MODE=EXTERNAL")
endif()
else()
set(HAVE_ZFP TRUE)
message(STATUS "ZFP codec support enabled (${BLOSC_ZFP_PROVIDER})")
endif()
endif()
if(BUILD_PLUGINS)
set(HAVE_PLUGINS TRUE)
endif()
# create the config.h file
configure_file("${PROJECT_SOURCE_DIR}/blosc/config.h.in"
"${PROJECT_BINARY_DIR}/blosc/config.h")
# now make sure that you set the build directory on your "Include" path when compiling
include_directories("${PROJECT_BINARY_DIR}/blosc/")
# If the build type is not set, default to Release.
set(BLOSC_DEFAULT_BUILD_TYPE Release)
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "No build type specified. Defaulting to '${BLOSC_DEFAULT_BUILD_TYPE}'.")
set(CMAKE_BUILD_TYPE ${BLOSC_DEFAULT_BUILD_TYPE} CACHE STRING
"Choose the type of build." FORCE)
# Set the possible values of build type for cmake-gui
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
"Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()
# Propagate CMAKE_OSX_ARCHITECTURES env variable into CMAKE_SYSTEM_PROCESSOR
if(DEFINED ENV{CMAKE_OSX_ARCHITECTURES})
if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")
set(CMAKE_SYSTEM_PROCESSOR arm64)
endif()
endif()
# Based on the target system's processor and the compiler being used,
# set build variables indicating which hardware features can be targeted
# by the compiler. Note we DO NOT check which hardware features are supported
# by this (the host) system, because we want to be able to support compiling
# for newer hardware on older machines as well as cross-compilation.
message(STATUS "Building for system processor ${CMAKE_SYSTEM_PROCESSOR}")
message(STATUS "Building for compiler ID ${CMAKE_C_COMPILER_ID}")
if(CMAKE_SYSTEM_PROCESSOR MATCHES i386|i686|x86_64|amd64|AMD64)
if(CMAKE_C_COMPILER_ID MATCHES GNU)
# We need C99 (GNU99 more exactly)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
set(COMPILER_SUPPORT_SSE2 TRUE)
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.7)
set(COMPILER_SUPPORT_AVX2 TRUE)
else()
set(COMPILER_SUPPORT_AVX2 FALSE)
endif()
# GCC 10.3.2 (the version in manylinux_2014) seems to have issues supporting dynamic dispatching
# of AVX512. GCC 11.4 is the first minimal version that works well here.
# That means that Linux wheels will have AVX512 disabled, but that's life.
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 11.4)
set(COMPILER_SUPPORT_AVX512 TRUE)
else()
set(COMPILER_SUPPORT_AVX512 FALSE)
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES Clang|AppleClang)
set(COMPILER_SUPPORT_SSE2 TRUE)
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 3.2)
set(COMPILER_SUPPORT_AVX2 TRUE)
else()
set(COMPILER_SUPPORT_AVX2 FALSE)
endif()
# Clang 13 is the minimum version that we know that works with AVX512 dynamic dispatch.
# Perhaps lesser versions work too, better to err on the safe side.
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
set(COMPILER_SUPPORT_AVX512 TRUE)
else()
set(COMPILER_SUPPORT_AVX512 FALSE)
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES Intel|IntelLLVM)
# All Intel compilers since the introduction of AVX512 in 2016 should support it, so activate all SIMD flavors
set(COMPILER_SUPPORT_SSE2 TRUE)
set(COMPILER_SUPPORT_AVX2 TRUE)
set(COMPILER_SUPPORT_AVX512 TRUE)
elseif(MSVC)
set(COMPILER_SUPPORT_SSE2 TRUE)
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 18.00.30501)
set(COMPILER_SUPPORT_AVX2 TRUE)
else()
set(COMPILER_SUPPORT_AVX2 FALSE)
endif()
# AVX512 starts to be supported since Visual Studio 17 15.0
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 19.10.25017)
set(COMPILER_SUPPORT_AVX512 TRUE)
else()
set(COMPILER_SUPPORT_AVX512 FALSE)
endif()
else()
set(COMPILER_SUPPORT_SSE2 FALSE)
set(COMPILER_SUPPORT_AVX2 FALSE)
set(COMPILER_SUPPORT_AVX512 FALSE)
# Unrecognized compiler. Emit a warning message to let the user know hardware-acceleration won't be available.
message(WARNING "Unable to determine which ${CMAKE_SYSTEM_PROCESSOR} hardware features are supported by the C compiler (${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}).")
endif()
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES armv7l|aarch64|arm64)
if(CMAKE_C_COMPILER_ID MATCHES GNU)
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 5.2)
set(COMPILER_SUPPORT_NEON TRUE)
else()
set(COMPILER_SUPPORT_NEON FALSE)
endif()
elseif(CMAKE_C_COMPILER_ID MATCHES Clang|AppleClang)
if(CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 3.3)
set(COMPILER_SUPPORT_NEON TRUE)
else()
set(COMPILER_SUPPORT_NEON FALSE)
endif()
else()
set(COMPILER_SUPPORT_NEON FALSE)
# Unrecognized compiler. Emit a warning message to let the user know hardware-acceleration won't be available.
message(WARNING "Unable to determine which ${CMAKE_SYSTEM_PROCESSOR} hardware features are supported by the C compiler (${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}).")
endif()
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv32|riscv64|riscv)")
set(COMPILER_SUPPORT_SSE2 FALSE)
set(COMPILER_SUPPORT_AVX2 FALSE)
set(COMPILER_SUPPORT_AVX512 FALSE)
set(COMPILER_SUPPORT_NEON FALSE)
set(COMPILER_SUPPORT_ALTIVEC FALSE)
message(STATUS "Detected RISC-V target; hardware-accelerated shuffle implementations are not available, so the generic path will be used.")
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)")
if(CMAKE_C_COMPILER_ID MATCHES GNU AND CMAKE_C_COMPILER_VERSION VERSION_GREATER 8)
set(COMPILER_SUPPORT_ALTIVEC TRUE)
elseif(CMAKE_C_COMPILER_ID MATCHES Clang AND CMAKE_C_COMPILER_VERSION VERSION_GREATER 13)
set(COMPILER_SUPPORT_ALTIVEC TRUE)
else()
set(COMPILER_SUPPORT_ALTIVEC FALSE)
endif()
else()
# If the target system processor isn't recognized, emit a warning message to alert the user
# that hardware-acceleration support won't be available but allow configuration to proceed.
message(WARNING "Unrecognized system processor ${CMAKE_SYSTEM_PROCESSOR}. Cannot determine which hardware features (${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}) supports, so hardware-accelerated implementations will not be available.")
endif()
# disable AVX2 if specified
if(DEACTIVATE_AVX2)
set(COMPILER_SUPPORT_AVX2 FALSE)
# AVX512 functions in bitshuffle depend on AVX2 too
set(COMPILER_SUPPORT_AVX512 FALSE)
endif()
# disable AVX512 if specified
if(DEACTIVATE_AVX512)
set(COMPILER_SUPPORT_AVX512 FALSE)
endif()
# flags
# @TODO: set -Wall
# @NOTE: -O3 is enabled in Release mode (CMAKE_BUILD_TYPE="Release")
# Set the "-msse2" build flag only if the CMAKE_C_FLAGS is not already set.
# Probably "-msse2" should be appended to CMAKE_C_FLAGS_RELEASE.
if(CMAKE_C_COMPILER_ID MATCHES GNU|Clang|Intel|IntelLLVM)
if(NOT CMAKE_C_FLAGS AND COMPILER_SUPPORT_SSE2)
set(CMAKE_C_FLAGS -msse2 CACHE STRING "C flags." FORCE)
endif()
endif()
if(CMAKE_C_COMPILER_ID MATCHES Intel|IntelLLVM|Clang OR HAIKU)
# We need to tell Intel and Clang compilers about what level of POSIX they support.
# Note this cannot be dropped on the BSDs even though it hides their BSD extensions
# (see the _XOPEN_SOURCE handling at the top of frame.c): my_malloc() in blosc2.c
# keys the posix_memalign() path off it, and there is nothing else defining
# _POSIX_C_SOURCE there, so dropping it would silently fall back to plain malloc().
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_XOPEN_SOURCE=600")
endif()
if(MSVC)
if(NOT CMAKE_C_FLAGS)
set(CMAKE_C_FLAGS "/Ox" CACHE STRING "C flags." FORCE)
endif()
# Turn off misguided "secure CRT" warnings in MSVC.
# Microsoft wants people to use the MS-specific _s
# versions of certain C functions but this is difficult to do
# in platform-independent code.
add_definitions(-D_CRT_SECURE_NO_WARNINGS)
else()
add_compile_options(-Wall -Wextra)
endif()
if(WIN32)
# For some supporting headers
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/blosc")
endif()
if(NOT DEFINED BLOSC_IS_SUBPROJECT)
if("^${CMAKE_SOURCE_DIR}$" STREQUAL "^${PROJECT_SOURCE_DIR}$")
set(BLOSC_IS_SUBPROJECT FALSE)
else()
set(BLOSC_IS_SUBPROJECT TRUE)
message(STATUS "Detected that BLOSC is used a subproject.")
endif()
endif()
if(NOT DEFINED BLOSC_INSTALL)
if(BLOSC_IS_SUBPROJECT)
set(BLOSC_INSTALL FALSE)
else()
set(BLOSC_INSTALL TRUE)
endif()
endif()
# include directories
include_directories(include)
# subdirectories
set(SOURCES)
if(BUILD_PLUGINS)
enable_testing()
add_subdirectory(plugins)
if(DEFINED ZFP_SOURCE_DIR)
include_directories(${ZFP_SOURCE_DIR}/include)
endif()
endif()
# Determine which linkage the unified target will use
if(BUILD_SHARED AND NOT BUILD_STATIC)
if(NOT BUILD_SHARED_LIBS)
message(WARNING "BUILD_SHARED_LIBS is OFF, but BUILD_SHARED is ON and BUILD_STATIC is OFF, which takes precedence, so Blosc2::blosc2 is a shared library")
endif()
set(UNIFIED_TARGET_ALIAS blosc2_shared)
elseif(NOT BUILD_SHARED AND BUILD_STATIC)
if(BUILD_SHARED_LIBS)
message(WARNING "BUILD_SHARED_LIBS is ON, but BUILD_SHARED is OFF and BUILD_STATIC is ON, which takes precedence, so Blosc2::blosc2 is a static library")
endif()
set(UNIFIED_TARGET_ALIAS blosc2_static)
else()
# If both BUILD_SHARED and BUILD_STATIC are set, which is the default,
# fallback to using BUILD_SHARED_LIBS to determine whether to set
# Blosc2::blosc2 to static or shared.
if(BUILD_SHARED_LIBS)
set(UNIFIED_TARGET_ALIAS blosc2_shared)
else()
set(UNIFIED_TARGET_ALIAS blosc2_static)
endif()
endif()
add_subdirectory(blosc)
if(BUILD_TESTS)
enable_testing()
add_subdirectory(tests)
add_subdirectory(compat)
endif()
if(BUILD_FUZZERS)
if(NOT BUILD_STATIC)
message(FATAL_ERROR "BUILD_FUZZERS requires BUILD_STATIC to be enabled.")
endif()
enable_testing()
add_subdirectory(tests/fuzz)
endif()
if(BUILD_BENCHMARKS)
add_subdirectory(bench)
endif()
if(BUILD_EXAMPLES)
add_subdirectory(examples)
endif()
# collecting SOURCES is now complete
if(BUILD_SHARED)
target_sources(blosc2_shared PRIVATE ${SOURCES})
endif()
if(BUILD_STATIC)
target_sources(blosc2_static PRIVATE ${SOURCES})
endif()
if(BUILD_TESTS)
target_sources(blosc_testing PRIVATE ${SOURCES})
endif()
# install targets
if(BLOSC_INSTALL)
include(GNUInstallDirs)
# C++ files
install(FILES ${PROJECT_SOURCE_DIR}/include/blosc2.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV)
install(FILES ${PROJECT_SOURCE_DIR}/include/b2nd.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT DEV)
install(FILES
${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-export.h
${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-common.h
${PROJECT_SOURCE_DIR}/include/blosc2/blosc2-stdio.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV)
if(BUILD_PLUGINS)
install(FILES
${PROJECT_SOURCE_DIR}/include/blosc2/filters-registry.h
${PROJECT_SOURCE_DIR}/include/blosc2/codecs-registry.h
${PROJECT_SOURCE_DIR}/include/blosc2/tuners-registry.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/blosc2 COMPONENT DEV)
endif()
if(BUILD_SHARED)
install(TARGETS blosc2_shared
LIBRARY COMPONENT LIB
ARCHIVE COMPONENT DEV
RUNTIME COMPONENT LIB)
endif()
if(BUILD_STATIC)
install(TARGETS blosc2_static COMPONENT DEV)
endif()
# Bundled dependency headers are private implementation details and are not
# installed. Bundled dependency objects are folded into libblosc2.a for
# static builds and into libblosc2 shared libraries by the regular linker.
# config files
include(CMakePackageConfigHelpers)
# we need a general location for Unix and Windows to install our
# Blosc2Config.cmake files to. This is defined in CMake:
# https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure
if(NOT Blosc2_INSTALL_CMAKEDIR)
if(WIN32 AND NOT MINGW)
set(Blosc2_INSTALL_CMAKEDIR "cmake")
else()
set(Blosc2_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/Blosc2")
endif()
endif()
# CMake config file
# This stores our targets and find and populates the targets we depend on,
# including third party interface libraries that we added.
set(Blosc2_INSTALL_TARGET_NAMES)
set(Blosc2_INSTALL_THIRDPARTY_TARGET_NAMES)
if(BUILD_SHARED)
list(APPEND Blosc2_INSTALL_TARGET_NAMES blosc2_shared)
endif()
if(BUILD_STATIC)
list(APPEND Blosc2_INSTALL_TARGET_NAMES blosc2_static)
endif()
# Add unified target
add_library(blosc2 INTERFACE)
list(APPEND Blosc2_INSTALL_TARGET_NAMES blosc2)
target_link_libraries(blosc2 INTERFACE ${UNIFIED_TARGET_ALIAS})
configure_file(
${PROJECT_SOURCE_DIR}/Blosc2Config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/Blosc2Config.cmake
@ONLY
)
install(TARGETS ${Blosc2_INSTALL_TARGET_NAMES}
EXPORT Blosc2Targets
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
)
if(Blosc2_INSTALL_THIRDPARTY_TARGET_NAMES)
install(TARGETS ${Blosc2_INSTALL_THIRDPARTY_TARGET_NAMES}
EXPORT Blosc2Targets
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/blosc2
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}/blosc2
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)
endif()
install(EXPORT Blosc2Targets
FILE Blosc2Targets.cmake
NAMESPACE Blosc2::
DESTINATION ${Blosc2_INSTALL_CMAKEDIR}
)
write_basic_package_version_file("Blosc2ConfigVersion.cmake"
VERSION ${BLOSC2_VERSION_STRING}
COMPATIBILITY SameMajorVersion
)
install(
FILES
${CMAKE_CURRENT_BINARY_DIR}/Blosc2Config.cmake
${CMAKE_CURRENT_BINARY_DIR}/Blosc2ConfigVersion.cmake
DESTINATION ${Blosc2_INSTALL_CMAKEDIR}
)
# CMake Find*.cmake files used in Blosc2Config.cmake
install(
DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake/
DESTINATION ${Blosc2_INSTALL_CMAKEDIR}/Modules
)
# pkg-config .pc file
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/blosc2.pc.in"
"${CMAKE_CURRENT_BINARY_DIR}/blosc2.pc"
@ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/blosc2.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" COMPONENT DEV)
# uninstaller
if(NOT TARGET uninstall)
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
@ONLY)
add_custom_target(uninstall
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
endif()
endif()
# packaging
if(NOT BLOSC_IS_SUBPROJECT)
include(InstallRequiredSystemLibraries)
set(CPACK_GENERATOR TGZ ZIP)
set(CPACK_SOURCE_GENERATOR TGZ ZIP)
set(CPACK_PACKAGE_VERSION_MAJOR ${BLOSC2_VERSION_MAJOR})
set(CPACK_PACKAGE_VERSION_MINOR ${BLOSC2_VERSION_MINOR})
set(CPACK_PACKAGE_VERSION_PATCH ${BLOSC2_VERSION_PATCH})
set(CPACK_PACKAGE_VERSION ${BLOSC_STRING_VERSION})
set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.rst")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
"A blocking, shuffling and lossless compression library")
set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt")
set(CPACK_SOURCE_IGNORE_FILES "/build.*;.*~;\\\\.git.*;\\\\.DS_Store")
set(CPACK_STRIP_FILES TRUE)
set(CPACK_SOURCE_STRIP_FILES TRUE)
include(CPack)
endif()
Blosc-c-blosc2-d3bee7c/COMPILING_WITH_WHEELS.rst 0000664 0000000 0000000 00000011561 15255167342 0020727 0 ustar 00root root 0000000 0000000 =============================================================================
C-Blosc2 libraries come with Python-Blosc2 wheels
=============================================================================
C-Blosc2 binary libraries (including plugins functionality) can easily be installed from Python-Blosc2 wheels:
.. code-block:: console
$ pip install blosc2
As a result, one can easily update to the latest version of C-Blosc2 binaries without the need to manually compile it. Following are instructions on how to use the libraries in wheels for different platforms.
Compiling C files with Blosc2 wheels on Windows
-----------------------------------------------
- The wheels for Windows have been produced with the Microsoft MSVC compiler, so we recommend that you use it too. You can get it for free at: https://visualstudio.microsoft.com/es/downloads/.
- In order to check that the MSVC command line is set up correctly, enter ``cl`` in the command prompt window and verify that the output looks something like this:
.. code-block:: console
> cl
Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24245 for x64
Copyright (C) Microsoft Corporation. All rights reserved.
usage: cl [ option... ] filename... [ /link linkoption... ]
- Make the compiler available. Its typical installation location is `C:\\Program files (x86)\\Microsoft Visual Studio`, so change your current directory there. Then, to set up the build architecture environment you can open a command prompt window in the `VC\\Auxiliary\\Build` subdirectory and execute `vcvarsall.bat x64` if your architecture is 64 bits or `vcvarsall.bat x86` if it is 32 bits.
- You will need to know the path where the Blosc2 wheel has installed its files. For this we will use the `dir /s` command (but you can use your preferred location method):
.. code-block:: console
> dir /s c:\blosc2.lib
Volume in drive C is OS
Volume Serial Number is 7A21-A5D5
Directory of c:\Users\user\miniconda3\Lib
14/12/2020 09:56 20.848 blosc2.lib
1 File(s) 20.848 bytes
Total list files:
1 File(s) 20.848 bytes
0 dirs 38.981.902.336 free bytes
- The output shows the path of blosc2.lib in your system, but we are rather interested in the parent one:
.. code-block:: console
> set WHEEL_DIR=c:\Users\user\miniconda3
- Now, it is important to copy the library `libblosc2.dll` to C:\\Windows\\System32 directory, so it can be found by the executable when it is necessary.
- Finally, to compile C files using Blosc2 libraries, enter this command:
.. code-block:: console
> cl .c /Ox /Fe.exe /I /MT /link/NODEFAULTLIB:MSVCRT
- For instance, in the case of blosc "examples/urcodecs.c":
.. code-block:: console
> cl urcodecs.c %WHEEL_DIR%\lib\blosc2.lib /Ox /Feurcodecs.exe /I%WHEEL_DIR%\include /MT /link/NODEFAULTLIB:MSVCRT
Microsoft (R) C/C++ Optimizing Compiler Version 19.10.25017 for x86
Copyright (C) Microsoft Corporation. All rights reserved.
urcodecs.c
Microsoft (R) Incremental Linker Version 14.10.25017.0
Copyright (C) Microsoft Corporation. All rights reserved.
/out:urcodecs.exe
/NODEFAULTLIB:MSVCRT
urcodecs.obj
/NODEFAULTLIB:MSVCRT
.\miniconda3\lib\blosc2.lib
- And you can run your program:
.. code-block:: console
> urcodecs
Blosc version info: 2.0.0 ($Date:: 2021-05-26 #$)
Compression ratio: 381.5 MB -> 0.0 MB (14013.5x)
Compression time: 0.261 s, 1462.1 MB/s
Decompression time: 0.0669 s, 5698.2 MB/s
Successful roundtrip data <-> schunk !
Compiling C files with Blosc2 wheels on Linux
---------------------------------------------
- Find the path where Blosc2 wheel has installed its files:
.. code-block:: console
$ find / -name libblosc2.so 2>/dev/null
/home/user/miniconda3/lib/libblosc2.so
- The output shows the path of libblosc2.so, but we are rather interested in the parent one:
.. code-block:: console
$ WHEEL_DIR=/home/user/miniconda3
- To compile C files using Blosc2 you only need to enter the commands:
.. code-block:: console
$ export LD_LIBRARY_PATH=
$ gcc .c -I -o -L -lblosc2
- For instance, let's compile blosc's "examples/urcodecs.c":
.. code-block:: console
$ export LD_LIBRARY_PATH=$WHEEL_DIR/lib # note that you need the LD_LIBRARY_PATH env variable
$ gcc urcodecs.c -I$WHEEL_DIR/include -o urcodecs -L$WHEEL_DIR/lib -lblosc2
- Run your program:
.. code-block:: console
$ ./urcodecs
Blosc version info: 2.0.0-dev0 ($Date:: 2021-05-26 #$)
Compression ratio: 381.5 MB -> 0.0 MB (14013.5x)
Compression time: 1.46 s, 260.7 MB/s
Decompression time: 0.509 s, 749.1 MB/s
Successful roundtrip data <-> schunk !
- Rejoice!
Blosc-c-blosc2-d3bee7c/CONTRIBUTING.rst 0000664 0000000 0000000 00000005017 15255167342 0017412 0 ustar 00root root 0000000 0000000 Contributing guidelines
=======================
We want to make contributing to this project as easy and transparent as
possible.
Our Development Process
-----------------------
New versions are being developed in the "main" branch,
or in their own feature branch.
When they are deemed ready for a release, they are merged back into "main"
again.
So all contributions must stage first through "main"
or their own feature branch.
Pull Requests
-------------
We actively welcome your pull requests.
1. Fork the repo and create your branch from `main`.
2. If you've added code that should be tested, add tests.
3. If you've changed APIs, update the documentation.
4. Ensure the test suite passes.
5. Make sure your code does not issue new compiler warnings.
Automated Contributions Policy
------------------------------
Contributing to Blosc requires human judgment, contextual understanding, and familiarity with the project's structure and goals. It is not a task suitable for fully-automated AI tools.
Please refrain from submitting issues or pull requests generated by fully-automated tools. Maintainers reserve the right, at their sole discretion, to close such submissions and to block any account responsible for them.
Review all code or documentation changes made by AI tools and make sure you understand all changes and can explain them on request, before submitting them under your name. Do not submit any AI-generated code that you haven’t personally reviewed, understood and tested, as this wastes maintainers’ time.
Please do not paste AI generated text in the description of issues, PRs or in comments as this makes it harder for reviewers to assess your contribution. We are happy for it to be used to improve grammar or if you are not a native English speaker.
If you used AI tools, please state so in your PR description.
PRs that appear to violate this policy will be closed without review.
For more information on why this is becoming a big issue for open source projects, please see the `Software Review in the Era of AI `_ by the rOpenSci folks.
Issues
------
We use GitHub issues to track public bugs. Please ensure your description is
clear and has sufficient instructions to be able to reproduce the issue.
Coding Style
------------
* 2 spaces for indentation rather than tabs.
License
-------
By contributing to C-Blosc2, you agree that your contributions will be licensed
under the `LICENSE `_
file of the project.
Blosc-c-blosc2-d3bee7c/DEVELOPING-GUIDE.rst 0000664 0000000 0000000 00000001340 15255167342 0017765 0 ustar 00root root 0000000 0000000 Some conventions used in C-Blosc2
=================================
* Use C99 designated initialization only in examples. Libraries should use C89 initialization, which is more portable, specially with C++ (designated initialization in C++ is supported only since C++20).
* Use _new and _free for memory allocating constructors and destructors and _init and _destroy for non-memory allocating constructors and destructors.
* Lines must not exceed 120 characters. If a line is too long, it must be broken into several lines.
* Conditional bodies must always use braces, even if they are one-liners. The only exception that can be is when the conditional is a single line and the body is a single line:
if (condition) whatever();
Blosc-c-blosc2-d3bee7c/FAQ.md 0000664 0000000 0000000 00000001634 15255167342 0015703 0 ustar 00root root 0000000 0000000 ## Which is the compatibility policy for the Blosc library and format?
### Compatibility among libraries with different minor versions
The compatibility between minor versions (e.g. 2.0 <-> 2.1) will always be *both* backward and forward. This means that any version of the library will be able to read any data produced with another version differing only in the patch level or minor version. Not being able to achieve that will be considered a bug, and action will be taken to fix that as soon as possible (see e.g. https://github.com/Blosc/c-blosc/issues/215).
### Compatibility among libraries with different major versions
There will be an *attempt* (but not an absolute guarantee) of backward compatibility, but it won’t be an effort for guaranteeing forward compatibility. For example, Blosc2 2.1 is able to read any data produced by Blosc 1.x, but Blosc 1.x are not able to read data produced by Blosc2 2.x.
Blosc-c-blosc2-d3bee7c/FUNDING.yml 0000664 0000000 0000000 00000000105 15255167342 0016557 0 ustar 00root root 0000000 0000000 github: [numfocus]
custom: ['https://numfocus.org/donate-to-blosc']
Blosc-c-blosc2-d3bee7c/LICENSE.txt 0000664 0000000 0000000 00000003154 15255167342 0016574 0 ustar 00root root 0000000 0000000 BSD License
For Blosc - A blocking, shuffling and lossless compression library
Copyright (c) 2009-2018 Francesc Alted
Copyright (c) 2019-present Blosc Development Team
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Francesc Alted nor the names of its contributors may be used
to endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Blosc-c-blosc2-d3bee7c/LICENSES/ 0000775 0000000 0000000 00000000000 15255167342 0016153 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/LICENSES/BITSHUFFLE.txt 0000664 0000000 0000000 00000002174 15255167342 0020353 0 ustar 00root root 0000000 0000000 Bitshuffle - Filter for improving compression of typed binary data.
Copyright (c) 2014 Kiyoshi Masui (kiyo@physics.ubc.ca)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
Blosc-c-blosc2-d3bee7c/LICENSES/FASTLZ.txt 0000664 0000000 0000000 00000002157 15255167342 0017724 0 ustar 00root root 0000000 0000000 FastLZ - Byte-aligned LZ77 compression library
Copyright (C) 2005-2020 Ariya Hidayat
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
Blosc-c-blosc2-d3bee7c/LICENSES/LZ4.txt 0000664 0000000 0000000 00000002437 15255167342 0017333 0 ustar 00root root 0000000 0000000 LZ4 Library
Copyright (c) 2011-2020, Yann Collet
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Blosc-c-blosc2-d3bee7c/LICENSES/ZLIB.txt 0000664 0000000 0000000 00000001543 15255167342 0017457 0 ustar 00root root 0000000 0000000 (C) 1995-2013 Jean-loup Gailly and Mark Adler
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
Blosc-c-blosc2-d3bee7c/LICENSES/ZSTD.txt 0000664 0000000 0000000 00000002772 15255167342 0017510 0 ustar 00root root 0000000 0000000 BSD License
For Zstandard software
Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Facebook nor the names of its contributors may be used to
endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Blosc-c-blosc2-d3bee7c/README.rst 0000664 0000000 0000000 00000025003 15255167342 0016435 0 ustar 00root root 0000000 0000000 ========
C-Blosc2
========
A fast, compressed and persistent data store library for C
==========================================================
:Author: Blosc Development Team
:Contact: blosc@blosc.org
:URL: https://www.blosc.org
:Gitter: |gitter|
:Actions: |actions|
:NumFOCUS: |numfocus|
:Code of Conduct: |Contributor Covenant|
.. |gitter| image:: https://badges.gitter.im/Blosc/c-blosc.svg
:alt: Join the chat at https://gitter.im/Blosc/c-blosc
:target: https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
.. |actions| image:: https://github.com/Blosc/c-blosc2/workflows/CI%20CMake/badge.svg
:target: https://github.com/Blosc/c-blosc2/actions?query=workflow%3A%22CI+CMake%22
.. |numfocus| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A
:target: https://numfocus.org
.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg
:target: https://github.com/Blosc/community/blob/master/code_of_conduct.md
What is it?
===========
`Blosc2 `_ is a high-performance compressor and data format optimized for binary data, including numerical arrays, tensors, and other structured formats. It provides a flexible framework of codecs and filters, enabling developers to balance compression speed and ratio for specific use cases, from high-throughput data pipelines to persistent storage. As the successor to the original Blosc library (released in 2010), Blosc2 is built on a mature foundation and is integrated into many popular scientific computing libraries, such as PyTables, h5py, and Zarr.
C-Blosc2 is the latest major version of `C-Blosc `_, and is backward compatible with both the C-Blosc1 API and its in-memory format. However, the reverse thing is generally not true for the format; buffers generated with C-Blosc2 are not format-compatible with C-Blosc1 (i.e. forward compatibility is not supported). In case you want to ensure full API compatibility with C-Blosc1 API, define the `BLOSC1_COMPAT` symbol.
See a 3-minute `introductory video to Blosc2 `_.
Blosc2 NDim: an N-Dimensional store
===================================
One of the more exciting additions in C-Blosc2 is the `Blosc2 NDim layer `_ (or B2ND for short), which allows creating *and* reading n-dimensional datasets in an extremely efficient way thanks to a n-dim 2-level partitioning, that allows slicing and dicing arbitrary large and compressed data in a more fine-grained way:
.. image:: https://github.com/Blosc/c-blosc2/blob/main/images/b2nd-2level-parts.png?raw=true
:width: 75%
To whet your appetite, here is how the `NDArray` object in the `Python wrapper`_ performs on getting slices orthogonal to the different axis of a 4-dim dataset:
.. image:: https://github.com/Blosc/c-blosc2/blob/main/images/Read-Partial-Slices-B2ND.png?raw=true
:width: 75%
We have blogged about this: https://www.blosc.org/posts/blosc2-ndim-intro
We also have a ~2 min explanatory video on `why slicing in a pineapple-style (aka double partition)
is useful `_:
.. image:: https://github.com/Blosc/blogsite/blob/master/files/images/slicing-pineapple-style.png?raw=true
:width: 50%
:alt: Slicing a dataset in pineapple-style
:target: https://www.youtube.com/watch?v=LvP9zxMGBng
.. include:: ../WHATS-NEW.rst
More info about the `improved capabilities of C-Blosc2 can be found in this paper `_. Please, cite it if you use C-Blosc2 in your research!
Open format
===========
The Blosc2 format is open and `fully documented `_.
The format specs are defined in less than 4000 words, so they should be easy to read and understand. In our opinion, this is critical for the long-term success of the library, as it allows for third-party implementations of the format, and also for the users to understand what is going on under the hood.
Python wrapper
==============
We are officially supporting (thanks to the Python Software Foundation) a `Python wrapper for Blosc2 `_. It supports all the features of the predecessor `python-blosc `_ package plus most of the bells and whistles of C-Blosc2, like 64-bit and multidimensional containers. As a bonus, the `python-blosc2` package comes with wheels and binary versions of the C-Blosc2 libraries, so anyone, even non-Python users can install C-Blosc2 binaries easily with:
.. code-block:: console
pip install blosc2
Compiling the C-Blosc2 library with CMake
=========================================
Blosc can be built, tested and installed using `CMake `_. The following procedure describes a typical CMake build.
Create the build directory inside the sources and move into it:
.. code-block:: console
git clone https://github.com/Blosc/c-blosc2
cd c-blosc2
mkdir build
cd build
Now run CMake configuration and optionally specify the installation
directory (e.g. '/usr' or '/usr/local'):
.. code-block:: console
cmake -DCMAKE_INSTALL_PREFIX=your_install_prefix_directory ..
CMake allows configuring Blosc in many different ways, including selecting external codec libraries, fetching pinned upstream codec sources during configuration, or disabling selected codecs altogether. Please note that configuration can also be performed using UI tools provided by CMake (`ccmake` or `cmake-gui`):
.. code-block:: console
ccmake .. # run a curses-based interface
cmake-gui .. # run a graphical interface
Build, test and install Blosc:
.. code-block:: console
cmake --build .
ctest
cmake --build . --target install
The static and dynamic version of the Blosc library, together with header files, will be installed into the specified CMAKE_INSTALL_PREFIX.
Once you have compiled your Blosc library, you can easily link your apps with it as shown in the `examples/ directory `_.
Handling support for codecs (LZ4, LZ4HC, Zstd, Zlib, ZFP)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
C-Blosc2 can either fetch pinned upstream codec releases via CMake ``FetchContent`` or use codec libraries already installed on the system. The top-level switch is ``BLOSC_DEPENDENCY_MODE``:
``BUNDLED``
Default. Fetch and build the bundled dependency versions. This preserves the traditional self-contained C-Blosc2 build behavior. Bundled dependencies are private: their headers and libraries are not installed separately, and static builds fold them into ``libblosc2.a``.
``EXTERNAL``
Require system packages for mandatory dependencies and fail with a clear error if they are missing. This is the recommended mode for distro packagers that must avoid bundled third-party code::
cmake -DBLOSC_DEPENDENCY_MODE=EXTERNAL ..
``AUTO``
Try system packages first and fall back to bundled dependencies. This is convenient for local development, but less reproducible for distro packaging.
The older ``PREFER_EXTERNAL_LZ4``, ``PREFER_EXTERNAL_ZLIB`` and ``PREFER_EXTERNAL_ZSTD`` options are still accepted as per-dependency compatibility options; when enabled they try the system package first for that dependency.
ZFP support is controlled separately because ZFP packages are not available in all distributions:
``BLOSC_ENABLE_ZFP=AUTO``
Default. In ``BUNDLED`` mode, build bundled ZFP. In ``EXTERNAL`` mode, enable ZFP only when a system ZFP package is found; otherwise build C-Blosc2 without ZFP support. In ``AUTO`` dependency mode, try system ZFP first and fall back to bundled ZFP.
``BLOSC_ENABLE_ZFP=ON``
Require ZFP support. In ``EXTERNAL`` mode this fails if no system ZFP package is found.
``BLOSC_ENABLE_ZFP=OFF``
Disable the ZFP codec.
For fully local bundled builds without network access, point CMake at local codec checkouts instead of downloading them:
.. code-block:: console
cmake \
-DBLOSC_LZ4_SOURCE_DIR=/path/to/lz4 \
-DBLOSC_ZLIBNG_SOURCE_DIR=/path/to/zlib-ng \
-DBLOSC_ZSTD_SOURCE_DIR=/path/to/zstd \
-DBLOSC_ZFP_SOURCE_DIR=/path/to/zfp \
..
You can also exclude support for selected codecs. For example, to disable Zstd support:
.. code-block:: console
cmake -DDEACTIVATE_ZSTD=ON ..
Supported platforms
~~~~~~~~~~~~~~~~~~~
C-Blosc2 is meant to support all platforms where a C99 compliant C compiler can be found. The ones that are mostly tested are Intel (Linux, Mac OSX and Windows), ARM (Linux, Mac), and PowerPC (Linux). More on ARM support in `README_ARM.rst`.
For Windows, you will need at least VS2015 or higher on x86 and x64 targets (i.e. ARM is not supported on Windows).
For Mac OSX, make sure that you have the command line developer tools available. You can always install them with:
.. code-block:: console
xcode-select --install
For Mac OSX on arm64 architecture, you may want to compile it like this:
.. code-block:: console
CC="clang -arch arm64" cmake ..
Display error messages
~~~~~~~~~~~~~~~~~~~~~~
By default error messages are disabled. To display them, you just need to activate the Blosc tracing machinery by setting
the ``BLOSC_TRACE`` environment variable.
Contributing
============
If you want to collaborate in this development you are welcome. We need help in the different areas listed at the `ROADMAP `_; also, be sure to read our `DEVELOPING-GUIDE `_ and our `Code of Conduct `_. Blosc is distributed using the `BSD license `_.
Social feeds
============
Follow `@Blosc2 `_ on Mastodon to get informed about the latest developments.
Citing Blosc
============
You can cite our work on the different libraries under the Blosc umbrella as:
.. code-block:: console
@ONLINE{blosc,
author = {{Blosc Development Team}},
title = "{A fast, compressed and persistent data store library}",
year = {2009-2026},
note = {https://blosc.org}
}
Acknowledgments
===============
See `THANKS document `_.
----
*Compress Better, Compute Bigger*
-- Blosc Development Team
Blosc-c-blosc2-d3bee7c/README_ARM.rst 0000664 0000000 0000000 00000037115 15255167342 0017143 0 ustar 00root root 0000000 0000000 Support for Apple arm64
=======================
Since November 2020 Apple is releasing Mac boxes using CPUs with its own
arm64 implementation. Currently, you can compile C-Blosc2 with:
.. code-block:: console
mkdir build
cd build
CC="clang -arch arm64" cmake ..
The NEON extensions are used automatically in this case.
Here are a couple of benchmarks on BloscLZ and LZ4 codecs running on a MacBook Air
with a new M1 arm64 processor:
.. code-block:: console
(base) francesc@Francescs-MacBook-Air build % bench/b2bench blosclz
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
List of supported compressors in this build: blosclz,lz4,lz4hc,zlib,zstd
Supported compression libraries:
BloscLZ: 2.3.0
LZ4: 1.9.1
Zlib: 10.0.3
Zstd: 1.4.5
Using compressor: blosclz
Using shuffle type: shuffle
Running suite: single
--> 4, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 4
********************** Running benchmarks *********************
memcpy(write): 327.0 us, 12234.1 MB/s
memcpy(read): 127.1 us, 31460.5 MB/s
Compression level: 0
comp(write): 136.6 us, 29274.7 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 101.8 us, 39281.3 MB/s OK
Compression level: 1
comp(write): 443.7 us, 9014.3 MB/s Final bytes: 1102576 Ratio: 3.80
decomp(read): 99.4 us, 40242.1 MB/s OK
Compression level: 2
comp(write): 414.2 us, 9656.0 MB/s Final bytes: 1083528 Ratio: 3.87
decomp(read): 99.6 us, 40178.6 MB/s OK
Compression level: 3
comp(write): 451.0 us, 8869.1 MB/s Final bytes: 1083528 Ratio: 3.87
decomp(read): 102.3 us, 39107.8 MB/s OK
Compression level: 4
comp(write): 568.7 us, 7033.2 MB/s Final bytes: 165194 Ratio: 25.39
decomp(read): 135.8 us, 29447.2 MB/s OK
Compression level: 5
comp(write): 565.2 us, 7076.6 MB/s Final bytes: 165194 Ratio: 25.39
decomp(read): 135.5 us, 29512.3 MB/s OK
Compression level: 6
comp(write): 523.5 us, 7640.9 MB/s Final bytes: 141642 Ratio: 29.61
decomp(read): 138.2 us, 28943.9 MB/s OK
Compression level: 7
comp(write): 520.4 us, 7687.0 MB/s Final bytes: 138042 Ratio: 30.38
decomp(read): 137.3 us, 29136.4 MB/s OK
Compression level: 8
comp(write): 524.1 us, 7632.7 MB/s Final bytes: 137610 Ratio: 30.48
decomp(read): 138.4 us, 28893.1 MB/s OK
Compression level: 9
comp(write): 534.4 us, 7485.3 MB/s Final bytes: 73254 Ratio: 57.26
decomp(read): 127.5 us, 31371.5 MB/s OK
Round-trip compr/decompr on 7.5 GB
Elapsed time: 1.2 s, 13694.7 MB/s
(base) francesc@Francescs-MacBook-Air build % bench/b2bench lz4
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
List of supported compressors in this build: blosclz,lz4,lz4hc,zlib,zstd
Supported compression libraries:
BloscLZ: 2.3.0
LZ4: 1.9.1
Zlib: 10.0.3
Zstd: 1.4.5
Using compressor: lz4
Using shuffle type: shuffle
Running suite: single
--> 4, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 4
********************** Running benchmarks *********************
memcpy(write): 331.2 us, 12076.7 MB/s
memcpy(read): 134.7 us, 29689.2 MB/s
Compression level: 0
comp(write): 132.2 us, 30259.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 109.0 us, 36709.8 MB/s OK
Compression level: 1
comp(write): 589.3 us, 6787.9 MB/s Final bytes: 991408 Ratio: 4.23
decomp(read): 244.4 us, 16367.1 MB/s OK
Compression level: 2
comp(write): 658.0 us, 6078.7 MB/s Final bytes: 877344 Ratio: 4.78
decomp(read): 313.3 us, 12765.8 MB/s OK
Compression level: 3
comp(write): 670.3 us, 5967.1 MB/s Final bytes: 786514 Ratio: 5.33
decomp(read): 312.4 us, 12804.1 MB/s OK
Compression level: 4
comp(write): 420.5 us, 9512.9 MB/s Final bytes: 428754 Ratio: 9.78
decomp(read): 244.8 us, 16340.0 MB/s OK
Compression level: 5
comp(write): 419.5 us, 9536.0 MB/s Final bytes: 428754 Ratio: 9.78
decomp(read): 245.2 us, 16311.3 MB/s OK
Compression level: 6
comp(write): 413.5 us, 9673.5 MB/s Final bytes: 379116 Ratio: 11.06
decomp(read): 239.3 us, 16713.1 MB/s OK
Compression level: 7
comp(write): 414.2 us, 9656.7 MB/s Final bytes: 379116 Ratio: 11.06
decomp(read): 240.0 us, 16666.0 MB/s OK
Compression level: 8
comp(write): 413.2 us, 9680.5 MB/s Final bytes: 379116 Ratio: 11.06
decomp(read): 237.7 us, 16830.1 MB/s OK
Compression level: 9
comp(write): 414.3 us, 9654.1 MB/s Final bytes: 379116 Ratio: 11.06
decomp(read): 239.8 us, 16684.0 MB/s OK
Round-trip compr/decompr on 7.5 GB
Elapsed time: 1.4 s, 11720.1 MB/s
Yes, the speed is really good on the new Apple architecture!
ARM environment for Ubuntu Intel
================================
In order to create executable code for a platform other than the one on
which the compiler is running we can use cross compilers. In this case we
want to create executables for ARM platforms but execute them on
Intel platform with Ubuntu. The only package that is necessary
(at least in Ubuntu 15.04) is gcc-arm-linux-gnueabihf.
Once this is installed the way to compile an example is:
.. code-block:: console
$ arm-linux-gnueabihf-gcc -static -mfpu=neon -flax-vector-conversions -DSHUFFLE_NEON_ENABLED -O3 simple.c ../blosc/blosc.c ../blosc/blosclz.c ../blosc/shuffle.c ../blosc/shuffle-generic.c ../blosc/bitshuffle-generic.c ../blosc/shuffle-neon.c ../blosc/bitshuffle-neon.c -I../blosc -o simple -lpthread
Another example for running a bench with the cross compiler:
.. code-block:: console
$ arm-linux-gnueabihf-gcc -static -mfpu=neon -flax-vector-conversions -DSHUFFLE_NEON_ENABLED -O3 bench.c ../blosc/blosc.c ../blosc/blosclz.c ../blosc/shuffle.c ../blosc/shuffle-generic.c ../blosc/bitshuffle-generic.c ../blosc/shuffle-neon.c ../blosc/bitshuffle-neon.c -I../blosc -o bench -lpthread
In these cases the NEON flags are: `-mfpu=neon -flax-vector-conversions`
This is explained in detail in: https://linux-sunxi.org/Toolchain
This way you can develop and debug applications for ARM on intel machines as if
you were in ARM platforms.
Benchmark for ODROID-XU3
========================
This is a benchmark to compare the speeds between the NEON and the generic
implementation.
::
CPU: ARMv7 Processor rev 3 (v7l)
Compiler: gcc, version gcc-4.8.real
Optimizations: -O3
OS: Ubuntu 14.04 trusty
• NEON implementation results:
::
Blosc version: 2.0.0a1 ($Date:: 2015-07-30 #$)
List of supported compressors in this build: blosclz
Supported compression libraries:
BloscLZ: 1.0.5
LZ4: unknown
Zlib: unknown
Using compressor: blosclz
Using shuffle type: shuffle
Running suite: suite
--> 1, 2097152, 8, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0a1 ($Date:: 2015-07-30 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 2097152 bytes Type size: 8 bytes
Working set: 64.0 MB Number of threads: 1
********************** Running benchmarks *********************
memcpy(write): 2265.1 us, 883.0 MB/s
memcpy(read): 1196.9 us, 1671.0 MB/s
Compression level: 0
comp(write): 984.8 us, 2030.9 MB/s Final bytes: 2097168 Ratio: 1.00
decomp(read): 1202.8 us, 1662.8 MB/s OK
Compression level: 1
comp(write): 5863.7 us, 341.1 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 1014.1 us, 1972.3 MB/s OK
Compression level: 2
comp(write): 6229.8 us, 321.0 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 1013.4 us, 1973.5 MB/s OK
Compression level: 3
comp(write): 6603.8 us, 302.9 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 1012.9 us, 1974.4 MB/s OK
Compression level: 4
comp(write): 6792.6 us, 294.4 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 983.5 us, 2033.6 MB/s OK
Compression level: 5
comp(write): 8598.3 us, 232.6 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 983.4 us, 2033.9 MB/s OK
Compression level: 6
comp(write): 9866.2 us, 202.7 MB/s Final bytes: 546320 Ratio: 3.84
decomp(read): 1079.1 us, 1853.3 MB/s OK
Compression level: 7
comp(write): 9334.9 us, 214.2 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 1959.4 us, 1020.7 MB/s OK
Compression level: 8
comp(write): 9221.1 us, 216.9 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 1972.1 us, 1014.2 MB/s OK
Compression level: 9
comp(write): 8452.0 us, 236.6 MB/s Final bytes: 153160 Ratio: 13.69
decomp(read): 2780.0 us, 719.4 MB/s OK
--> 2, 2097152, 8, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0a1 ($Date:: 2015-07-30 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 2097152 bytes Type size: 8 bytes
Working set: 64.0 MB Number of threads: 2
********************** Running benchmarks *********************
memcpy(write): 2258.0 us, 885.8 MB/s
memcpy(read): 1194.9 us, 1673.7 MB/s
Compression level: 0
comp(write): 831.2 us, 2406.0 MB/s Final bytes: 2097168 Ratio: 1.00
decomp(read): 1162.6 us, 1720.3 MB/s OK
Compression level: 1
comp(write): 2975.1 us, 672.2 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 738.4 us, 2708.4 MB/s OK
Compression level: 2
comp(write): 3156.8 us, 633.6 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 738.7 us, 2707.5 MB/s OK
Compression level: 3
comp(write): 3347.0 us, 597.6 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 749.6 us, 2668.1 MB/s OK
Compression level: 4
comp(write): 3486.2 us, 573.7 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 745.1 us, 2684.4 MB/s OK
Compression level: 5
comp(write): 4488.5 us, 445.6 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 725.7 us, 2755.9 MB/s OK
Compression level: 6
comp(write): 4998.5 us, 400.1 MB/s Final bytes: 546320 Ratio: 3.84
decomp(read): 796.8 us, 2510.2 MB/s OK
Compression level: 7
comp(write): 4780.3 us, 418.4 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 1383.9 us, 1445.1 MB/s OK
Compression level: 8
comp(write): 4778.6 us, 418.5 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 1398.0 us, 1430.6 MB/s OK
Compression level: 9
comp(write): 5884.6 us, 339.9 MB/s Final bytes: 153160 Ratio: 13.69
decomp(read): 2647.7 us, 755.4 MB/s OK
Round-trip compr/decompr on 3.8 GB
Elapsed time: 13.9 s, 609.1 MB/s
• Generic implementation results:
::
Blosc version: 2.0.0a1 ($Date:: 2015-07-30 #$)
List of supported compressors in this build: blosclz
Supported compression libraries:
BloscLZ: 1.0.5
LZ4: unknown
Zlib: unknown
Using compressor: blosclz
Using shuffle type: shuffle
Running suite: suite
--> 1, 2097152, 8, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0a1 ($Date:: 2015-07-30 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 2097152 bytes Type size: 8 bytes
Working set: 64.0 MB Number of threads: 1
********************** Running benchmarks *********************
memcpy(write): 2194.1 us, 911.5 MB/s
memcpy(read): 1170.8 us, 1708.2 MB/s
Compression level: 0
comp(write): 896.2 us, 2231.7 MB/s Final bytes: 2097168 Ratio: 1.00
decomp(read): 1179.3 us, 1695.9 MB/s OK
Compression level: 1
comp(write): 7534.4 us, 265.4 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 4117.1 us, 485.8 MB/s OK
Compression level: 2
comp(write): 7895.6 us, 253.3 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 4106.7 us, 487.0 MB/s OK
Compression level: 3
comp(write): 8262.2 us, 242.1 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 4113.9 us, 486.2 MB/s OK
Compression level: 4
comp(write): 8495.6 us, 235.4 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 4042.6 us, 494.7 MB/s OK
Compression level: 5
comp(write): 10321.0 us, 193.8 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 4033.9 us, 495.8 MB/s OK
Compression level: 6
comp(write): 11675.3 us, 171.3 MB/s Final bytes: 546320 Ratio: 3.84
decomp(read): 4096.4 us, 488.2 MB/s OK
Compression level: 7
comp(write): 10193.0 us, 196.2 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 7150.9 us, 279.7 MB/s OK
Compression level: 8
comp(write): 10192.3 us, 196.2 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 7167.7 us, 279.0 MB/s OK
Compression level: 9
comp(write): 10418.7 us, 192.0 MB/s Final bytes: 153160 Ratio: 13.69
decomp(read): 7870.8 us, 254.1 MB/s OK
--> 2, 2097152, 8, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0a1 ($Date:: 2015-07-30 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 2097152 bytes Type size: 8 bytes
Working set: 64.0 MB Number of threads: 2
********************** Running benchmarks *********************
memcpy(write): 2243.1 us, 891.6 MB/s
memcpy(read): 1219.9 us, 1639.4 MB/s
Compression level: 0
comp(write): 846.8 us, 2361.9 MB/s Final bytes: 2097168 Ratio: 1.00
decomp(read): 1182.1 us, 1691.9 MB/s OK
Compression level: 1
comp(write): 3867.6 us, 517.1 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 2162.8 us, 924.7 MB/s OK
Compression level: 2
comp(write): 4054.2 us, 493.3 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 2156.6 us, 927.4 MB/s OK
Compression level: 3
comp(write): 4241.2 us, 471.6 MB/s Final bytes: 584976 Ratio: 3.59
decomp(read): 2169.6 us, 921.8 MB/s OK
Compression level: 4
comp(write): 4377.1 us, 456.9 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 7556.9 us, 264.7 MB/s OK
Compression level: 5
comp(write): 5276.3 us, 379.1 MB/s Final bytes: 557840 Ratio: 3.76
decomp(read): 7556.7 us, 264.7 MB/s OK
Compression level: 6
comp(write): 6026.8 us, 331.9 MB/s Final bytes: 546320 Ratio: 3.84
decomp(read): 3108.2 us, 643.5 MB/s OK
Compression level: 7
comp(write): 5877.0 us, 340.3 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 3668.5 us, 545.2 MB/s OK
Compression level: 8
comp(write): 5882.0 us, 340.0 MB/s Final bytes: 216528 Ratio: 9.69
decomp(read): 3531.2 us, 566.4 MB/s OK
Compression level: 9
comp(write): 7621.3 us, 262.4 MB/s Final bytes: 153160 Ratio: 13.69
decomp(read): 4978.3 us, 401.7 MB/s OK
Round-trip compr/decompr on 3.8 GB
Elapsed time: 21.9 s, 385.0 MB/s
We have achieved to implement shuffle NEON instructions for ARM that are twice
as fast as the generic implementation.
Enjoy developing for ARM!
Lucian Marc
Blosc-c-blosc2-d3bee7c/README_B2ND_FORMAT.rst 0000664 0000000 0000000 00000001012 15255167342 0020244 0 ustar 00root root 0000000 0000000 B2ND Format
===========
The B2ND format is meant for storing multidimensional datasets defined by a shape and a data type.
Both the shape and the data type follow the NumPy conventions.
It is just a `B2ND metalayer `_
on top of a Blosc2 `CFrame `_
(for contiguous storage) or `SFrame `_
(for sparse storage).
Blosc-c-blosc2-d3bee7c/README_B2ND_METALAYER.rst 0000664 0000000 0000000 00000006425 15255167342 0020614 0 ustar 00root root 0000000 0000000 B2ND Metalayer Format
=====================
This is a `metalayer `_ on top of a Blosc2
`CFrame `_ or
`SFrame `_
that is meant for storing multidimensional information.
Specifically, this metalayer is named 'b2nd' and follows this format::
|-0-|-1-|-2-|-3-|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~|
| 9X| v | nd| 9X| shape | 9X| chunkshape | 9X| blockshape |
|---|---|---|---|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~|---|~~~~~~~~~~~~~~~~|
^ ^ ^ ^ ^ ^
| | | | | |
| | | | | +--[msgpack] fixarray with X=nd elements
| | | | +--[msgpack] fixarray with X=nd elements
| | | +--[msgpack] fixarray with X=nd elements
| | +--[msgpack] positive fixnum for the number of dimensions (up to 127)
| +--[msgpack] positive fixnum for the metalayer format version (up to 127)
+---[msgpack] fixarray with X=7 elements
The `shape` section is meant to store the actual shape info::
|---|--8 bytes---|---|--8 bytes---|~~~~~|---|--8 bytes---|
| d3| first_dim | d3| second_dim | ... | d3| nth_dim |
|---|------------|---|------------|~~~~~|---|------------|
^ ^ ^
| | |
| | +--[msgpack] int64
| +--[msgpack] int64
+--[msgpack] int64
Next, the `chunkshape` section is meant to store the actual chunk shape info::
|---|--4 bytes---|---|--4 bytes---|~~~~~|---|--4 bytes---|
| d2| first_dim | d2| second_dim | ... | d2| nth_dim |
|---|------------|---|------------|~~~~~|---|------------|
^ ^ ^
| | |
| | +--[msgpack] int32
| +--[msgpack] int32
+--[msgpack] int32
Next, the `blockshape` section is meant to store the actual block shape info::
|---|--4 bytes---|---|--4 bytes---|~~~~~|---|--4 bytes---|
| d2| first_dim | d2| second_dim | ... | d2| nth_dim |
|---|------------|---|------------|~~~~~|---|------------|
^ ^ ^
| | |
| | +--[msgpack] int32
| +--[msgpack] int32
+--[msgpack] int32
Finally, the `dtype` section is meant to store the data type information::
|---|---|--4 bytes---|--------------|
| XX| db| dtype_len | dtype_string |
|---|---|------------|--------------|
^ ^
| |
| +--[msgpack] str32
+--[msgpack] positive fixint (7-bit integer). dtype_format; 0 means NumPy format.
The 0 value for dtype_format means that the dtype_string field follows the NumPy convention
(e.g. an `int32_t` dtype is represented as "`_ contiguously,
either in-memory or on-disk.
The frame is composed of a header, a chunks section, and a trailer::
+---------+--------+---------+
| header | chunks | trailer |
+---------+--------+---------+
Each of the three parts of the frame are variable length; with the header and trailer both stored using the
`msgpack `_ format.
*Note:* Integer types are stored in big endian for msgpack format. All the rest are stored in little endian.
Header
------
The header contains information needed to decompress the Blosc chunks contained in the frame. It is encoded using
msgpack and the format is as follows::
|-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-|-10|-11|-12|-13|-14|-15|-16|-17|
| 9X| aX| "b2frame\0" | d2| header_len | cf| frame_len |
|---|---|-------------------------------|---|---------------|---|-------------------------------|
^ ^ ^ ^ ^
| | | | |
| | | | +--[msgpack] uint64
| | | |
| | | +--[msgpack] int32
| | +---magic number, currently "b2frame"
| +------[msgpack] str with X=8 elements
+---[msgpack] fixarray with X=0xE (14) elements
|-18|-19|-1A|-1B|-1C|-1D|-1E|-1F|-20|-21|-22|-23|-24|-25|-26|-27|-28|-29|-2A|-2B|-2C|-2D|-2E|
| a4|_f0|_f1|_f2|_f3| d3| uncompressed_size | d3| compressed_size |
|---|---|---|---|---|---|-------------------------------|---|-------------------------------|
^ ^ ^ ^ ^ ^ ^
| | | | | | +--[msgpack] int64
| | | | | +--[msgpack] int64
| | | | +-- other flags
| | | +--codec_flags (see below)
| | +---frame_type (see below)
| +------general_flags (see below)
+---[msgpack] str with 4 elements (flags)
|-2F|-30|-31|-32|-33|-34|-35|-36|-37|-38|-39|-3A|-3B|-3C|-3D|-3E|-3F|-40|-41|-42|-43|-44|
| d2| type_size | d2| block_size | d2| chunk_size | d1| tcomp | d1|tdecomp| cX|
|---|---------------|---|---------------|---|---------------|---|-------|---|-------|---|
^ ^ ^ ^ ^ ^ ^ ^
| | | | | | | +-- [msgpack] bool for has_vlmetalayers
| | | | | | +-- number of threads for decompression
| | | | | +-- [msgpack] int16
| | | | +-- number of threads for compression
| | | +-- [msgpack] int16
| | +-- [msgpack] int32
| +-- [msgpack] int32
+-- [msgpack] int32
The filter pipeline is stored next in the header. It contains 6 slots, one for each filter that can be applied. For
each slot there is a byte used to store the filter ID in `filters` and an associated byte used to store any
possible filter meta-info in `filters_meta`::
|-45|-46|-47|-48|-49|-4A|-4B|-4C|-4D|-4E|-4F|-50|-51|-52|-53|-54|-55|-56|
| d2| X | filters |_f4|_f5| filters_meta |_of| |
|---|---|-------------------------------|-------------------------------|
^ ^ ^ ^ ^ ^
| | | | | +-- reserved
| | | | +-- other_flags2 (see below)
| | | +-- compcodec_meta
| | +-- udcodec
| +--number of filters
+--[msgpack] fixext 16
The last section of the header is for the *metalayers*, which contain meta-information about the data in the
frame. It is mandatory the use of the msgpack format for storing them, although the user may use another format
(e.g. json) encoded as msgpack (in this case as a string). Here is the format for the *metalayers*::
|-57|-58|-59|-5A|-5B|-5C|-5D|====================|---|---|---|================|
| 93| cd| idx | de| size | meta keys/values | dc| idy | meta content |
|---|---|-------|---|---|---|====================|---|-------|================|
^ ^ ^ ^ ^ ^ ^ ^ ^
| | | | | | | | +-- [msgpack] bin32
| | | | | | | +-- number of elements in the array
| | | | | | +-- [msgpack] array16
| | | | | +-- [msgpack] fixstr/int32
| | | | +-- number of elements in the map
| | | +-- [msgpack] map of name/offset pairs
| | +-- size of metalayers
| +-- [msgpack] uint16
+-- [msgpack] fixarray with 3 elements
:header_len:
(``int32``) Length of the header of the frame (including metalayers).
:frame_len:
(``uint64``) Length of the whole frame (including compressed data).
:general_flags:
(``uint8``) General flags.
:``0`` to ``3``:
Format version.
:``4`` and ``5``:
Enumerated for chunk offsets.
:``0``:
32-bit
:``1``:
64-bit
:``2``:
128-bit
:``3``:
256-bit
:``6``:
Chunks of fixed length (0) or variable length (1)
:``7``:
All chunks in the frame use variable-length blocks (1) or regular blocks (0)
Frames must not mix regular chunks and variable-length-block chunks.
:frame_type:
(``uint8``) The type of frame.
:``0`` to ``3``:
Enumerated for the type of frame (up to 16).
:``0``:
``Contiguous``
:``1``:
``Sparse (directory)``
:``2 to 15``:
Reserved
:``4`` to ``7``: Reserved for user-defined frame types (up to 16)
:codec_flags:
(``uint8``) Compressor enumeration (defaults for all the chunks in storage).
:``0`` to ``3``:
Enumerated for codecs (up to 16).
:``0``:
``blosclz``
:``1``:
``lz4`` or ``lz4hc``
:``2``:
reserved (slot previously occupied by ``snappy`` and free now)
:``3``:
``zlib``
:``4``:
``zstd``
:``5``:
reserved
:``6``:
The compressor is defined in the user-defined codec slot (see below).
:``7 to 15``:
Reserved
:``4`` to ``7``: Compression level (up to 16)
:other_flags:
(``uint8``) Split mode.
:``0`` to ``1``:
Enumerated for splitmodes (up to 4).
:``0``:
``BLOSC_ALWAYS_SPLIT``
:``1``:
``BLOSC_NEVER_SPLIT``
:``2``:
``BLOSC_AUTO_SPLIT``
:``3``:
``BLOSC_FORWARD_COMPAT_SPLIT``
:``2`` to ``7``: Reserved.
:other_flags2:
(``uint8``) Additional compression flags.
:``0``:
Use dictionary compression (1) or not (0). Only meaningful for codecs that support
dictionaries (e.g. ZSTD).
:``1`` to ``7``: Reserved.
:uncompressed_size:
(``int64``) Size of uncompressed data in frame (excluding metadata).
:compressed_size:
(``int64``) Size of compressed data in frame (excluding metadata).
:type_size:
(``int32``) Size of each item.
:block_size:
(``int32``) Size of data blocks when all data chunks are equal size (the only case supported so far).
:chunk_size:
(``int32``) Size of each data chunk. 0 if not a fixed chunksize (not supported yet).
:tcomp:
(``int16``) Number of threads for compression. If 0, same than `cctx`.
:tdecomp:
(``int16``) Number of threads for decompression. If 0, same than `dctx`.
:udcodec:
(``uint8``) User-defined codec identifier.
:compcode_meta:
(``uint8``) Compression codec metadata associated with the compression codec. Only used in user-defined codecs.
:map of metalayers:
This is a *msgpack-formatted* map for the different metalayers. The keys will be a string (0xa0 + namelen) for
the names of the metalayers, followed by an int32 (0xd2) for the *offset* of the value of this metalayer. The
actual value will be encoded as a bin32 (0xc6) value later in header.
Dumping info in metalayers
~~~~~~~~~~~~~~~~~~~~~~~~~~
**Note:** The method in this section only works for Unix.
Here is a trick for printing the content of metalayers using the nice set of
`msgpack-tools `_ command line utilities. After installing the package we
can do e.g.::
$ msgpack2json -Bi plugins/test_data/example_day_month_temp.b2nd
["b2frame\u0000",166,3947,"\u0012\u0000P\u0003",5472,3682,4,684,1368,1,1,false,
"ext:6:base64:AAAAAAABAAAAAAAAAAAAAA==",[17,{"b2nd":107},
["lgACktMAAAAAAAABkNMAAAAAAAAAA5LSAAAAbtIAAAADktIAAAA50gAAAAPbAAAABXVpbnQ4"]]]
Here we see that we have a `b2nd` metalayer that starts at position 107; but as there is a msgpack `bin32` there, we
must add 5 bytes (4 bytes for an int32 and 1 byte for the msgpack `bin32` header), so the actual starting position is
112 (107 + 5). Also, although we don't know the length of the `b2nd` metalayer, it is typically less than 100 bytes,
so let's err on the safe side and dump the first 1000 bytes, just in case::
$ dd bs=1 skip=112 count=1000 < plugins/test_data/example_day_month_temp.b2nd | msgpack2json -B
[0,2,[400,3],[110,3],[57,3],0,"|u1"]
By having a look at the
`Blosc2 NDim metalayer format `_
one may note that the number of dimensions is 2, `shape` is [400, 3], `chunkshape` is [110, 3], blockshape is
[57, 3], dtype format is 0 (NumPy) and dtype is ``|u1``, which is a NumPy shortcut for `np.uint8`.
Chunks
------
The chunks section is composed of one or more Blosc data chunks followed by an index chunk::
+========+========+========+========+===========+
| chunk0 | chunk1 | ... | chunkN | chunk idx |
+========+========+========+========+===========+
Each chunk is stored contiguously one after the other, and each follows the format described in the
`chunk format `_ document.
The `chunk idx` is a Blosc2 chunk containing the offsets (starting from the beginning of the header)
to each chunk in this section. The data in the chunk is a list of offsets (they can be 32-bit, 64-bit
or more, see above; currently only 64-bit are implemented) to each chunk. The index chunk follows the
regular Blosc2 chunk format and can be compressed (the default).
**Note:** The offsets can take *special values* so as to represent chunks with run-length (equal) values.
The codification for the offsets is as follows::
+========+========+========+========+
| byte 0 | byte 1 | ... | byte N |
+========+========+========+========+
^
|
+--> Byte for special values
If the most significant bit (7) of the most significant byte above (byte N, as little endian is used) is set,
that represents a chunk with a run-length of special values.
More specifically the **byte for special values** has this format:
:bits 0, 1 and 2:
Indicate special values for the entire chunk.
:``0``:
Reserved.
:``1``:
A run of zeros.
:``2``:
A run of NaN (Not-a-Number) floats (whether f32 or f64 depends on typesize).
:``3``:
Reserved.
:``4``:
Values that are not initialized.
:``5``:
Reserved.
:``6``:
Reserved.
:``7``:
Reserved.
:bit 3 (``0x08``):
Reserved.
:bit 4 (``0x10``):
Reserved.
:bit 5 (``0x20``):
Reserved.
:bit 6 (``0x40``):
Reserved.
:bit 7 (``0x80``):
Indicates a special value. If not set, a regular value.
Trailer
-------
The trailer for the frame is encoded via `msgpack `_ and contains a user meta data chunk and
a fingerprint.::
|-0-|-1-|================|---|---------------|---|---|---------------|
| 9X| aX| vlmetalayers | ce| trailer_len | d8|fpt| fingerprint |
|---|---|================|---|---------------|---|---|---------------|
^ ^ ^ ^ ^ ^ ^ ^
| | | | | | | +-- fingerprint type
| | | | | | +--[msgpack] fixext 16
| | | | | +-- trailer length
| | | | +--[msgpack] uint32 for trailer length
| | | +--Variable-length metalayers (See header metalayers)
| | +---[msgpack] bin32 for vlmetalayers
| +------[msgpack] int8 for trailer version
+---[msgpack] fixarray with X=4 elements
The *vlmetalayers* object which stores the variable-length user meta data can change in size during the lifetime of the frame.
This is an important feature and the reason why the *vlmetalayers* are stored in the trailer and not in the header.
However, the *vlmetalayers* follows the same format as the ones stored in the header.
:trailer_len:
(``uint32``) Size of the trailer of the frame (including vlmetalayers chunk).
:fpt:
(``int8``) Fingerprint type: 0 -> no fp; 1 -> 32-bit; 2 -> 64-bit; 3 -> 128-bit
:fingerprint:
(``uint128``) Fix storage space for the fingerprint (16 bytes), padded to the left.
Sidecar lock file
-----------------
When the optional file locking is enabled for an on-disk cframe (via the `locking` member of `blosc2_stdio_params`, passed in the `params` member of the `blosc2_io` struct, or globally via the `BLOSC_LOCKING` environment variable), a small sidecar file appears next to the frame file, with the same name plus a `.b2lock` suffix. It is used to serialize accesses from several handles (typically in different processes) to the same cframe: readers share the lock, mutating operations take it exclusively, and it also carries a change counter so that open handles detect mutations made through other handles. See the `file-locking.c example `_ for usage.
The sidecar is *not* part of the cframe format: it carries no frame data, it is safe to delete whenever no process has the cframe open, and `blosc2_remove_urlpath()` removes it together with the frame file. Note that the locking is advisory (it only protects the cframe if every handle enables it) and that it is not supported on network filesystems (NFS).
Blosc-c-blosc2-d3bee7c/README_CHUNK_FORMAT.rst 0000664 0000000 0000000 00000025172 15255167342 0020444 0 ustar 00root root 0000000 0000000 Blosc/Blosc2 Chunk Format
=========================
A regular chunk is composed of a header and a blocks section::
+---------+--------+
| header | blocks |
+---------+--------+
Also, there are the so-called lazy chunks that do not have the actual compressed data,
but only meta-information about how to read it. Lazy chunks typically appear when reading
data from persistent media. A lazy chunk has header and bstarts sections in place and
in addition, an additional trailer for allowing to read the data blocks::
+---------+---------+---------+
| header | bstarts | trailer |
+---------+---------+---------+
All these sections are described below. Note that the bstarts section is described as
part of the blocks section.
*Note:* All integer types in this document are stored in little endian.
Header
------
Blosc (as of version 1.0.0) has the following 16 byte header that stores
information about the compressed chunk::
|-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-|
^ ^ ^ ^ | nbytes | blocksize | cbytes |
| | | |
| | | +--typesize
| | +------flags
| +----------versionlz
+--------------version
Starting in Blosc 2.0.0, there is an extension of the header above that allows
for encoding blocks with a filter pipeline::
1+|-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-|
| filters | ^ | ^ | filters_meta | ^ | ^ |
| | | |
| +- compcode_meta | +-blosc2_flags
+- user-defined codec +-blosc2_flags2
:version:
(``uint8``) Blosc format version.
:versionlz:
(``uint8``) Version of the *format* of the internal compressor used (normally always 1).
:flags and compressor enumeration:
(``bitfield``) The flags of the buffer
:bit 0 (``0x01``):
Whether the byte-shuffle filter has been applied or not.
:bit 1 (``0x02``):
Whether the internal buffer is a pure memcpy or not.
:bit 2 (``0x04``):
Whether the bit-shuffle filter has been applied or not.
:bit 3 (``0x08``):
Whether the delta codec has been applied or not.
:bit 4 (``0x10``):
If set, blocks are *not* split into multiple compressed data streams.
:bit 5 (``0x20``):
Part of the enumeration for compressors.
:bit 6 (``0x40``):
Part of the enumeration for compressors.
:bit 7 (``0x80``):
Part of the enumeration for compressors.
Note:: If both bit 0 and bit 2 are both set, that means that an
extended header (see above) is used.
The last three bits form an enumeration that allows for the use of alternative compressors.
:``0``:
``blosclz``
:``1``:
``lz4`` or ``lz4hc``
:``2``:
reserved (slot previously occupied by ``snappy`` and free now)
:``3``:
``zlib``
:``4``:
``zstd``
:``5``:
Reserved
:``6``:
The compressor is defined in the user-defined codec slot (see below).
:``7``:
The compressor is defined in the super-chunk.
:typesize:
(``uint8``) Number of bytes for the atomic type.
:nbytes:
(``int32``) Uncompressed size of the buffer (this header is not included).
:blocksize:
(``int32``) Size of internal blocks for regular chunks.
When `blosc2_flags2` bit 0 is set, this field stores the number of blocks in the chunk instead.
:cbytes:
(``int32``) Compressed size of the buffer (including this header).
:filters:
(``uint8``) Filter ID.
:``0``:
No shuffle (for compatibility with Blosc1).
:``0``:
No filter.
:``1``:
Byte-wise shuffle.
:``2``:
Bit-wise shuffle.
:``3``:
Delta filter.
:``4``:
Truncate precision filter.
:``5``:
Sentinel. IDs larger than this are either global registered or user-defined filters.
The filter pipeline has 6 reserved slots for the filters IDs. They are applied sequentially
to the chunk according to their index (in increasing order). The type of filter applied is
specified by the ID. Each ID has an associated field in `filters_meta` that can contain metadata
about the filter.
:udcodec:
(``uint8``) User-defined codec identifier.
:compcode_meta:
(``uint8``) Compression codec metadata.
Metadata associated with the compression codec.
:filters_meta:
(``uint8``) Filter metadata associated to each filter ID.
Metadata associated with the filter ID.
:blosc2_flags:
(``bitfield``) The flags for a Blosc2 buffer.
:bit 0 (``0x01``):
Whether the codec uses dictionaries or not.
:bit 1 (``0x02``):
Whether the header is extended with +32 bytes coming right after this byte.
:bit 2 (``0x04``):
Whether the codec is stored in a byte previous to this compressed buffer
or it is in the global `flags` for chunk.
:bit 3 (``0x08``):
Whether the chunk is 'lazy' or not.
:bits 4, 5 and 6:
Indicate special values for the entire chunk.
:``0``:
No special values.
:``1``:
A run of zeros.
:``2``:
A run of NaN (Not-a-Number) floats (whether f32 or f64 depends on typesize).
:``3``:
Run-length of a value that follows the header (i.e. no blocks section).
:``4``:
Values that are not initialized.
:``5``:
Reserved.
:``6``:
Reserved.
:``7``:
Reserved.
:bit 7 (``0x80``):
Indicate whether codec has been instrumented or not.
:blosc2_flags2:
(``bitfield``) Secondary flags for a Blosc2 buffer.
:bit 0 (``0x01``):
Whether the chunk uses variable-length blocks or not.
:bits 1 to 7:
Reserved.
Blocks
------
The blocks section is composed of a list of offsets to the start of each block, an optional dictionary to aid in
compression, and finally a list of compressed data streams::
+=========+======+=========+
| bstarts | dict | streams |
+=========+======+=========+
For regular chunks, each block is equal-sized as specified by the `blocksize` header field. The size of the last
block can be shorter or equal to the rest.
When `blosc2_flags2` bit 0 is set, the chunk uses variable-length blocks instead:
- `blocksize` in the header stores the number of blocks
- each block still has one entry in `bstarts`
- each block is stored in a single compressed stream
**Block starts**
The *block starts* section contains a list of offsets `int32 bstarts` that indicate where each block starts in the
chunk. These offsets are relative to the start of the chunk and point to the start of one or more compressed
data streams containing the contents of the block::
+=========+=========+========+=========+
| bstart0 | bstart1 | ... | bstartN |
+=========+=========+========+=========+
**Dictionary (optional)**
*Only for C-Blosc2*
Dictionaries are small datasets that are known to be repeated a lot and can help to compress data in blocks better.
The dictionary section contains the size of the dictionary `int32_t dsize` followed by the dictionary data::
+=======+=================+
| dsize | dictionary data |
+=======+=================+
**Compressed Data Streams**
Compressed data streams are the compressed set of bytes that are passed to codecs for decompression. For regular
chunks, each compressed data stream (`uint8_t* cdata`) is stored with the size of the stream (`int32_t csize`)
preceding it::
+=======+=======+
| csize | cdata |
+=======+=======+
There are a couple of special cases for `int32_t csize`. If zero, that means that the stream is fully made of zeros, *and* there is not a `cdata` section. The actual size of the stream is inferred from `blocksize` and whether or not the block is split.
If negative, the stream is stored like this::
+=======+=======+=======+
| csize | token | cdata |
+=======+=======+=======+
where **token** is a byte for providing different meanings to `int32_t csize`:
:bit 0:
Repeated byte (stream is a run-length of bytes). This byte, representing the repeated value in the stream, is encoded in the LSB of the `int32_t csize`. In this case there is not a `cdata` section. Note that repeated zeros cannot happen here (already handled by the `csize == 0` case above).
:bits 1 and 2:
Reserved for two-codecs in a row. TODO: complete description
:bits 3, 4 and 5:
Reserved for secondary codec. TODO: complete description
:bits 6 and 7:
Reserved for future use.
If bit 4 of the `flags` header field is set, each block is stored in a single data stream::
+=========+
| stream0 |
+=========+
| block0 |
+=========+
If bit 4 of the `flags` header is *not* set, each block can be stored using multiple data streams::
+=========+=========+=========+=========+
| stream0 | stream1 | ... | streamN |
+=========+=========+=========+=========+
| block0 |
+=========+=========+=========+=========+
The uncompressed size for each block is equivalent to the `blocksize` field in the header, with the exception
of the last block which may be equal to or less than the `blocksize`.
For variable-length-block chunks (`blosc2_flags2` bit 0 set), each block is always stored in a single stream::
+=========+
| stream0 |
+=========+
| block0 |
+=========+
In this variant:
- `csize` stores the uncompressed size of the block
- the compressed size is derived from adjacent entries in `bstarts` and the end of the chunk
- the special `csize == 0` and `csize < 0` encodings are not used
Trailer
-------
This is an optional section, mainly for lazy chunks use. A lazy chunk is similar to a regular one, except that
only the meta-information has been loaded. The actual data from blocks is 'lazily' only loaded on demand.
This allows for improved selectivity, and hence less input bandwidth demands, during partial chunk reads
(e.g. `blosc1_getitem`) from data that is on disk. Both regular chunks and variable-length-block
(VL-blocks) chunks support lazy loading; the trailer format is identical for both.
It is arranged like this::
+=========+=========+========+========+=========+
| nchunk | offset | bsize0 | ... | bsizeN |
+=========+=========+========+========+=========+
:nchunk:
(``int32_t``) The number of the chunk in the super-chunk.
:offset:
(``int64_t``) The offset of the chunk in the frame (contiguous super-chunk).
:bsize0 .. bsizeN:
(``int32_t``) The sizes in bytes for every block.
Blosc-c-blosc2-d3bee7c/README_EXTENSION_FILENAMES.rst 0000664 0000000 0000000 00000001174 15255167342 0021457 0 ustar 00root root 0000000 0000000 Extensions for Blosc2 Filenames
===============================
Blosc2 has some recommendations for different file extensions for different purposes. Here is a list of the currently supported ones:
- `.b2frame` (but also `.b2f` or `.b2`) (Blosc2 Frame): this is the main extension for storing `Blosc2 Contiguous Frames `_.
- `.b2nd` (Blosc2 N-Dim): this is just a contiguous frame file with `a metalayer for storing n-dimensional information `_ like shape, chunkshape, blockshape and dtype.
Blosc-c-blosc2-d3bee7c/README_FORMAT.rst 0000664 0000000 0000000 00000002517 15255167342 0017512 0 ustar 00root root 0000000 0000000 Blosc2 Format
=============
The Blosc2 format is a specification for storing compressed data in a way that is simple to read and parse,
and that allows for fast random access to the compressed data. The format is designed to be used with
the Blosc2 library, but it is not tied to it, and can be used independently. Emphasis has been put on
simplicity and robustness, so that the format can be used in a wide range of applications.
See a diagram of a Contiguous Frame (aka CFrame), the most important part of the Blosc2 format below:
.. image:: blosc2-cframe.png
:width: 25%
:alt: Blosc2 CFrame format diagram
And here, the list of the different parts of the format, from the highest level to the lowest:
- `B2ND format `_
- `B2ND metalayer `_
- `SFrame format `_
- `CFrame format `_
- `Chunk format `_
Finally, the recommended extension file names for the different parts of the format:
- `Blosc2 extension file names `_
Blosc-c-blosc2-d3bee7c/README_FUZZER.md 0000664 0000000 0000000 00000001131 15255167342 0017326 0 ustar 00root root 0000000 0000000 How to look into fuzzer issues
==============================
* Look into the output of the test case in the logs in CI (look for something like `Test unit written to ./crash-85283a6341337b0ede4bf3b9b2657dcce83ee0b4\nBase64: AiACDQEAAAAPAAAAFAAAABMAbZ0=\n".`). Use the blosc.oss.fuzz@gmail.com account so as to access it.
* This output always gives the base64 of the data that reproduced it (e.g. AiACDQEAAAAPAAAAFAAAABMAbZ0=). This can be converted to hex and saved to a file.
* This file can be passed as an argument to the corresponding test of `tests/fuzz/` folder, so as to test it locally.
Blosc-c-blosc2-d3bee7c/README_SFRAME_FORMAT.rst 0000664 0000000 0000000 00000012157 15255167342 0020550 0 ustar 00root root 0000000 0000000 Blosc2 Sparse Frame Format
==========================
Blosc (as of version 2.0.0) has a Sparse Frame (SFrame for short) format that allows for non-contiguous storage of `Blosc2 data chunks `_ on disk.
When creating a sparse frame one must denote the `contiguous` flag in `storage` struct as false and provide a name (which represents a directory, but in the future it could be an arbitrary URL) in `storage.urlpath` for the sframe to be stored. It is recommended to name the directory with the `.b2frame` (or `.b2f` for short) extension.
A SFrame is made up of a frame index file and the chunks stored in the same directory on-disk. The frame index file follows the format described in the `contiguous frame format `_ document, with the difference that the frame's chunks section is made up of multiple files (one per chunk). The frame index file name is always `chunks.b2frame`, and it also contains the metadata for the sframe.
Chunks
------
The chunks are stored in the directory as binary files. Each chunk file name will be composed of the index of the chunk in hexadecimal written in capital letters with a length of 8 characters padded with zeros with the `.chunk` extension. As an example, 15 chunks could be named as follows::
00000000.chunk, 00000001.chunk, ··· , 0000000E.chunk, 0000000F.chunk
Each chunk follows the format described in the `chunk format `_ document.
*Note:* The real order of the chunks is in the index chunk and may not follow the order of the names. This can occur when doing an insertion or a reorder. For more information see the **Examples** section below.
Sidecar lock file
-----------------
When the optional file locking is enabled (via the `locking` member of `blosc2_stdio_params`, passed in the `params` member of the `blosc2_io` struct, or globally via the `BLOSC_LOCKING` environment variable), a small sidecar file named `.b2lock` appears inside the sframe directory. It is used to serialize accesses from several handles (typically in different processes) to the same sframe: readers share the lock, mutating operations take it exclusively, and it also carries a change counter so that open handles detect mutations made through other handles. See the `file-locking.c example `_ for usage.
The sidecar is *not* part of the sframe format: it carries no frame data, it is safe to delete whenever no process has the sframe open, and it is removed together with the sframe directory. Note that the locking is advisory (it only protects the sframe if every handle enables it) and that it is not supported on network filesystems (NFS).
Examples
--------
Structure example
^^^^^^^^^^^^^^^^^
As shown below, an sframe of 4 chunks will be composed of a directory with each chunk file and the frame file::
dir.b2frame/
│
├── 00000000.chunk
│
├── 00000001.chunk
│
├── 00000002.chunk
│
├── 00000003.chunk
│
└── chunks.b2frame
Insertion example
^^^^^^^^^^^^^^^^^
When doing an insertion in the nth position, in the same position of the index chunk will be the real chunk index which will be the numbers of chunks that there were before inserting the new one. Following the previous example, the content of the directory and the index chunk is shown before and after an insertion in the 2nd position::
Before After
dir.b2frame/ dir.b2frame/
│ │
├── 00000000.chunk ├── 00000000.chunk
│ │
├── 00000001.chunk ├── 00000001.chunk
│ │
├── 00000002.chunk ├── 00000002.chunk
│ │
├── 00000003.chunk ├── 00000003.chunk
│ │
└── chunks.b2frame ├── 00000004.chunk
│
└── chunks.b2frame
Possible index New index
chunk content: [0, 1, 2, 3] chunk content: [0, 1, 4, 2, 3]
Note that neither the file names nor their contents change, so when accessing the 2nd chunk the `00000004.chunk` file will be read.
Reorder example
^^^^^^^^^^^^^^^
As in the insertion case, when doing a reorder the chunks names and their contents are not changed, but the content of the index chunk does. When reordering the chunks, a new order list is passed and the index chunk is changed according to that list. Following with the first example of this section, the content of the index chunk is shown before and after reordering::
Before After
Possible index New index
chunk content: [0, 1, 2, 3] chunk content: [3, 1, 0, 2]
New order list: [3, 1, 0, 2]
Blosc-c-blosc2-d3bee7c/README_THREADED.rst 0000664 0000000 0000000 00000006052 15255167342 0017700 0 ustar 00root root 0000000 0000000 Blosc supports threading
========================
Threads are the most efficient way to program parallel code for
multi-core processors, but also the more difficult to program well.
Also, they have a non-negligible start-up time that does not fit well
with a high-performance compressor as Blosc tries to be.
In order to reduce the overhead of threads as much as possible, I've
decided to implement a pool of threads (the workers) that are waiting
for the main process (the master) to send them jobs (basically,
compressing and decompressing small blocks of the initial buffer).
Despite this and many other internal optimizations in the threaded
code, it does not work faster than the serial version for buffer sizes
around 64/128 KB or less. This is for Intel Quad Core2 (Q8400 @ 2.66
GHz) / Linux (openSUSE 11.2, 64 bit), but your mileage may vary (and
will vary!) for other processors / operating systems.
In contrast, for buffers larger than 64/128 KB, the threaded version
starts to perform significantly better, being the sweet point at 1 MB
(again, this is with my setup). For larger buffer sizes than 1 MB,
the threaded code slows down again, but it is probably due to a cache
size issue and besides, it is still considerably faster than serial
code.
This is why Blosc falls back to use the serial version for such a
'small' buffers. So, you don't have to worry too much about deciding
whether you should set the number of threads to 1 (serial) or more
(parallel). Just set it to the number of cores in your processor and
your are done!
Francesc Alted
Pluggable Threading Backend
---------------------------
Instead of having Blosc use its *own* thread pool, you can instead call `blosc2_set_threads_callback(threads_callback, callback_data)` to install your own threading backend. This gives Blosc the possibility to use the same threading mechanism as one you are using in the rest of your program (e.g. OpenMP or Intel TBB), sharing the same threads, rather than starting its own threads that compete with yours for the CPU cores.
Here, `threads_callback` is a function of the form:
.. code-block:: c
void threads_callback(void *callback_data, void (*dojob)(void *), int numjobs, size_t jobdata_elsize, void *jobdata)
{
int i;
for (i = 0; i < numjobs; ++i)
dojob(((char *) jobdata) + ((unsigned) i)*jobdata_elsize);
}
that simply calls `dojob` on the given `jobdata` array for `numjobs` elements of size `jobdata_elsize`, returning when all of the `dojob` calls have completed. The key point is that your `threads_callback` routine can execute the `dojob` calls *in parallel* if it wants. For example, if you are using OpenMP your `threads_callback` function might use `#pragma omp parallel for`.
The `blosc2_set_threads_callback` function should be called before any Blosc function (before any Blosc contexts are created), to inhibit Blosc from spawning its own worker threads. In this case, `blosc2_set_nthreads` and similar functions set an upper bound to the `numjobs` that is passed to your `threads_callback` rather than an actual number of threads.
Blosc-c-blosc2-d3bee7c/RELEASE_NOTES.md 0000664 0000000 0000000 00000241161 15255167342 0017325 0 ustar 00root root 0000000 0000000 Release notes for C-Blosc2
==========================
Changes from 3.3.4 to 3.3.5
===========================
This is a bugfix and maintenance release addressing several robustness and
security issues reported via OSS-Fuzz in BloscLZ, ZFP codec plugins, and frame
handling.
* **Fixed integer overflow in BloscLZ match-length accumulation (heap-buffer-overflow)**:
In `blosclz_decompress`, the `MATCH_LONG` decoding loop (`ctrl >> 5 == 7`)
accumulated continuation bytes (`0xFF`) into a signed 32-bit integer (`len`)
without overflow protection. After ~8 MB of `0xFF` bytes, `len` could wrap
past `INT32_MAX` into negative values, bypassing the `len > maxout` check and
causing a severe out-of-bounds heap write in `copy_match`. This is resolved
by checking `len > maxout - code` before the addition, rejecting invalid matches
before any overflow can occur. Reported via OSS-Fuzz (`decompress_chunk_fuzzer`).
* **Fixed heap-buffer-overflow in ZFP decompression paths**:
In `zfp_acc_decompress`, `zfp_prec_decompress`, and `zfp_rate_decompress`,
a ZFP bitstream was opened on the compressed input without checking whether the
buffer was sufficiently sized for the stream reads dictated by codec parameters
and field dimensions. A crafted frame with a large `compcode_meta` and a small
payload could trigger an out-of-bounds read in ZFP's `stream_read_word`. Added
validation of the required bitstream size before decompression to ensure the
input buffer is large enough, returning `BLOSC2_ERROR_FAILURE` otherwise.
Reported via OSS-Fuzz (`decompress_frame_fuzzer`).
* **Fixed division by zero (FPE) in `frame_get_chunk` and `frame_get_lazychunk`**:
When processing special-value chunks, a crafted frame with `chunksize <= 0`
caused a floating-point exception (division by zero) in the `nbytes % chunksize`
calculation for the last chunk. Added validation to ensure `chunksize > 0` before
evaluating the modulo expression, returning `BLOSC2_ERROR_INVALID_HEADER` on
invalid chunk sizes. Reported via OSS-Fuzz (`decompress_frame_fuzzer`).
There are no API or format changes in this release.
Changes from 3.3.3 to 3.3.4
===========================
This release introduces bounded-memory tail compaction for contiguous on-disk
frames and clean file eviction for payload-free chunks in sparse frames.
* **Bounded-buffer tail compaction in contiguous frames**:
When updating a chunk in place within an on-disk contiguous frame (`cframe`),
a change in compressed size requires shifting all subsequent chunks in the
payload tail. Previously, Blosc2 allocated a single heap buffer sized to the
entire remaining tail (`malloc(tail_nbytes)`). For multi-gigabyte or terabyte
frames, updating an early chunk could require gigabytes of RAM, risking
out-of-memory aborts.
Tail compaction now uses an internal bounded-buffer movement helper
(`frame_move_range`) capped at 1 MiB (`FRAME_TAIL_COPY_BUFFER_CAP`), decoupling
RAM consumption from frame size:
- Shifts forward when shrinking (`dst < src`) and backward when expanding
(`dst > src`) to guarantee overlap safety across any displacement.
- Reads each segment fully into owned scratch memory before writing, avoiding
pointer invalidation when memory-mapped files are remapped during expansion.
- Adds checked arithmetic to prevent integer overflow and negative buffer
allocations.
- Consolidates error cleanup paths in `frame_update_chunk()` and
`frame_update_trailer()`, and accounts for `frame->file_offset` on truncation.
* **Sparse frame special chunk file removal**:
Updating a chunk to a payload-free special value (such as ZERO, UNINIT, or NAN)
in a sparse frame (`sframe`) previously truncated the chunk file to 0 bytes
rather than deleting it. Repeated cache eviction and refill cycles could leave
thousands of empty chunk files (`0000000X.chunk`), cluttering directories and
exhausting filesystem inodes.
Updating to a payload-free special value now unlinks and deletes the old
physical chunk file (`sframe_delete_chunk()`), matching the representation of
never-materialized special chunks:
- `BLOSC2_SPECIAL_VALUE` chunks (which store repeated payload data) retain
their physical chunk file.
- Adopts recoverable-orphan ordering: the chunk file is deleted only after
publishing the updated offsets index and trailer to disk.
- Properly tracks the preceding physical chunk ID when logical chunks have been
reordered.
- Custom I/O backends that map file paths are handled cleanly: standard backends
use `remove()` (treating `ENOENT` as idempotent success), while custom backends
use their registered callbacks (`open`, `truncate` to 0, `close`), preventing
unintended file operations on host paths.
- Eviction failures occurring after the metadata update has been committed are
logged as warnings rather than returning an error, preventing committed
updates from being reported as failures.
* **Memory mapping (`mmap`) fixes and external file growth preservation**:
- Memory remapping on non-Linux POSIX systems (macOS, BSD) in
`blosc2_stdio_mmap_write()` now maps cleanly to a new address before
unmapping the old one, avoiding `MAP_FIXED` pitfalls.
- In `blosc2_stdio_mmap_destroy()`, file size truncation respects external
growth. On Windows, file padding to `mapping_size` is only trimmed if the
file has not grown externally, avoiding truncating appends made by other
handles. On POSIX, avoids truncating away external writes or appends.
* **New benchmarks and tests**:
- Added `bench/bench_compact_tail.c` to benchmark tail updates across
different buffer caps and chunk positions (with portable timing support
across POSIX and Windows).
- Added comprehensive test suites in `tests/test_compact_tail_update.c` and
`tests/test_sframe_special_eviction.c`.
There are no API or format changes in this release.
Changes from 3.3.2 to 3.3.3
===========================
This is a bugfix and security release fixing bounds checking issues in BloscLZ.
* **Fixed integer and pointer overflow in BloscLZ decompression bounds checks.**
In `blosclz_decompress`, match length accumulation in the length-extension
loop (`while (code == 255)`) was unbounded, allowing crafted chunks to grow
`len` arbitrarily large.
On 32-bit platforms (and WebAssembly `wasm32`), pointer arithmetic `op + len`
wrapped modulo 2^32 when destination buffers were allocated high in the
virtual address space, bypassing the `op + len > op_limit` guard and leading
to out-of-bounds heap writes. On 64-bit platforms, an excessively large input
could similarly trigger signed `int32_t` overflow.
This has been resolved by:
- Capping match length accumulation inside the extension loop early with
`if (len > maxout) return 0;`.
- Converting all pointer-addition bounds checks (`op + len > op_limit`, literal
copy checks, and compressor macros) into safe subtraction form
(`len > op_limit - op`).
- Adding a regression test in `tests/test_blosclz_bounds.c`.
Thanks to Lylan from QI-ANXIN Technology Research Institute for discovering
and responsibly reporting this vulnerability.
There are no API or format changes in this release.
Changes from 3.3.1 to 3.3.2
===========================
This is a bugfix release, mainly about BYTEDELTA.
The BYTEDELTA filter silently corrupted the tail of any block whose length is
not a multiple of the typesize. It splits a block into `typesize` byte streams
of `length / typesize` bytes each, and the last `length % typesize` bytes fell
outside every stream, so they were never written to the output -- in both the
forward and the backward pass, leaving whatever the destination buffer happened
to hold. Those trailing bytes are now passed through verbatim in both
directions, which is what SHUFFLE already does with the same remainder (and why
SHUFFLE alone roundtrips fine while any pipeline containing BYTEDELTA did not).
Blocks whose length is a multiple of the typesize are unaffected, so existing
data still decodes bit for bit. Data written by the old encoder at a
non-multiple length cannot be recovered: those tail bytes were never encoded in
the first place.
Also, the installed CMake package now exports the configured
`CMAKE_INSTALL_INCLUDEDIR` instead of a hardcoded `include`, so consumers of
installs that use a different include directory get the right path.
There are no API or format changes in this release.
Changes from 3.3.0 to 3.3.1
===========================
This release is about making reads from on-disk frames faster.
Reads from on-disk frames got noticeably faster. The default filesystem I/O
backend now uses positioned reads and writes (`pread`/`pwrite`, `ReadFile`/
`WriteFile` with an explicit offset on Windows) instead of seek + stdio, and a
frame keeps a single read handle open instead of opening and closing the file
several times per chunk fetch. Scattered small reads out of a cframe are about
3x faster in our microbenchmarks; the gain grows with how many small reads a
workload does. User-registered I/O backends are unaffected: they keep their
one-handle-per-reader contract.
Concurrent readers gain the most, since the per-access open/close this removes
was paid by every process and contended in the kernel. Eight processes each
doing 300 random slice reads over the same 269 MB frame went from 0.52 s to
0.36 s of wall time, and from 3.85 s to 2.58 s of CPU (Apple M4 Pro): about
1.4x, against 13% for a single reader. The advantage of opening such files with
`mmap_mode="r"` narrows accordingly -- not because memory mapping got slower,
but because the regular path caught up.
Note that an open on-disk schunk now holds one file descriptor for as long as it
stays open, where before descriptors were only taken for the duration of each
read. To keep that from exhausting the process's descriptor budget, the number
of cached handles is capped at min(96, RLIMIT_NOFILE/16) -- schunks past the cap
fall back to the previous open-per-read behaviour. Set the
**BLOSC_MAX_CACHED_READERS** environment variable to choose a different cap, or
to 0 to switch the cache off entirely. Handle caching is currently POSIX-only:
on Windows the C runtime opens files without FILE_SHARE_DELETE, so a cached
handle would make unlinking or renaming an open frame file fail.
As a side effect of dropping `fseek()`, reads and writes past 2 GB no longer
fail on 32-bit builds whose `off_t` is 64-bit wide.
There are no API or format changes in this release.
Changes from 3.2.3 to 3.3.0
===========================
This release hardens the paths that read data described by untrusted metadata,
and adds a small API for reading a byte range out of a chunk.
Crafted `.b2nd` files could overflow the b2nd shape arithmetic and reach a
division by zero or an undersized allocation, and crafted chunks without an
extended header could be read past their end. Both are fixed. Users opening
b2nd files or chunks they did not produce themselves should upgrade.
Separately, `blosc2_getitem_ctx()` turned out to be easy to misuse for typesizes
above `BLOSC_MAX_TYPESIZE`, where its unit silently becomes the byte rather than
the element; `blosc2_getitem_bytes_ctx()` is the unambiguous counterpart, and
two schunk read paths that got this wrong are fixed.
This is a minor release: `blosc2_getitem_bytes_ctx()` is new, but nothing was
removed or changed incompatibly, so the SOVERSION is unchanged and existing
binaries keep working.
Security fixes
--------------
* **Fixed integer overflows in b2nd shape handling reachable from a crafted
``.b2nd`` file** (#795). ``update_shape_struct()`` accumulated the shape,
chunkshape and blockshape products into ``int32_t`` (``chunknitems``,
``blocknitems``) and ``int64_t`` (``nitems``, ``extnitems``,
``extchunknitems``) fields without any overflow check. Signed overflow is
undefined behaviour, and a wrapped product reached both divisors and
allocation sizes: a metalayer declaring ``blockshape={65536, 65536}`` was
accepted by ``b2nd_open()`` with ``blocknitems == 0``, and the first data
access then divided by zero (SIGFPE on x86-64; on aarch64 the division
silently yields 0 instead). Products that wrapped to a small positive value
could instead drive out-of-bounds offsets.
All the products are now computed with checked multiplications and rejected
with ``BLOSC2_ERROR_INVALID_PARAM``. The same guard covers the creation path,
which shared the overflow, plus a third site in ``b2nd_create_ctx()`` that
wrapped ``cparams->blocksize``.
* **Fixed int64 to int32 truncation of chunk buffer sizes** (#795).
``b2nd_get_slice_cbuffer()``/``b2nd_set_slice_cbuffer()`` and
``orthogonal_selection()`` sized a chunk buffer with a truncating cast of
``extchunknitems * typesize`` while the copy offsets kept using the full
extent, so a shape whose product fits in ``int32_t`` but whose byte count does
not produced a heap buffer overflow. Both sites now compute in 64 bits and
refuse anything above ``INT32_MAX``.
Reported by Akhil Koul.
* **Fixed an out-of-bounds read of the flags2 byte in the schunk append, insert
and update paths** (#792). That byte lives at offset 0x1e, inside the
extended header, but a Blosc1-style chunk is only
``BLOSC_MIN_HEADER_LENGTH`` (16) bytes long and carries no flags2 at all, so
handing such a chunk to ``blosc2_schunk_append_chunk()`` read 15 bytes past
the end of the caller's buffer. All the sites now go through one helper that
looks the size up first.
The helper also decides whether a chunk has a flags2 byte from the flags, the
way ``read_chunk_header()`` does, rather than from the chunk size: a
Blosc1-style chunk with 16 or more bytes of payload is long enough to reach
offset 0x1e, where it holds compressed data. Reading that as flags2 could
mark a schunk as using variable-length blocks out of thin air, and the flag
is persisted into the frame.
Reported and largely fixed by Farkhalit Rida.
New features
------------
* **New ``blosc2_getitem_bytes_ctx()``**, a byte-counting counterpart to
``blosc2_getitem_ctx()``. The unit of the latter is the typesize the
*chunk* records, which is 1 for typesizes above ``BLOSC_MAX_TYPESIZE``
(255), so its meaning silently changes with the typesize; bytes do not.
Prefer it in generic code that does not choose the typesize itself, and
``blosc2_getitem_ctx()`` where the typesize is known and at most 255, as
counting in items reads better there. ``start`` and ``nbytes`` must be
multiples of the typesize stored in the chunk, which is vacuous above the
cap and automatic for callers deriving offsets from the real typesize.
``blosc2_getitem_ctx()`` is unchanged and is not deprecated; its docs now
spell out the unit and point at the new entry point. The three in-tree
call sites (``blosc2_schunk_get_slice_buffer()``,
``blosc2_schunk_get_sparse_buffer()`` and the b2nd compact get path) now
use it, so the typesize-cap rule lives in exactly one place again.
Bug fixes
---------
* **Fixed wrong results in schunk slice and sparse reads for typesize > 255.**
``blosc2_schunk_get_slice_buffer()`` derived its ``blosc2_getitem_ctx()``
item counts by dividing byte offsets by ``schunk->typesize``, but chunks
whose typesize exceeds ``BLOSC_MAX_TYPESIZE`` (255) are compressed with an
internal typesize of 1, so getitem counts bytes for them. Slices that did
not cover a chunk exactly mostly failed with ``BLOSC2_ERROR_FAILURE``, and
single-element slices returned the wrong bytes with a success return code.
The single-coordinate path of ``blosc2_schunk_get_sparse_buffer()`` had the
same confusion. Both now convert through the chunk's actual item unit.
See #796.
* ``blosc2_getitem_ctx()`` now returns ``BLOSC2_ERROR_DATA`` instead of a
short byte count when it cannot decode every requested item, so a partial
decode no longer looks like a success to callers that only test for a
negative return.
* Fixed a memory leak and a dangling non-NULL output pointer in
``array_without_schunk()`` when the shape could not be validated, and a leak
of the deserialized dtype in ``b2nd_from_schunk()`` on the same path. These
were only reachable once the shape checks above made that path possible.
* Fixed the build on FreeBSD and the other BSDs (#794). ``-D_XOPEN_SOURCE=600``
leaves ``__BSD_VISIBLE`` undefined there, which hides ``flock()`` and the
``LOCK_*`` constants behind the guard in ````, so ``frame.c``
failed to compile. It now drops ``_XOPEN_SOURCE`` for that one file; it
cannot be dropped build-wide, because ``my_malloc()`` keys its
``posix_memalign()`` path off it. Reported by Erik Schnetter.
Changes from 3.2.2 to 3.2.3
===========================
Bug fixes
---------
* **Fixed data corruption in b2nd get_slice for typesize > 255.** The
compact get path introduced in 3.2.2 passed item counts to
``blosc2_getitem_ctx()`` in array-typesize units, but chunks whose
typesize exceeds ``BLOSC_MAX_TYPESIZE`` (255) are compressed with an
internal typesize of 1, so small slice reads returned truncated blocks
(only the first ``blocknitems`` bytes were correct). Item counts are
now expressed in the chunk's actual item unit, keeping the fast path
both correct and O(request) for large typesizes. Added a regression
test covering typesize > 255 on the get_slice path.
Notes
-----
* This is a hot-fix release with no API/ABI changes. Users of 3.2.2
reading slices from b2nd arrays with typesize > 255 should upgrade.
Changes from 3.2.1 to 3.2.2
===========================
Performance improvements
------------------------
* **Avoid chunk-sized scratch allocation for small b2nd get_slice
requests.** ``get_set_slice()`` in the b2nd path previously malloc'd a
full extended-chunk-sized scratch buffer on every call, even when the
block maskout meant only one block was decompressed into it. For large
chunks the mmap/munmap page-table work alone is O(chunksize), so reading
a single 1.6 MB block from a 128 MB chunk could cost up to 3x more than
the same read from a small chunk. Now the get path counts the needed
blocks after building the maskout, and when they are a small part of the
chunk, decompresses just those blocks one at a time via
``blosc2_getitem_ctx()`` into a reusable block-sized buffer — small
reads stay O(request) instead of O(chunksize), and on-disk frames only
read the touched blocks. Larger requests keep the existing parallel
block-decompression path unchanged, and the chunk-sized scratch is now
allocated lazily only when that path (or a set operation) actually runs.
Single-block reads from large chunks are now flat ~0.33 ms regardless
of chunk size.
Documentation
-------------
* Broad refresh: annotated every ROADMAP-TO-3.0 item as Done/Ongoing/
Deferred now that 3.0 is out, and added a new ROADMAP-TO-4.0 with the
deferred items as current goals. Fixed several stale references
(python-blosc2 status, NEON bitshuffle, citation year, Twitter→Mastodon,
master→main, header paths), and assorted grammar/markup fixes.
Notes
-----
* This is a maintenance release with no API/ABI changes.
Changes from 3.2.0 to 3.2.1
===========================
* New `blosc2_schunk_refresh()` API: an explicit, data-free re-sync point
for a plain `blosc2_schunk` reader handle in a Growth-SWMR setup, mirroring
`b2nd_refresh()` for b2nd arrays. Useful for polling for another handle's
changes without touching data.
Changes from 3.1.5 to 3.2.0
===========================
New features
------------
* **Opt-in cross-process locking** for disk-based frames, via a sidecar
lock file (``.b2lock``, ``flock``/``LockFileEx``) plus a generation
counter that detects mutations other handles made even when the on-disk
length is unchanged. Enable it per-handle (``blosc2_stdio_params.locking``,
or the equivalent for other I/O backends) or fleet-wide with the
``BLOSC_LOCKING`` environment variable. Advisory: every handle touching
the container must opt in. Not supported over NFS.
* **`blosc2_schunk_lock()` / `blosc2_schunk_unlock()`**: a bracket API to
hold the exclusive frame lock across several operations, making a
multi-step mutation atomic to other locked handles instead of only each
individual call. No-ops on a handle without locking enabled.
`b2nd_resize()` and `b2nd_set_slice_cbuffer()` now bracket their own
multi-chunk sequences this way too, so a locked reader never observes a
half-applied resize or slice write, and two overlapping slice writers no
longer interleave at chunk granularity.
* **Growth-SWMR** (single writer, multiple readers): a reader handle
opened on a disk-based frame or b2nd array now follows shape/length
growth made through another handle (typically another process) on its
next access, without reopening. New public `b2nd_refresh()` reader-side
API, and `examples/b2nd/example_growth_swmr.c` demonstrating the
contract. Works with or without locking; only the on-disk length is
polled without it, so growth is virtually always noticed but a
same-length mutation may not be until the next one that changes it.
* **Stale-handle coherence fixes**: a handle whose cached view of an
on-disk frame goes stale (another handle rewrote it) now re-syncs before
trusting cached state, closing gaps in `blosc2_vlmeta_exists()` and in
the counters (`nbytes`/`cbytes`/`nchunks`) that
`blosc2_schunk_append_chunk()`/`blosc2_schunk_insert_chunk()` apply their
deltas to. Also fixed:
- A race between opening a frame and a concurrent writer growing or
otherwise mutating it (`blosc2_schunk_open_offset_udio()` could
return ``NULL`` on a transient "frame length exceeds file boundary"
read instead of retrying it).
- A torn read during the (unlocked) staleness poll used to hard-error
instead of keeping the cached view and retrying on the next poll, as
intended.
- A leak, and a free of an uninitialized pointer in
`blosc2_schunk_copy()`, on an unreadable-frame staleness poll (hit
legitimately by a freshly created, not-yet-flushed file-backed
schunk).
API/ABI notes
-------------
* Appends `change_tick` to `blosc2_schunk` struct and `last_tick` to
`b2nd_array_t`, and adds `locking` to `blosc2_stdio_params` and the new
`BLOSC2_ERROR_LOCK` error code. No existing field or function signature
was altered, but this alters the ABI as two public structs have an
additional field; the shared library's `SOVERSION` is bumped accordingly
(8 to 9).
Notes
-----
* Advisory locking is not a replacement for a real lock manager: it only
protects a container if every handle on it (across every process)
enables it. It does not work on network filesystems (NFS), and is
incompatible with memory-mapped I/O.
* See ``plans/high-level-formats-locking.md`` and
``plans/todo-locking-swmr.md`` in the source tree for the full design
rationale and known follow-ups.
Changes from 3.1.4 to 3.1.5
===========================
Fixes
-----
* Fix decompression of **all-zeros buffers** whose length is not a multiple
of ``typesize``. ``read_chunk_header()`` previously rejected
``BLOSC2_SPECIAL_ZERO`` chunks whose ``nbytes`` was not a multiple of
``typesize``. However, all-zeros decompression is just a ``memset`` and
works regardless of element alignment, so the check was overly strict for
this case (it remains valid for ``SPECIAL_NAN``, ``SPECIAL_VALUE``, and
``UNINIT``, whose callers do enforce alignment). This affected, for
example, python-blosc2's ``compress2`` → ``decompress2`` round-trip for
all-zeros payloads such as ``bytes(707658)`` with ``typesize=8``
(``707658 % 8 == 2``). A regression test with 9 parametrized cases has
been added. Closes Blosc/python-blosc2#665.
Notes
-----
* This is a maintenance release with no API/ABI changes.
Changes from 3.1.3 to 3.1.4
===========================
Security hardening
------------------
This release extends the block-geometry validation introduced in 3.1.3 to
two more multidimensional plugin filters: **ndcell** and **ndmean**.
* **ndcell**: ``ndcell_forward()`` and ``ndcell_backward()`` now validate
the ``b2nd`` metalayer geometry in 64-bit arithmetic, rejecting non-positive
block dimensions and blockshapes whose product exceeds the buffer size.
This prevents a heap buffer overflow in the decompression path that could
be triggered by a crafted blockshape whose 32-bit product wraps back onto
``length``, passing the old validity check while the scatter-write index
arithmetic used the true (huge) dimensions.
* **ndmean**: Same class of fix applied to ``ndmean_forward()`` and
``ndmean_backward()``. Both now use 64-bit block-size computation with
overflow-rejection and non-positive dimension checks at the earliest
possible point.
Both fixes ship with dedicated regression tests that explicitly craft the
overflowing/wrapping geometries to prevent silent regressions.
Thanks to @metsw24-max and @saddamr3e for all these improvements.
Build / portability
-------------------
* Recognize **riscv64** (and riscv32) targets in CMake. RISC-V builds now
correctly disable SSE/AVX/NEON/ALTIVEC acceleration and fall back to the
generic shuffle path, avoiding miscompilation or silent build failures on
this emerging architecture. Thanks to @carlosqwqqwq.
Fixes
-----
* Fix a **memory leak** in ``create_shared_pool()`` when thread creation
fails partway through building a shared thread pool. Previously, a failed
``pthread_create`` leaked the pool struct, thread and context arrays,
synchronization primitives, and all per-thread scratch buffers initialized
so far. This was most visible under Emscripten/WASM (e.g., Pyodide),
where every compress/decompress with ``nthreads > 1`` leaked the
partially-built pool. The fix adds a single error-unwind path that
properly signals shutdown to already-started workers, joins them, tears
down orphaned contexts, and releases all resources.
* Fix a potential **NULL thread-pool SEGV** when shared-pool creation fails
in ``do_job()``. ``do_job()`` previously ignored ``check_nthreads()``'s
return code, so a failed parallel backend (``context->thread_pool == NULL``)
would still reach ``parallel_blosc()`` and crash on multi-block data.
It now falls back to the serial path when ``rc < 0``. The fallback is
self-healing: the next ``do_job()`` call re-attempts the attach.
Notes
-----
* This is a maintenance release with no API/ABI changes.
Changes from 3.1.2 to 3.1.3
===========================
Security hardening
------------------
* Harden ndlz, zfp, and ndmean plugin codecs/filters against malformed or
attacker-crafted ``b2nd`` metalayers. The block geometry for these
multidimensional plugins is read from the ``b2nd`` metalayer, which is
under user/remote control. Without validation, a crafted metalayer could
trigger heap buffer overflows or out-of-bounds writes during decompression.
* **zfp**: Added ``zfp_check_output_size()`` that validates the b2nd
metadata return value, ndim range, typesize, and performs an
overflow-safe computation of ``prod(blockshape) * typesize`` against the
output buffer size. Non-positive block dimensions are rejected.
Compressors now also check deserialize returns and free buffers on every
early-return path. A regression test covering oversized, zero-dimension,
and int64-overflow blockshapes has been added.
* **ndlz**: Both ``ndlz4x4`` and ``ndlz8x8`` decoders now validate the
``b2nd_deserialize_meta`` return value and properly free
shape/chunkshape/blockshape buffers (and ``bufarea``) on early-return
paths.
* **ndmean**: Validates the deserialize return and ndim range, and sizes
the shape/chunkshape/blockshape buffers for ``B2ND_MAX_DIM``.
* Validate NDLZ decompression input references, preventing potential issues
with untrusted or malformed inputs reaching the ndlz4x4 and ndlz8x8
decompressors.
Thanks to @metsw24-max for all these improvements.
Performance improvements
------------------------
* Use a lazy chunk instead of eagerly reading the whole chunk in the frame
code path. This avoids unnecessary I/O and decompression when only part
of a chunk is needed.
Fixes
-----
* Better handling of the ZFP codec workflow when ``input_len`` is smaller than
the block size in the zfp plugin.
Notes
-----
* This is a maintenance release with no API/ABI changes.
Changes from 3.1.1 to 3.1.2
===========================
Hotfixes
--------
* Fix a regression in ``b2nd_get_orthogonal_selection()`` and
``b2nd_set_orthogonal_selection()`` introduced by the 3.1.1 batching
optimization.
* Add a regression test covering orthogonal get/set selections across outer
dimensions.
Notes
-----
* This is a hot-fix release with no API/ABI changes.
Changes from 3.1.0 to 3.1.1
===========================
Performance improvements
------------------------
* Optimize ``b2nd_get_orthogonal_selection()`` for axis-based row/column
selections by reusing the chunk decompression buffer across visited chunks
and batching consecutive innermost-dimension element copies.
In internal benchmarks, this makes orthogonal selections up to about
2.6x faster (for example, on the ``blosc2.take()`` ndim=2 benchmark with
100M elements, 6388 indices, and ``axis=0``).
Documentation improvements
--------------------------
* Include static-inline functions in the user-facing API docs, so header-only
entry points such as ``b2nd_deserialize_meta_inline()`` show up correctly in
the generated reference manual.
* Add the public functions that were missing from the reference docs,
including recently added sparse getters, VL-block/context helpers,
defaults accessors, and utility functions.
* Add ``doc/check_missing_docs.py`` to help ensure that public APIs are not
accidentally left undocumented in future releases.
Notes
-----
* This is a maintenance release with no API/ABI changes.
Changes from 3.0.3 to 3.1.0
===========================
New features
------------
* New sparse coords getter API for extracting arbitrary sets of
coordinates in a single call, much faster than repeated individual
`getitem` operations:
- ``blosc2_schunk_get_sparse_buffer()`` extracts a set of flat
(1-dimensional) coordinates from a schunk into a caller-provided
buffer. It batches coordinates by chunk internally to minimize
decompression overhead.
- ``b2nd_get_sparse_cbuffer()`` does the same for multidimensional
(b2nd) arrays, accepting an array of *n*-dimensional coordinates and
returning the corresponding values into a C buffer.
Both APIs support regular and special-value (zero/NaN/run-length)
schunks, as well as frame-backed and callback-threaded schunks.
* ``b2nd_deserialize_meta_inline()`` — a new static-inline version of
``b2nd_deserialize_meta()`` available from the header ``b2nd.h`` without
linking against ``libblosc2``. Together with the already-inline
``blosc2_meta_get()``, this lets external codec/filter plugins (like
``blosc2_grok``) use the b2nd metadata API without pulling all of
``libblosc2``'s internal symbols (ZFP, Zstd, …) into the global
namespace, which could otherwise shadow symbols from other libraries
that need differently-configured builds of the same dependencies.
The existing ``b2nd_deserialize_meta()`` ABI entry point is preserved
as a thin wrapper for backward compatibility.
* Globally registered codec IDs **J2K** (124) and **HTJ2K** (125) for
upcoming JPEG 2000 / High-Throughput JPEG 2000 plugins.
Thanks to @alemirone.
Fixes
-----
* Fix ``swap_store()`` for big-endian machines.
* Fix divide-by-zero in ``b2nd_update_shape``. Thanks to @metsw24-max.
* Fix trailer vlmetalayer-parsing and a NULL-check on a missing allocation.
* Rename ``blosc2_schunk_get_sparse`` → ``blosc2_schunk_get_sparse_buffer``
in the test file and error messages for consistency with the public header.
Thanks to @metsw24-max.
Changes from 3.0.2 to 3.0.3
===========================
* Harden frame parsing against malformed trailer lengths, invalid metalayer
offsets and lengths, and lazy-chunk compressed sizes that extend past the
end of a frame. This prevents invalid offset computation,
out-of-bounds reads, and a double-free on malformed inputs. Thanks to
@metsw24-max.
* Fix integer-overflow paths in b2nd chunk-size and slice-buffer
calculations that could otherwise lead to heap corruption or invalid
allocations. Thanks to @metsw24-max.
* Improve stdio and mmap I/O safety by validating NULL streams, file sizes,
offsets, and integer conversions, and by tightening cleanup/error paths.
Thanks to @metsw24-max.
* Replace unsafe string formatting/concatenation in compressor listing,
codec-version reporting, directory helpers, and frame path handling with
bounded operations and allocation checks. Thanks to @metsw24-max.
* Validate negative indices in offset reordering to reject malformed offset
tables more reliably. Thanks to @uwezkhan.
* Add regression tests covering malformed frame trailer/metalayer data,
lazy-chunk bounds, b2nd chunk-size overflows, stdio validation, mmap error
handling, and negative reorder offsets.
Changes from 3.0.1 to 3.0.2
===========================
* Fix for windows when using ctx API from multiple threads. Closes #763.
Thanks to Christoph Gohlke (@cgohlke).
* Harden metalayer APIs against invalid lengths and unsafe memory usage.
PR #758. Thanks to @metsw24-max.
* Fix DELTA pipelines after byte-transforming filters (e.g. shuffle).
Changes from 3.0.0 to 3.0.1
===========================
* Add BLOSC_DEPENDENCY_MODE to choose BUNDLED, EXTERNAL, or AUTO
dependency resolution, defaulting to BUNDLED for compatibility.
* Add BLOSC_ENABLE_ZFP to make ZFP optional in external builds while
still allowing users to require or disable it explicitly.
* Add external ZFP discovery and improve LZ4/Zstd find modules so
external builds provide proper imported targets and include dirs.
* Avoid dependency CMake variables polluting the Blosc2 package install
dir, and install Blosc2 config files under the expected Blosc2 path.
* Document the new options for distro packagers.
Changes from 3.0.0-rc2 to 3.0.0
===============================
* Fixes an integer overflow in VL-block decompression where cumulative
blocknbytes was tracked in a 32-bit integer. PR #753. Thanks to @metsw24-max.
* Fix data races in global configuration APIs using mutex protection. PR #753.
Thanks to @metsw24-max.
* Different typos fixed. Thanks to @DimitriPapadopoulos.
Changes from 3.0.0-rc1 to 3.0.0-rc2
===================================
* `blosc2_get_slice_nchunks()`, `schunk_get_slice_nchunks()`, and
`b2nd_get_slice_nchunks()` now return `int64_t` instead of `int`.
This removes an artificial `INT_MAX` limit on the number of chunks
reported for large slices. Because these public signatures changed,
this is an API/ABI break and callers should be rebuilt against 3.0.0.
* The internal parallel execution model has been reworked around a shared
managed thread pool. Instead of owning a private worker pool per
compression/decompression context, contexts now attach lazily to a pool
shared by other contexts with the same `nthreads` setting, while
caller-managed callback threading remains supported as before. This greatly
reduces redundant thread creation and idle thread accumulation when many
contexts coexist, improves scalability and resource usage, and fixes
reliability problems in downstream workloads such as python-blosc2 that
create large numbers of arrays/contexts over time. The new queue-based
scheduling model also supports true concurrent submissions to the same pool.
* Landed a broad set of robustness and security hardening fixes in frame,
schunk, lazy-chunk, metadata, mmap, and getitem paths. These changes tighten
bounds checking, reject malformed headers/offset tables/VL-block metadata more
aggressively, prevent integer overflows and out-of-bounds reads/writes, and
add regression tests for malformed inputs and edge cases. Thanks to
@metsw24-max for many of these.
* Modernized codec dependency handling in CMake. `lz4`, `zlib-ng`, and
`zstd` are now resolved either from external packages (when preferred and
available) or via `FetchContent` using pinned upstream versions, instead of
being built from vendored in-tree copies. The optional ZFP plugin is now
also obtained via `FetchContent` too. As a result, `blosclz` is now the only
codec still vendored in-tree.
* Added explicit CMake cache variables for pinned codec versions and local
source overrides:
`BLOSC_LZ4_VERSION`, `BLOSC_ZLIBNG_VERSION`, `BLOSC_ZSTD_VERSION`, and the
matching `BLOSC_*_SOURCE_DIR` variables.
* Improved CMake install/export support for static builds so downstream
`find_package(Blosc2)` consumers keep working when fetched codec libraries
are embedded into the Blosc package.
* Embedded third-party headers installed by the CMake package are now placed
under a Blosc-owned include subtree (`blosc2/thirdparty/...`) instead of the
top-level include directory, reducing the risk of header name collisions.
* Replaced deprecated `exec_program()` usage in `cmake/FindSIMD.cmake` with
`execute_process()`, avoiding warnings with newer CMake versions.
* Removed the unmaintained Intel IPP integration. C-Blosc2 no longer exposes
a CMake option to enable/disable IPP, does not probe for IPP at configure
time, and always uses the maintained native codec paths instead.
Changes from 2.23.1 to 3.0.0-rc1
================================
* This release introduces support for variable-length chunks and variable-length
blocks, which is the main reason for the major version bump.
Until now, a schunk/frame generally assumed that all chunks shared the same
logical chunk size, and regular Blosc2 chunks assumed fixed-size internal blocks
(except for the last remainder block). In 3.0.0-rc.1, schunks can switch to
variable chunk sizes when needed, and there is also a new chunk layout for
variable-length blocks (VL-blocks), where each block can carry a different
uncompressed size inside the same chunk.
This is especially useful for workloads made of naturally variable-size pieces
of data, like strings, records, JSON fragments, or other irregular payloads
that previously had to be padded, split awkwardly, or stored as independent
chunks. The new layout keeps these pieces grouped together while still making
them individually recoverable.
Together with this, there are new public APIs for VL-block chunks:
`blosc2_vlcompress_ctx()`, `blosc2_vldecompress_ctx()`,
`blosc2_vlchunk_get_nblocks()`, `blosc2_vldecompress_block_ctx()`, and
`blosc2_schunk_get_vlblock()`. Lazy loading also works with VL-block chunks,
so individual blocks can be fetched on demand without materializing the whole
chunk first.
* The chunk and cframe formats have been extended to represent variable chunk
sizes, VL-block chunks, and dictionary usage more explicitly. Forward
compatibility checks were tightened as part of this work, and regular chunks
keep their previous stable format version while VL-block chunks use a new one.
* Dictionary compression has been expanded and improved:
`use_dict` now works with LZ4 and LZ4HC in addition to ZSTD, the dictionary
state is preserved correctly across chunk compression/decompression, and the
frame metadata now round-trips the dictionary setting. There is also a new
minimum useful dictionary threshold to avoid training or using dictionaries
that are too small to help.
* The necessary changes for accommodating all these improvements have been fully
documented in README_CHUNK_FORMAT.md and README_CFRAME_FORMAT.md. Again,
care has been taken to ensure that the chunk and frame formats are backward
compatible with previous versions of C-Blosc2.
* Fixed several safety issues in the core library, as well as in ndcell/ndlz
plugins. Thanks to Trail of Bits (in collaboration with Anthropic).
* Additional compatibility and portability fixes include better protection
against unsupported future chunk/frame versions and proper VSX shuffle support
detection on big-endian ppc64. Thanks to @AutoJanitor for these.
Changes from 2.23.0 to 2.23.1
=============================
* Fix some memory leaks/undefined behaviour
* Allow resizing of empty arrays
* `filters_meta` (interpreted as number of bytestreams) for the byteshuffle filter
Changes from 2.22.0 to 2.23.0
=============================
* Changes to allow use of miniexpr. This breaks the ABI as a public struct has an additional field.
* Changes to enable blosc2-openzl plugin
Changes from 2.21.3 to 2.22.0
=============================
* `b2nd_squeeze` and `b2nd_squeeze_index` now return views and their signatures have consequently changed.
* bug fix for indexing `[ndim - 1]` which caused an error when `ndim` is 0
Changes from 2.21.2 to 2.21.3
=============================
* Increase MAX_DIMS from 8 to 16
* Fix compatibility with glibc v2.42
* Bug fix in ``unidim_to_multidim``
Changes from 2.21.1 to 2.21.2
=============================
* Allocate strides array on stack if ndim <= B2ND_MAX_DIM in unidim_to_multidim.
Should allow to speed up code. Thanks to Barak Ugav.
* Handle frame_to_schunk failure in schunk_open. Thanks to Barak Ugav.
* Fix fastpath decision logic for ndarray slicing. Thanks to Preeyan Parmar.
Changes from 2.21.0 to 2.21.1
=============================
* Fixed a regression introduced in PR #682 causing incompatibility
at least with Fedora and Gentoo. Fixed in PR #690. Thanks to
Michał Górny (@mgorny) for reporting and Teodor Tsirpanis
(@teo-tsirpanis) for the actual fix.
* Restrict overmatching MACH ifdef to only trigger on OSX and Mach.
Thanks to Maximilian Downey Twiss (@Zopolis4) for the PR (#688).
Changes from 2.20.0 to 2.21.0
=============================
* New public API `b2nd_save_append`. It is useful for appending a b2nd
array into a file. Thanks to Barak Ugav (@barakugav) for the PR (#683).
* Added new public APIs for the shuffling filters:
* `blosc2_shuffle`
* `blosc2_unshuffle`
* `blosc2_bitshuffle`
* `blosc2_bitunshuffle`
Thanks to Theodore Tsirpanis (@teo-tsirpanis) for the PR (#681).
* Fix a potential nullptr exception on `b2nd_create_ctx` with null storage.
Thanks to Barak Ugav (@barakugav) for the PR (#684).
* The CMake dependency acquisition has been improved.
Thanks to Theodore Tsirpanis (@teo-tsirpanis) for the PR (#682).
* Now, a blosc/config.h is generated in the binary directory. Fixes
failures when configuring from the same source directory in parallel.
Thanks to Theodore Tsirpanis (@teo-tsirpanis) for the PR (#679).
Changes from 2.19.1 to 2.20.0
=============================
* Extended `b2nd_expand_dims()` to allow for list of axes like NumPy.
See PR #676. This represents a breaking change for the API, but as
it was introduced in 2.18.0, it is unlikely to be used in practice.
Thanks to Luke Shaw (@lshaw8317).
* Fix detection of AVX512 in MSVC. See PR #678.
Thanks to Theodore Tsirpanis (@teo-tsirpanis).
* Add unified Blosc2::blosc2 CMake target. See PR #677.
Thanks to Theodore Tsirpanis (@teo-tsirpanis).
* Rename windows pthread functions and blosc_pthread_{xyz}. Fixes #673
(Duplicate symbols for pthread functions on Windows).
Thanks to Barak Ugav (@barakugav).
Changes from 2.19.0 to 2.19.1
=============================
* New `blosc2_error_string()` public function for getting the error string
corresponding to a given error code. This is useful for debugging purposes.
Thanks to Barak Ugav (@barakugav).
* Support BLOSC2_ERROR_TUNER in print_error. Thanks to Barak Ugav (@barakugav).
* Eagerly register io_cb's to avoid race. Thanks to Tom Birch (@froody).
Changes from 2.18.0 to 2.19.0
=============================
* New `b2nd_expand_dims()` function for expanding the dimensions of a b2nd array.
This is useful for adding new dimensions to an existing b2nd array, e.g.,
when you want to add a new dimension to a 2D array to make it 3D.
Thanks to @lshaw8317 for the PR (#666 and #667).
* New internal view object that allows accessing the data in a b2nd array
without copying it. The new `b2nd_expand_dims()` function makes use of this.
In the future, it can also be useful for implementing e.g. a view on a slice
of a b2nd array.
Changes from 2.17.1 to 2.18.0
=============================
* New `b2nd_concatenate()` function for concatenating two b2nd arrays.
See PR #661 and #663.
* Flush modified pages only in write mode for mmap files. Thanks to Jan Sellner.
Changes from 2.17.0 to 2.17.1
=============================
Several fixes affecting uninitialized memory access and others:
* Fix uninitialized memory access in newly added unshuffle12_sse2 and unshuffle12_avx2 functions
* Fix unaligned access in _sw32 and sw32_
* Fix DWORD being printed as %s in sprintf call
* Fix warning on unused variable (since this variable was only being used in the linux branch)
* `splitmode` variable was uninitialized if goto was triggered
See PR #658. Many thanks to @EmilDohne for this nice job.
Changes from 2.16.0 to 2.17.0
=============================
* New b2nd_copy_buffer2() function for copying buffers with typesizes
larger than 255. The previous b2nd_copy_buffer() function is now
deprecated and will be removed in a future release.
* Support repeated values larger than 8-bit, also for n-dim arrays.
This is useful for compressing arrays with large runs of repeated
values, like in the case of images with large areas of the same color.
* Fix a leak in the pthreads emulation for Windows. Fixes #647.
Thanks to @jocbeh for the report and fix (#655).
* Update zstd to 1.5.7. Thanks to Tom Birch.
* Add BLOSC2_MAXTYPESIZE constant.
### Deprecated Functions
- `int b2nd_copy_buffer(...)` is deprecated and will be removed in
a future release. Please use `b2nd_copy_buffer2(...)` instead.
Changes from 2.15.2 to 2.16.0
=============================
* Use _fseeki64/_ftelli64/_stat64 on Windows for large file (>2 GB) support.
Thanks to Abhi Jaiantilal (@ajaiantilal) for the report and help.
* Add 12-byte unshuffle for avx2. Thanks to Tom Birch (@froody).
* Add 12-byte sse2 unshuffle implementation. Thanks to Tom Birch (@froody).
* Better description of the Blosc2 format as a whole.
Changes from 2.15.1 to 2.15.2
=============================
* Support wasm32 by disabling ZLIB WITH_OPTIM option. Thanks to Miles Granger.
* Avoid rip-relative addressing for OSX x86_64. Thanks to Miles Granger.
* Added support for nvcc (NVidia Cuda Compiler) in CMake. Thanks to @dqwu.
* Fix public include directories for blosc2 targets. Thanks to Dmitry Mikushin.
* Fix ub in shuffle and unshuffle by marking _dst non-const. Thanks to Emil Dohne.
Changes from 2.15.0 to 2.15.1
=============================
* Do not pass `-m` flags when compiling `shuffle.c`. This prevents the
compiler from incidentally optimizing the code called independently
of the runtime CPU check to these instruction sets, effectively
causing `SIGILL` on other CPUs. Fixes #621. Thanks to @t20100 and @mgorny.
* Internal LZ4 sources bumped to 1.10.0.
* Allow direct loading of plugins by name, without relying on
the presence of python. Thanks to @boxerab.
* Add `b2nd_nans` method (PR #624). Thanks to @waynegm.
Changes from 2.14.4 to 2.15.0
=============================
* Removed some duplicated functions. See https://github.com/Blosc/c-blosc2/issues/503.
* Added a new io mode to memory map files. This forced to change the `io_cb` read API.
See https://github.com/Blosc/c-blosc2/blob/main/tests/test_mmap.c to see an example on
how to use it.
* Updated the `SOVERSION` to 4 due to the API change in `io_cb` read.
* Added functions to get cparams, dparams, storage and io defaults respectively.
* Internal zstd sources updated to 1.5.6.
* Fixed a bug when setting a slice using prefilters.
Changes from 2.14.3 to 2.14.4
=============================
* Bumped SONAME due to recent API changes. See https://github.com/Blosc/c-blosc2/issues/581.
Changes from 2.14.2 to 2.14.3
=============================
* More fixes for internal fuzzer.
Changes from 2.14.1 to 2.14.2
=============================
* Fixes for CVE-2024-3203 and CVE-2024-3204.
Changes from 2.14.0 to 2.14.1
=============================
* When loading plugins, first try with `python` and then `python3`.
This is because many linux distros do not have `python` as a
symlink to `python3` anymore.
Changes from 2.13.2 to 2.14.0
=============================
* Fixed a bug preventing buffers to be appended to empty (0-sized) b2nd arrays.
* New acceleration path for `b2nd_append()`. This new path is
much faster (up to 4x) than the previous one, specially for large arrays.
See `bench/bench_stack_append.c` for the bench of use.
* New examples for using the `b2nd_set_slice_cbuffer()` and
`b2nd_append()` functions for adding data into existing b2nd arrays.
See `examples/example_stack_images.c`.
* Now, ``python3`` is used for finding plugins instead of ``python``.
This is because many linux distros do not have ``python`` as a symlink
to ``python3`` anymore.
* New round of fixing warnings. Now, C-Blosc2 should be relatively free of them.
* Small performance tweak for clevel 1 in BloscLZ codec.
* Fixed a leak in frame code. Closes #591. Thanks to @LuMingYinDetect.
* Disable shuffle repeat in filters pipeline. This was broken
since the initial implemented, and it was never documented.
Also, compression ratios do not seem to be improved in our experiments,
so this capability has been removed completely.
* Support for new Intel compilers (2023.0.1 and on). Fixes #533.
Thanks to Nick Papior.
Changes from 2.13.1 to 2.13.2
=============================
* Better checking for `SSSE3` availability in Visual Studio. Probably fixes #546 too.
Thanks to @t20100 (Thomas Vincent) for the PR (#586).
* Documented the globally registered filters and codecs. See:
https://www.blosc.org/c-blosc2/reference/utility_variables.html#codes-for-filters
https://www.blosc.org/c-blosc2/reference/utility_variables.html#compressor-codecs
Changes from 2.13.0 to 2.13.1
=============================
* Removed private include in `b2nd.h`. This fixes issue #579.
Changes from 2.12.0 to 2.13.0
=============================
* Added a new BLOSC_FILTER_INT_TRUNC filter for truncating integers to a
given number of bits. This is useful for compressing integers that are
not using all the bits of the type. See PR #577.
* Optimized zstd, specially when using dicts. See PR #578.
* Initialize grok library when loading the plugin. This is needed for other plugins
to be able to use it without the need of importing the package.
Changes from 2.11.3 to 2.12.0
=============================
* New `blosc2_get_slice_nchunks` function for getting the unidimensional chunk indexes of a Blosc2 container slice.
* Globally registered new codec `grok`. This will be loaded dynamically.
Changes from 2.11.2 to 2.11.3
=============================
* Frames accept now typesizes that are larger than 255 (and up to 2**31).
See https://github.com/PyTables/PyTables/issues/1086. Thanks to
@chris-allan for the report.
* AVX512 runtime dispatching has been fixed (basically disabled) for GCC
versions <= 10.
* Use typedef for blosc_timestamp_t. Thanks to Magnus Ulimoen.
Changes from 2.11.1 to 2.11.2
=============================
* Added support for ARMv7l platforms (Raspberry Pi). The NEON version
of the bitshuffle filter was not compiling there, and besides it offered
no performance advantage over the generic bitshuffle version (it is 2x to
3x slower actually). So bitshuffle-neon.c has been disabled by default in
all ARM platforms.
* Also, unaligned access has been disabled in all ARM non-64bits platforms.
It turned out that, at least the armv7l CPU in Raspberry Pi 4, had issues
because `__ARM_FEATURE_UNALIGNED` C macro was asserted in the compiler
(both gcc and clang), but it actually made binaries to raise a "Bus error".
* Thanks to Ben Nuttall for providing a Raspberry Pi for tracking down these
issues.
Changes from 2.11.0 to 2.11.1
=============================
* Fix ALTIVEC header. Only affects to IBM POWER builds. Thanks to
Michael Kuhn for providing a patch.
Changes from 2.10.5 to 2.11.0
=============================
* New AVX512 support for the bitshuffle filter. This is a backport of the upstream
bitshuffle project (https://github.com/kiyo-masui/bitshuffle). Expect up to [20%
better compression speed](https://github.com/Blosc/c-blosc2/pull/567#issuecomment-1789239842)
on AMD Zen4 architecture (7950X3D CPU).
* Add c-blosc2 package definition for Guix. Thanks to Ivan Vilata.
* Properly check calls to `strtol`. Fixes #558.
* Export the `b2nd_copy_buffer` function. This may be useful for other projects
dealing with multidimensional arrays in memory. Thanks to Ivan Vilata.
* Better check that nthreads must be >= 1 and <= INT16_MAX. Fixes #559.
* Fix compile arguments for armv7l. Thanks to Ben Greiner.
Changes from 2.10.4 to 2.10.5
=============================
* Fix a variable name in a test that was causing a segfault in some platforms.
* Change tuner's functions signature to return always an error code. This allows
for better error checking when using pluggable tuners in Blosc2.
* Do checks when creating contexts.
Changes from 2.10.3 to 2.10.4
=============================
* Remove duplicated tune initialization since it is already done in blosc2_create_cctx.
Thanks to Marta Iborra
* Typos fixed. Thanks to Dimitri Papadopoulos.
Changes from 2.10.2 to 2.10.3
=============================
* Globally registered new codec `openhtj2k`. This will be loaded dynamically. See PR #557.
* Added a `BLOSC_INFO` macro for details on compression params.
* Added `get_blocksize.c` example on automatic blocksizes.
* Warning fixes.
* Fixes for mingw.
Changes from 2.10.1 to 2.10.2
=============================
* Several fixes for the CMake system. Thanks to Axel Huebl. See PR #541 and #542.
* Several fixes for mingw platform. Thanks to Biswapriyo Nath. See PR #540 and #543.
Changes from 2.10.0 to 2.10.1
=============================
* `blosc2_remove_urlpath(const char *urlpath)` does not return an error
when path does not exist.
* Changes in CMake installer to conserve targets and properties
on install, so CMake users do not need to write `FindBlosc2.cmake`
files anymore. This also helps to preserve transitive dependencies on
CMake targets, especially useful for fully static builds, e.g., for
Python wheels. Thanks to @ax3l (Axel Huebl). See PR #537.
* Fix new typos. Thanks to @DimitriPapadopoulos. See PR #538.
Changes from 2.9.3 to 2.10.0
============================
* bytedelta filter has been fixed. For backward compatibility, the old
bytedelta filter is still available as `BLOSC_FILTER_BYTEDELTA_BUGGY`
symbol, with the same ID (34) than before. The new, fixed bytedelta
filter has received a new ID (35) and it can be used via the usual
`BLOSC_FILTER_BYTEDELTA` symbol. That means that old data written with
the buggy bytedelta filter should be decompressed without issues.
Thanks to @foody (Tom Birch) for the fix. See #531, #532 for more info.
* Filter buffers are correctly cycled now. Now it is possible to use e.g.
shuffle and bitshuffle filters in the pipeline. Thanks to @foody (Tom Birch)
for the fix. See #528 and PR #530.
* Assorted fixes for allowing better inclusion in external projects.
Thanks to @ax3l (Axel Huebel). See #525, #527 and #529.
* Minor fixes in the documentation. Thanks to @ivilata (Ivan Vilata).
See #523.
Changes from 2.9.2 to 2.9.3
===========================
* Thanks to Dimitri Papadopoulos for an extensive set of improvements in
documentation and code.
* `load_lib` is now a private function. Before was public, but
never meant to be.
* Several fixes for bugs discovered by the fuzzer.
Changes from 2.9.1 to 2.9.2
===========================
* Now is possible to register the same plugin (as long as they have the same
ID *and* name) without errors. This is useful for registering the same
plugin without worrying on whether it has been registered already.
* Improved detection of dynamic plugin locations. Now they must implement
`plugin_module.print_libpath()` as the canonical way to find the path for
the dynamic library plugin.
* The `blosc2_static` has gained the cmake POSITION_INDEPENDENT_CODE property.
This should allow to use the static library in more situations.
* `BLOSC_STUNE` is defined in `blosc2.h` now. Fixes #481. Thanks to
@DimitriPapadopoulos.
* Fixed an issue when having incompressible data in combination with lazy_chunks.
* Fix linking with static -DBUILD_STATIC=0. Fixes #480.
* Visual Studio 2010 (version 10.0) has been deprecated. Now, users will
need to use Visual Studio 2012 (version 11.0) or later.
* Many small fixes and code improvements. Thanks to @DimitriPapadopoulos,
@bnavigator.
Changes from 2.9.0 to 2.9.1
===========================
* Allow the use of BTUNE by detecting the ``BTUNE_BALANCE`` environment
variable.
Changes from 2.8.0 to 2.9.0
===========================
* Dynamic plugins as Python wheels are supported now!
This new feature allows for creating plugins in C, distribute
them as wheels, and load them dynamically in runtime.
Small example at https://github.com/Blosc/blosc2_plugin_example
* BloscLZ can achieve more speed in clevel 1 now.
* Internal Zstd sources updated to latest 1.5.5 version.
* Copyright notice updated. Thanks to @DimitriPapadopoulos.
Changes from 2.7.1 to 2.8.0
===========================
* New bytedelta filter added. SIMD support for Intel and ARM platforms is there.
We have blogged about this: https://www.blosc.org/posts/bytedelta-enhance-compression-toolset.rst
Thanks to Aras PranckeviÄŤius for inspiration and initial implementation.
* Minor improvements in BloscLZ, leading to better compression ratios in general.
BLoscLZ version bumped to 2.5.2.
* Updated internal zlib-ng to 2.0.7.
* Used `const` qualifier where possible in b2nd. Thanks to @cf-natali.
Changes from 2.6.1 to 2.7.1
===========================
* Caterva has been merged and carefully integrated in C-Blosc2 in the new b2nd interface.
For more info on the new interface, see https://www.blosc.org/c-blosc2/reference/b2nd.html.
Thanks to Marta Iborra, Oscar Guiñón, J. David Ibáñez and Francesc Alted. Also thanks to
Aleix Alcacer for his great work in the Caterva project.
We have a blog about this: https://www.blosc.org/posts/blosc2-ndim-intro
* Updated internal zstd sources to 1.5.4. Thanks to Dimitri Papadopoulos.
* `blosc2_schunk_avoid_cframe_free` and `blosc2_schunk_append_file` are exported as public functions now.
Thanks to @bnavigator.
* BloscLZ codec is now treated exactly the same as LZ4. Before BloscLZ was considered less capable of reaching
decent compression ratios, but this has changed quite a bit lately, so there is no point in treating both differently.
* Fixed some leaks, mainly on the test suite.
* Fixed quite a bit of compiler warnings.
Changes from 2.6.0 to 2.6.1
===========================
* Add support for macos universal2 binaries (arm64+x86_64 build). Thanks to Thomas Vincent.
Changes from 2.5.0 to 2.6.0
===========================
* [API] Now it is possible to pass filter ID to a User Defined Filter.
* Unified convention for BLOSC_SPLITMODE environment variable in Blosc
and Blosc2. The list of valid values is "ALWAYS", "NEVER", "AUTO" and
"FORWARD_COMPAT" now.
* Unified convention for BLOSC_SPLITMODE enum in Blosc and Blosc2 headers.
Changes from 2.4.3 to 2.5.0
===========================
* Fixed a nasty bug that prevented retrieving data correctly with large super-chunks (> 2^31 elements).
* Fixed an issue in `blosc2_schunk_get_slice_buffer()` in the interpretation of the `stop` param.
Now `stop` is not part of the selected slice (as advertised).
* Now `blosc2_create_cctx()` supports the same environment variables than `blosc2_compress()`.
* Now `blosc2_create_dctx()` supports the same environment variables than `blosc2_decompress()`.
* Added support for the split mode to be serialized in cframes/sframes.
* A new `splitmode` field has been added to the `blosc2_schunk` structure.
* Changed some fields in `blosc2_preparams` and `blosc2_postparams` structs:
* `in` -> `input`
* `out` -> `output`
* `out_size` -> `output_size`
* `out_typesize` -> `output_typesize`
* `out_offset` -> `output_offset`
This was needed to allow Cython to map the fields (`in` is a reserved word in Python).
* Disabled maskout reads in `blosc2_schunk_get_slice_buffer()` as they are not faster than getitem there.
* Add an intermediate block size in its automatic calculation based on `clevel`.
Changes from 2.4.2 to 2.4.3
===========================
* Disable automatic split of blocks when not using shuffle. Experiments are showing that cratio is suffering too much, specially when using BloscLZ.
* Changed xxhash.h and xxhash.c to the newest version. Thanks to Dimitri Papadopoulos.
Changes from 2.4.1 to 2.4.2
===========================
* Fixed BLOSC1_COMPAT mode. If the symbol `BLOSC1_COMPAT` is defined, one should actually get a correct map for the BLOSC1 API now. Thanks to @MehdiChinoune for pointing this out.
* Optimizations for `blosc2_schunk_get_slice_buffer()`. Now more caution has been put in avoiding memcpy's as much as possible.
* Now `blosc2_set_nthreads()` also modifies the schunk compression and decompression defaults.
* Several other fixes, specially when converting schunks to cframe format.
Changes from 2.4.0 to 2.4.1
===========================
* New `blosc2_schunk_avoid_cframe_free()` for avoiding the free of a cframe when destroying a super-chunk. Mainly useful for situations where you build a super-chunk out of an existing cframe, so you don't want it to be freed automatically.
Changes from 2.3.1 to 2.4.0
===========================
* New `blosc2_schunk_get_slice_buffer()` and `blosc2_schunk_set_slice_buffer()` functions for getting and setting slices from/to a super-chunk.
Changes from 2.3.0 to 2.3.1
===========================
* Support for negative values for BLOSC_TRUNC_PREC filter. Negatives values mean reduce mantissa precision bits, whereas positive values mean keep precision bits.
* Re-add the check for small buffers and a new test (see https://github.com/Blosc/python-blosc2/issues/46).
* Make `static` a couple of funcs to avoid collisions. This can be useful in case someone tries to compile both C-Blosc and C-Blosc2 in the same name space (however, don't do that unless you know what you are doing; better use dynamic libraries which allow for a much better name space separation).
Changes from 2.2.0 to 2.3.0
===========================
* [API change] In order to allow to compile with both C-Blosc and C-Blosc2 libraries, a new API has been created for the symbols and function names that had collisions. Here are the changed symbols and functions:
* Blosc2 symbols that take different values than in Blosc1:
- BLOSC_VERSION_MAJOR -> BLOSC2_VERSION_MAJOR
- BLOSC_VERSION_MINOR -> BLOSC2_VERSION_MINOR
- BLOSC_VERSION_RELEASE -> BLOSC2_VERSION_RELEASE
- BLOSC_VERSION_STRING -> BLOSC2_VERSION_STRING
- BLOSC_VERSION_DATE -> BLOSC2_VERSION_DATE
- BLOSC_MAX_OVERHEAD -> BLOSC2_MAX_OVERHEAD
- BLOSC_MAX_BUFFERSIZE -> BLOSC2_MAX_BUFFERSIZE
* Original Blosc1 API that takes the `blosc1_` prefix:
- blosc_compress -> blosc1_compress
- blosc_decompress -> blosc1_decompress
- blosc_getitem -> blosc1_getitem
- blosc_get_compressor -> blosc1_get_compressor
- blosc_set_compressor -> blosc_set_compressor
- blosc_cbuffer_sizes -> blosc1_cbuffer_sizes
- blosc_cbuffer_validate -> blosc1_cbuffer_validate
- blosc_cbuffer_metainfo -> blosc1_cbuffer_metainfo
- blosc_get_blocksize -> blosc1_get_blocksize
- blosc_set_blocksize -> blosc1_set_blocksize
- blosc_set_splitmode -> blosc1_set_splitmode
* API that has been migrated to blosc2_ prefix
- blosc_init -> blosc2_init
- blosc_destroy -> blosc2_destroy
- blosc_free_resources -> blosc2_free_resources
- blosc_get_nthreads -> blosc2_get_nthreads
- blosc_set_nthreads -> blosc2_set_nthreads
- blosc_compcode_to_compname -> blosc2_compcode_to_compname
- blosc_compname_to_compcode -> blosc2_compname_to_compcode
- blosc_list_compressors -> blosc2_list_compressors
- blosc_get_version_string -> blosc2_get_version_string
- blosc_get_complib_info -> blosc2_get_complib_info
- blosc_cbuffer_versions -> blosc2_cbuffer_versions
- blosc_cbuffer_complib -> blosc2_cbuffer_complib
It is recommended to migrate to the new API as soon as possible. In the meanwhile, you can still compile with the previous API (corresponding to C-Blosc2 pre-2.3.0), by defining the `BLOSC1_COMPAT` symbol in your C-Blosc2 app (before including the 'blosc2.h' header).
* Fixed some issues in converting from super-chunks to frames and back. Now it is possible to do a roundtrip without (known) problems.
* LZ4 codec has been bumped to 1.9.4.
Changes from 2.1.1 to 2.2.0
===========================
* Added new `blosc2_schunk_open_offset()` and `blosc2_schunk_append_file()` functions for being able to open a super-chunk inside of file, and append a super-chunk to the end of an existing file. See https://github.com/Blosc/c-blosc2/pull/409.
* Protect the update of a field in compression/decompression context. Fixes issues when compressing/decompressing super-chunks in multi-thread mode.
* Fix issue when inserting a chunk in a super-chunk. See https://github.com/Blosc/c-blosc2/pull/408.
* Fix issue when appending a special chunk in a super-chunk. See https://github.com/Blosc/c-blosc2/pull/407.
* Optimized the blockshape calculation when using the ZFP plugin in `BLOSC_CODEC_ZFP_FIXED_RATE` mode. See https://github.com/Blosc/c-blosc2/pull/406.
* New `blosc2_unidim_to_multidim` and `blosc2_multidim_to_unidim` which are useful for codecs that are meant to deal with multidimensional arrays (like ZFP).
* Another round of squashing warnings has been carried out. Thanks to Marta Iborra.
* Added locks in situations where different threads were trying to update the same variable. Thanks to Marta Iborra (and helgrind!).
* Use proper types for native zlib-ng interface. This allows for linking with zlib-ng native API. Thanks to Klaus Zimmermann.
Changes from 2.1.0 to 2.1.1
===========================
* Fixed a nasty bug introduced in 2.1.0 that affects to the blosclz codec in ARM arch. See https://github.com/Blosc/c-blosc2/issues/392.
* Added support for external zlib-ng (see https://github.com/Blosc/c-blosc2/pull/393). Thanks to Mark Harfouche.
* Fixed more compiler warnings (https://github.com/Blosc/c-blosc2/pull/394). Now C-Blosc2 should be relatively free of them.
* Fixed more fuzzer issues.
Changes from 2.0.4 to 2.1.0
===========================
* Support of the [ZFP lossy codec](https://zfp.readthedocs.io/) as a plugin. This is mainly meant for compressing multidimensional arrays of floating point data, so it is better used in combination with [Caterva](https://github.com/Blosc/caterva). More info at our blog: https://www.blosc.org/posts/support-lossy-zfp/. Thanks to Oscar Guiñon and Francesc Alted.
* Massive cleanup of warnings. Thanks to Marta Iborra.
* Fixed some bugs in test updates and insertions in 64-bit super chunks. Thanks to Francesc Alted.
* Fixed the build on FreeBSD/powerpc64le. Thanks to @pkubaj.
* Updated internal zstd sources to 1.5.2.
* Updated internal zlib-ng to 2.0.6.
Changes from 2.0.3 to 2.0.4
===========================
* Restored support for type size that is not a divisor of a buffer size for more info. See #356.
* Implemented a `blosc2_rename_urlpath`, a portable function to rename a file or a full directory. See #355.
* Several improvements for packaging. See #354, #357, #359.
* Fixed a bug in BloscLZ codec (bumped to 2.5.1).
Changes from 2.0.2 to 2.0.3
===========================
* Improved BloscLZ codec (bumped to 2.5.0) for achieving better compression ratios on data with high entropy. Although in general LZ4 still does a better job in this scenario, *BloscLZ* can sometimes achieve better compression ratios.
* Added `blosc2_vlmeta_delete()` for removing vlmeta data. Thanks to Marta Iborra.
* Update pkg-config file to support blosc2. Fixes #236. Thanks to HĂĄvard Flaget Aasen.
* Build system: Change hard coded library path with `CMAKE_INSTALL_LIBDIR` CMake variable. Thanks to HĂĄvard Flaget Aasen.
Changes from 2.0.1 to 2.0.2
===========================
* Fixed data chunk memory leaks in frame (see #335).
* Fixed blosc2_stdio_open never returns NULL if it cannot open file.
* Standardized places for headers in blosc/ and include/ dirs.
* `nthreads` is int16_t everywhere in the API. Fixes #331.
* Add blosc2_remove_urlpath function (see #330).
* Fixed a bug when a lazy_chunk was created from a small, memcpyed chunk.
(see #329).
* Fixed many issues in documentation (see #333).
Changes from 2.0.0 to 2.0.1
===========================
* The `blosc2_schunk_fill_special` function was not exported,
so not in the shared library. This has been fixed; see #328.
Thanks to Mark Kittisopikul.
Changes from 2.0.0.rc2 to 2.0.0 (final)
=======================================
* Now Blosc is always compiled with LZ4. See #324.
* Implemented a system to register plugins (PR #314).
See our blog at: https://www.blosc.org/posts/registering-plugins.
* Added Blosc Lite version. Just activate `BUILD_LITE` cmake option with:
`-DBUILD_LITE`. See #316.
* You can deactivate the plugins by setting cmake option `BUILD_PLUGINS` to OFF.
* Created `include` folder. See #310.
* Moved codecs-registry.h and filters-registry.h to include/blosc2. See #325.
* Fix error in endian-handler function affecting frames metadata. See #320.
* Improved tolerance to Win64 workflows failure. See #319.
* zlib-ng updated to 2.0.5.
* New COMPILING_WITH_WHEELS.rst doc added.
Changes from 2.0.0.rc.1 to 2.0.0.rc2
====================================
* New compatibility with MinGW32/64. See #302.
* Improved support for AArch64 (aka ARM64), ARMv7l, ARMv6l and powerpc64le.
See #306. Thanks to Alexandr Romanenko, Mark Kittisopikul and Mosè Giordano
from the Julia packaging team for their help.
* BloscLZ updated to 2.4.0. Aligned access in ARM has been enabled, as well
as other performance improvements. Expect much better performance,
specially on ARM platforms (like Apple Silicon).
* Zstd sources updated to 1.5.0.
* zlib-ng sources updated to 2.0.3.
Changes from 2.0.0-beta.5 to 2.0.0.rc.1
=======================================
* [API change] `blosc2_decompress_ctx()` gets a new `srcsize`
parameter to ensure that it does not read past the end
of the provided buffer. See #144. Thanks to Nathan Moinvaziri
(@nmoinvaz).
* [BREAKING CHANGE] The format for frames has changed and
BLOSC2_VERSION_FRAME_FORMAT is now set to 2. There is no attempt to support
previous formats, but there will probably be backward compatibility support
starting from version 2 on.
* New functionality for updating, inserting and deleting chunks in a super-chunk.
* Support for special values. Large sequences of repeated values can be represented
with an efficient, simple and fast run-length representation, without the need to use
regular codecs.
* Internal Zstd sources updated to 1.4.9.
* Internal LZ4 sources updated to 1.9.3.
* Internal zlib support is provided now by new zlib-ng 2.0.2 (replacing miniz).
* The support for Snappy codec has been completely removed. Snappy is a C++
library, which is not good for a library that aims to be fully pure C.
Snappy was removed from sources in C-Blosc(1) some years ago, so there
should not be a lot of data compressed with Blosc/Snappy out there (and
for the existing ones, a transcoding is always possible using C-Blosc(1)).
* The Lizard codec has been removed. Lizard is a pretty good one, but it
looks like it is not timely maintained. Zstd/Zlib can cover its place pretty
nicely.
* The split of blocks only happens for BLOSCLZ and LZ4 codecs. All
the rest are not split at all.
* Public APIs for frames have been removed. Frames should be considered an
storage detail, so having them accessible publicly should only bring
unnecessary cognitive load. Care have been carried out so as to ensure
the same functionality via the super-chunk (schunk) API.
* [FORMAT] New *sparse frame* format for on-disk I/O. This allows for storing
data chunks in super-chunks in separate files inside a directory. The way
to generate sparse frames is via `storage.contiguous=false` and
`storage.urlpath=dirname`. See `README_SFRAME_FORMAT.rst` for details.
Changes from 2.0.0-beta.4 to 2.0.0.beta5
========================================
* The prefilter functionality has been introduced and declared stable.
With that, you can callback arbitrary functions previous to any filter.
This can be useful for performing (parallel) computations on chunks.
For an example of use, see `tests/test_prefilter.c`.
* New blosc2_set_maskout() function to avoid decompressing blocks. This
can be handy when it is not needed to decompress all the blocks in a
chunk. This should be always called before blosc2_decompress_ctx() and
its effect is reset to the default (decompress all blocks) after that.
* New ALTIVEC optimizations for Power architecture. These include support
for both shuffle and bitshuffle filters. For details, see
https://github.com/Blosc/c-blosc2/pull/98. Thanks to Jerome Kieffer
and ESRF for making this happen.
* New frame_from_bframe() function for getting a `blosc2_frame`
out of an in-memory serialized frame.
* Zstd codec updated to 1.4.4.
Changes from 2.0.0-beta.3 to 2.0.0-beta.4
=========================================
* New pluggable threading backend. Instead of having Blosc use its own
thread pool, you can instead call
`blosc2_set_threads_callback(threads_callback, callback_data)` to install
your own threading backend. This gives Blosc the possibility to use the
same threading mechanism as one you are using in the rest of your program
(e.g. OpenMP or Intel TBB), sharing the same threads, rather than starting
its own threads that compete with yours for the CPU cores. See #PR 81.
Thanks to Steven G. Johnson.
* The endianness of the platform that is writing the data in chunks is stored
now in the headers of the chunks. This info is not used yet, but this
should allow a good hint for implementing format compatibility among
platforms with different endianness in other layers. See PR #84.
* Fixed a nasty bug that prevented frames to go more than 2 GB in size.
* Added a cache for on-disk offsets. This accelerates the reading of slices
from disk quite a lot (up to 50% with my benchmarks).
* Zstd codec upgraded from 1.4.0 to 1.4.3.
Changes from 2.0.0-beta.2 to 2.0.0-beta.3
=========================================
* Quick release to fix that beta.2 was tagged on top of a branch, not master.
* The msgpack trailer now properly starts with `0x90 + 4` value, not plain
wrong `0x09 + 4`.
* Trailer version bumped to 1.
Changes from 2.0.0-beta.1 to 2.0.0-beta.2
========================================
* A new `vlmetalayers` chunk in `schunk` allows to store arbitrary meta-information
that is up to the user. If the `schunk` has an attached `frame`, the later
will be updated accordingly too. For more info, see PR #74 and docstrings of
new `blosc2_update_vlmetalayer()` and `blosc2_get_vlmetalayer()` functions.
* Metalayers must now be attached to super-chunks, not frames. The reason is
that frames are increasingly treated as a storage specifier (in-memory or
disk now, but can be other means in the future), whereas the actual API for
I/O (including metainfo) goes into super-chunks. See PR #75.
* New frame format documented in
[README_CFRAME_FORMAT.rst](README_CFRAME_FORMAT.rst). Remember that the frame
format is not written in stone yet, so some changes may be introduced before
getting out of beta.
* BREAKING CHANGE: the format for frames has changed and
BLOSC2_VERSION_FRAME_FORMAT is now set to 1. There is no attempt to support
previous formats, but there will probably be backward compatibility support
starting from version 1 on.
* BREAKING CHANGE: the next APIs have been renamed:
+ blosc2_frame_has_metalayer -> blosc2_has_metalayer
+ blosc2_frame_add_metalayer -> blosc2_add_metalayer
+ blosc2_frame_update_metalayer -> blosc2_update_metalayer
+ blosc2_frame_metalayer -> blosc2_get_metalayer
Although the API was declared stable in beta.1, the fact that metalayers are
attached now to super-chunks directly, made this change completely necessary.
* BREAKING CHANGE: the next symbols have been renamed:
+ BLOSC_CPARAMS_DEFAULTS -> BLOSC2_CPARAMS_DEFAULTS
+ BLOSC_DPARAMS_DEFAULTS -> BLOSC2_DPARAMS_DEFAULTS
Changes from 2.0.0a5 to 2.0.0-beta.1
====================================
* The library is called now `blosc2` and not `blosc` anymore. This is necessary
so as to prevent collisions with existing `blosc` deployments.
* The `make install` now install all the necessary requirements out-of-the-box.
* Use Intel IPP's LZ4Safe when compressing/decompressing: this provides better
compression ratios and speed (in some cases). It is activated automatically
if Intel IPP is found in the system, but you can always disable it with:
`cmake -DDEACTIVATE_IPP=ON`
* BREAKING CHANGE: the next API have been made private:
+ blosc2_frame_append_chunk -> frame_append_chunk
+ blosc2_frame_get_chunk -> frame_get_chunk
+ blosc2_frame_decompress_chunk -> frame_decompress_chunk
Now the appending and retrieval of data in frames should always be made via the frame-backed super-chunk API. The idea is to deduplicate the I/O primitives as much as possible, and the super-chunks are the logical way for doing this.
* BREAKING CHANGE: the next APIs have been renamed:
+ blosc2_get_cparams - > blosc2_schunk_get_cparams
+ blosc2_get_dparams - > blosc2_schunk_get_dparams
* Internal BloscLZ sources bumped to 2.0.0.
* Internal LZ4 sources bumped to 1.9.1.
* Internal Zstd sources bumped to 1.4.0.
Changes from 2.0.0a4 to 2.0.0a5
===============================
- Delta filter now implemented as a XOR instead of a NEG for better numerical
stability.
- Zstd dictionaries are supported. This allows for (much) better compression
ratios when using small blocksizes, while improving compression speed.
- Performance upgrades in the BloscLZ codec, specially for decompression
in ARM architectures.
- Preliminary support for Neon instruction set in ARM CPUs. Shuffle seems
to work well, but still some issues with bitshuffle.
- Internal LZ4 sources bumped to 1.8.2.
- Internal Zstd sources bumped to 1.3.4.
Changes from 2.0.0a3 to 2.0.0a4
===============================
- New filter pipeline designed to work inside a chunk. The pipeline has a
current capacity of 5 slots, and is designed to apply different filters
sequentially over the same chunk. For this, a new extended header for the
chunk has been put in place (see README_HEADER.rst).
- New delta filter meant to work inside a chunk. Previously delta was
working inside a super-chunk, but the new implementation is both faster and
simpler and gets better compression ratios (at least for the tested datasets).
- New re-parametrization of BloscLZ for improved compression ratio
and also more speed (at least on modern Intel/AMD CPUs). Version
for internal BloscLZ codec bumped to 1.0.6.
- Internal zstd sources bumped to 1.3.0.
- The BLOSC2_MAX_OVERHEAD symbol is always 32 bytes, not 16 as in Blosc1.
This is needed in order to allow buffers larger than 32 bits (31 actually :).
Changes from 2.0.0a2 to 2.0.0a3
===============================
* New blosc2_create_context() and blosc2_free_context() for creating
contexts so that Blosc can be used from threaded applications
without the global lock. For now contexts can be only used from
blosc2_compress_ctx() and blosc2_decompress_ctx() (see below).
* The blosc_compress_ctx() and blosc_decompress_ctx() have been
replaced by blosc2_compress_ctx() and blosc2_decompress_ctx() that
do accept actual contexts.
* Added support for new Zstd codec (https://github.com/Cyan4973/zstd).
This is a new compressor by Yann Collet, the author of LZ4 and
LZ4HC. For details on Zstd, see this nice intro:
https://fastcompression.blogspot.com/2015/01/zstd-stronger-compression-algorithm.html.
* The blosc2_append_chunk() has been removed. This is this because an
existing chunk may not fulfill the sequence of filters in super
header. It is best that the user will use blosc2_schunk_append_buffer()
and compress it internally.
* The split of blocks only happens for BLOSCLZ and SNAPPY codecs. All
the rest are not split at all. This allows for faster operation for
the rest of codecs (most specially zstd).
* The internal zlib 1.2.8 sources have been replaced by the miniz
library, which is meant to be fully compatible with Zlib, but much
smaller and besides tends to be a bit faster. Also, miniz is
preferred to an external Zlib.
* The internal Snappy sources have been removed. If the Snappy library
is found, the support for it is still there.
* Internal LZ4 sources upgraded to 1.7.1.
Changes from 2.0.0a1 to 2.0.0a2
===============================
* The delta filter runs inside of the compression pipeline.
* Many fixes for the delta filter in super-chunks.
* Added tests for delta filter in combination with super-chunks.
Blosc-c-blosc2-d3bee7c/RELEASING.rst 0000664 0000000 0000000 00000012006 15255167342 0017010 0 ustar 00root root 0000000 0000000 Releasing a version
===================
Preliminaries
-------------
- Make sure that ``RELEASE_NOTES.md`` and ``ANNOUNCE.md`` are up to
date with the latest news in the release.
- Check that *VERSION* symbols in include/blosc2.h contains the correct info.
- Check that all public functions are covered in the reference docs::
$ python3 doc/check_missing_docs.py
If any are missing, add them to the appropriate ``.rst`` file under
``doc/reference/``. If a function is intentionally left undocumented,
add it to ``INTENTIONAL_SKIP`` in ``doc/check_missing_docs.py``.
- If API/ABI changes, please increase the minor number (e.g. 2.15 -> 2.16) *and*
bump the SOVERSION in blosc/CMakeLists.txt.
When in doubt on when SOVERSION should change, see these nice guidelines:
https://github.com/conda-forge/c-blosc2-feedstock/issues/62#issuecomment-2049675391
- Commit the changes with::
$ git commit -a -m "Getting ready for release X.Y.Z"
$ git push
Testing
-------
Create a new build/ directory, change into it and issue::
$ cmake ..
$ cmake --build .
$ ctest
Forward compatibility testing
-----------------------------
First, go to the compat/ directory and generate a file with the current
version::
$ cd ../compat
$ export LD_LIBRARY_PATH=../build/blosc # Linux
$ export DYLD_LIBRARY_PATH=../build/blosc # macOS
$ gcc -o filegen filegen.c -L../build/blosc -lblosc2 -I../include
$ ./filegen compress lz4 blosc-lz4-1.y.z.cdata
In order to make sure that we are not breaking forward compatibility,
link and run the `compat/filegen` utility against different versions of
the Blosc library (suggestion: 1.3.0, 1.7.0, 1.11.1, 1.14.1, 2.0.0).
You can compile the utility with different blosc shared libraries with::
$ export LD_LIBRARY_PATH=shared_blosc_library_path # Linux
$ export DYLD_LIBRARY_PATH=shared_blosc_library_path # macOS
$ gcc -o filegen filegen.c -Lshared_blosc_library_path -lblosc -Iblosc2.h_include_path
Then, test the file created with the new version with::
$ ./filegen decompress blosc-lz4-1.y.z.cdata
Repeat this for every codec shipped with Blosc (blosclz, lz4, lz4hc, zlib and
zstd).
For shuffle-only fixtures, use the matching inverse operation instead of
``decompress``::
$ ./filegen unshuffle shuffle-2.20.0.cdata
$ ./filegen bitunshuffle bitshuffle-2.20.0.cdata
For VL-block cframe compatibility checks, use ``compat/filegen-vl``.
This utility can generate two kinds of persistent cframes:
* a cframe with VL-block chunks and fixed chunk sizes
* a cframe with VL-block chunks and variable chunk sizes
Generate them with the current version::
$ cd ../compat
$ export LD_LIBRARY_PATH=../build/blosc # Linux
$ export DYLD_LIBRARY_PATH=../build/blosc # macOS
$ gcc -o filegen-vl filegen-vl.c -L../build/blosc -lblosc2 -I../include
$ ./filegen-vl compress lz4 vlblocks-lz4-fixed-2.y.z.b2frame
$ ./filegen-vl compress lz4 vlblocks-lz4-variable-2.y.z.b2frame --variable-chunks
$ ./filegen-vl compress lz4 varchunks-lz4-2.y.z.b2frame --regular
Then, build ``compat/filegen-vl`` against the target Blosc library and check
the fixtures with::
$ ./filegen-vl decompress vlblocks-lz4-fixed-3.0.0.b2frame
$ ./filegen-vl decompress vlblocks-lz4-variable-3.0.0.b2frame
$ ./filegen-vl decompress varchunks-lz4-3.0.0.b2frame
Expected results:
* These VL-block fixtures are only expected to work with releases that support
the VL-block cframe format.
* Pre-vlblocks releases are expected to reject them.
* The ``--regular`` fixture tests regular (non-VL-block) chunks of varying
sizes in the same cframe (frame format v3 ``FRAME_VARIABLE_CHUNKS`` flag).
Tagging
-------
- Create a tag ``X.Y.Z`` from ``main``. Use the next message::
$ git tag -a vX.Y.Z -m "Tagging version X.Y.Z"
- Push the tag to the github repo::
$ git push --tags
- Create a new release visiting https://github.com/Blosc/c-blosc2/releases/new
and add the release notes copying them from `RELEASE_NOTES.md` document.
Check documentation
-------------------
Go to `blogsite actions `_ and trigger a build
by going to the last run and clicking on "Re-run all jobs".
Wait up to 10 min and go to the `blosc2 docs `_
and check that it contains the updated docs.
Announcing
----------
- Send an announcement to the blosc and comp.compression mailing lists.
Use the ``ANNOUNCE.md`` file as skeleton (likely as the definitive version).
- Toot about it from the @Blosc2 account in https://fosstodon.org.
Post-release actions
--------------------
- Edit *VERSION* symbols in include/blosc2.h in main to increment the
version to the next patch one (i.e. X.Y.Z --> X.Y.(Z+1).dev).
- Create new headers for adding new features in ``RELEASE_NOTES.md``
and add this place-holder instead:
#XXX version-specific blurb XXX#
- Commit the changes::
$ git commit -a -m"Post X.Y.Z release actions done"
$ git push
That's all folks!
.. Local Variables:
.. mode: rst
.. coding: utf-8
.. fill-column: 70
.. End:
Blosc-c-blosc2-d3bee7c/ROADMAP-TO-3.0.rst 0000664 0000000 0000000 00000017454 15255167342 0017414 0 ustar 00root root 0000000 0000000 Roadmap to 3.0
==============
.. note::
C-Blosc2 3.0 was released in April 2026, so this document is now mostly
of historical interest. Every item below has been annotated with its
status: **[Done]**, **[Ongoing]** or **[Deferred]**. Deferred items did
not make it into the 3.x series (yet); they are still considered good
candidates for future releases, and most of them have been carried over
to `ROADMAP-TO-4.0.rst
`_ as
current goals.
New Features in C-Blosc2
------------------------
* **[Done]** Adopt `OpenZL `_ as another codec: OpenZL is a new codec that is being developed by Meta. It is meant to be a very fast codec with a good compression ratio, especially for floating point data. Support has been implemented as a separate plugin instead of being part of the C-Blosc2 library itself; see the `blosc2-openzl `_ repository.
* **[Deferred]** Move lazy expressions to C-Blosc2: right now, lazy expressions are only supported in the Python wrapper (python-blosc2). Moving them to C-Blosc2 would allow other wrappers (Java, R, Julia...) to benefit from this powerful feature.
* **[Deferred]** Optimization for multi-socket machines: right now, C-Blosc2 is optimized for single-socket machines. However, in multi-socket machines, memory access is not uniform (NUMA architecture), so optimizations are needed to make sure that every thread is accessing to local memory as much as possible. This would require to use e.g. `numactl `_ or `libnuma `_ so as to pin threads and memory allocations to the local socket.
* **[Deferred]** Support for GPUs: nowadays, GPUs are becoming more and more powerful, and having support for them in C-Blosc2 would be a great addition. The idea is to offload the compression, but most importantly, decompression tasks to the GPU, so that the CPU is free to do other tasks. This would require to use e.g. `CUDA `_ or `ROCm `_ so as to access to the GPU capabilities.
* **[Deferred]** ACID version of the frame format: right now, frames are not ACID compliant (Atomicity, Consistency, Isolation and Durability). This means that if a frame is being written and the process is interrupted (e.g. power failure, crash...), the frame may be left in an inconsistent state. Making frames ACID compliant would require to implement a journaling mechanism so that changes are first written to a journal before being applied to the frame. This would ensure that frames are always in a consistent state, even in the event of a failure. Note that the 3.2 series added related (but weaker) guarantees: growth-SWMR (single writer, multiple readers) support and optional file locking for multi-process access.
* **[Deferred]** Framed version of the `TreeStore `_: the TreeStore is a data structure developed in Python-Blosc2. It is meant to be a hierarchical data structure that allows to store and access data in a tree-like fashion. Making it framed would allow to transfer it more easily between different machines, as well as storing it in memory.
General improvements
--------------------
* **[Ongoing] Improve the safety of the library:** even if we have already made a long way in improving our safety, mainly thanks to the efforts of Nathan Moinvaziri, we take safety seriously, so this is always a work in progress.
* **[Deferred] Checksums:** the frame can benefit from having a checksum per every chunk/index/metalayer. This will provide more safety towards frames that are damaged for whatever reason. Also, this would provide better feedback when trying to determine the parts of the frame that are corrupted. Candidates for checksums can be the xxhash32 or xxhash64, depending on the goals (to be decided).
* **[Deferred] Multiple index chunks in frames:** right now, only `one chunk `_ is allowed for indexing other chunks. Provided the 2GB limit for a chunksize, that means that 'only' 256 million of chunks can be stored in a frame. Allowing for more than one index chunk would overcome this limitation.
* **[Deferred] More robust detection of CPU capabilities:** although currently this detection is quite sophisticated, the code responsible for that has organically grown for more than 10 years and it is time to come with a more modern and robust way of doing this. https://github.com/google/cpu_features may be a good helper for doing this refactoring.
* **[Ongoing] Documentation:** utterly important for attracting new users and making the life easier for existing ones. Important points to have in mind here:
- **Quality of API docstrings:** is the mission of the functions or data structures clearly and succinctly explained? Are all the parameters explained? Is the return value explained? What are the possible errors that can be returned? (mostly completed by Alberto Sabater).
- **Tutorials/book:** besides the API docstrings, more documentation materials should be provided, like tutorials or a book about Blosc (or at least, the beginnings of it). Due to its adoption in GitHub and Jupyter notebooks, one of the most extended and useful markup systems is Markdown, so this should also be the first candidate to use here.
* **[Done] Wrappers for other languages:** the steering `council discussed this `_ and the decision stands: the Python wrapper (`python-blosc2 `_) is maintained by the Blosc maintainers themselves, while wrappers for other languages (Rust, Java, R, Julia...) are left to the community.
* **[Done] Lock support for super-chunks:** implemented in the 3.2 series. Concurrent access from different processes to frame-backed super-chunks can now be serialized with an opt-in file locking mechanism: set the ``locking`` member of ``blosc2_stdio_params``, or the ``BLOSC_LOCKING`` environment variable to enable it globally (it ended up being a runtime opt-in rather than a build-time cmake flag). See the `file-locking.c example `_.
Outreaching
-----------
* **[Done] Improve the Blosc website:** `www.blosc.org `_ was revamped with a modern look, easy navigation and searchable sphinx-based documentation.
* **[Ongoing] Attend to meetings and conferences:** it is very important to plan going to conferences for advertising C-Blosc2 and meeting people in-person. We need to decide which meetings to attend. When on the Python arena, the answer would be quite clear, but for general C libraries like C-Blosc2, it is not that straightforward which ones are the most suited.
* **[Done]** Produce short introductory videos: see the 3-minute `introduction to Blosc2 `_ and the 2-minute video on `slicing pineapple-style `_.
Increase diversity
------------------
**[Ongoing] We strive to make our team as diverse as possible:** we are actively looking into more women and people from a variety of cultures to join our team. We have been very fortunate to have had Marta Iborra, our first female among us; she did a great range of first class contributions, like sparse storage, pre and post filters, new dynamic filters (e.g. JPEG2000), improvements in the Btune prediction engine and many others. After Marta, we had Oumaima Ech.Chdig who enhanced documentation and tutorials, especially for the Python wrapper. Thanks to the Python Software Foundation and NumFOCUS for providing funds for allowing this.
Blosc-c-blosc2-d3bee7c/ROADMAP-TO-4.0.rst 0000664 0000000 0000000 00000010366 15255167342 0017410 0 ustar 00root root 0000000 0000000 Roadmap to 4.0
==============
This document lists the main goals for the C-Blosc2 4.x series. Most of them
are features that were originally planned for 3.0 but were deferred (see
`ROADMAP-TO-3.0.rst `_
for the historical record). Contributions towards any of them are very welcome.
New Features in C-Blosc2
------------------------
* Move lazy expressions to C-Blosc2: right now, lazy expressions are only supported in the Python wrapper (python-blosc2). Moving them to C-Blosc2 would allow other wrappers (Java, R, Julia...) to benefit from this powerful feature.
* Optimization for multi-socket machines: right now, C-Blosc2 is optimized for single-socket machines. However, in multi-socket machines, memory access is not uniform (NUMA architecture), so optimizations are needed to make sure that every thread is accessing to local memory as much as possible. This would require to use e.g. `numactl `_ or `libnuma `_ so as to pin threads and memory allocations to the local socket.
* Support for GPUs: nowadays, GPUs are becoming more and more powerful, and having support for them in C-Blosc2 would be a great addition. The idea is to offload the compression, but most importantly, decompression tasks to the GPU, so that the CPU is free to do other tasks. This would require to use e.g. `CUDA `_ or `ROCm `_ so as to access to the GPU capabilities.
* ACID version of the frame format: right now, frames are not ACID compliant (Atomicity, Consistency, Isolation and Durability). This means that if a frame is being written and the process is interrupted (e.g. power failure, crash...), the frame may be left in an inconsistent state. Making frames ACID compliant would require to implement a journaling mechanism so that changes are first written to a journal before being applied to the frame. This would ensure that frames are always in a consistent state, even in the event of a failure. The 3.2 series already added related (but weaker) guarantees: growth-SWMR (single writer, multiple readers) support and optional file locking for multi-process access.
* Framed version of the `TreeStore `_: the TreeStore is a data structure developed in Python-Blosc2. It is meant to be a hierarchical data structure that allows to store and access data in a tree-like fashion. Making it framed would allow to transfer it more easily between different machines, as well as storing it in memory.
General improvements
--------------------
* **Improve the safety of the library:** even if we have already made a long way in improving our safety, mainly thanks to the efforts of Nathan Moinvaziri, we take safety seriously, so this is always a work in progress.
* **Checksums:** the frame can benefit from having a checksum per every chunk/index/metalayer. This will provide more safety towards frames that are damaged for whatever reason. Also, this would provide better feedback when trying to determine the parts of the frame that are corrupted. Candidates for checksums can be the xxhash32 or xxhash64, depending on the goals (to be decided).
* **Multiple index chunks in frames:** right now, only `one chunk `_ is allowed for indexing other chunks. Provided the 2GB limit for a chunksize, that means that 'only' 256 million of chunks can be stored in a frame. Allowing for more than one index chunk would overcome this limitation.
* **More robust detection of CPU capabilities:** although currently this detection is quite sophisticated, the code responsible for that has organically grown for more than 10 years and it is time to come with a more modern and robust way of doing this. https://github.com/google/cpu_features may be a good helper for doing this refactoring.
* **Documentation:** utterly important for attracting new users and making the life easier for existing ones. Besides keeping the quality of the API docstrings, more documentation materials should be provided, like tutorials or a book about Blosc (or at least, the beginnings of it).
Blosc-c-blosc2-d3bee7c/THANKS.rst 0000664 0000000 0000000 00000003732 15255167342 0016475 0 ustar 00root root 0000000 0000000 Thanks
======
* Valentin Haenel did a terrific work implementing the support for the Snappy compression, fixing typos and improving docs and the plotting script.
* Thibault North, with ideas from Oscar Villellas, contributed a way to call Blosc from different threads in a safe way. Christopher
Speller introduced contexts so that a global lock is not necessary anymore.
* The CMake support was initially contributed by Thibault North, and Antonio Valentino and Mark Wiebe made great enhancements to it.
* Christopher Speller also introduced the two new '_ctx' calls to avoid the use of the blosc_init() and blosc_destroy().
* Jack Pappas contributed important portability enhancements, specially runtime and cross-platform detection of SSE2/AVX2 as well as high precision timers (HPET) for the benchmark program.
* @littlezhou implemented the AVX2 version of shuffle routines.
* Julian Taylor contributed a way to detect AVX2 in runtime and calling the appropriate routines only if the underlying hardware supports it.
* Lucian Marc provided the support for ARM/NEON for the shuffle filter.
* Jerome Kieffer contributed support for PowerPC/ALTIVEC for the shuffle/bitshuffle filter.
* Alberto Sabater, for his great efforts on producing really nice Blosc2 docs, among other aspects.
* Kiyo Masui for relicensing his bitshuffle project for allowing the inclusion of part of his code in Blosc.
* Aleix Alcacer for his implementation of mutable super-chunks, multiple variable length metalayers and many other things.
* Oscar Guiñón for the optimization of reading a (sparse) set of blocks of a chunk in parallel.
* Nathan Moinvaziri for his outstanding work on the security side of the things via `fuzzer testing `_.
* Marta Iborra for her implementation of sparse storage for persistent super-chunks and her attention to detail in many other aspects of the library.
* Dimitri Papadopoulos for an extensive set of improvements in documentation and code.
Blosc-c-blosc2-d3bee7c/TODO-refactorization.txt 0000664 0000000 0000000 00000000111 15255167342 0021446 0 ustar 00root root 0000000 0000000 compcode -> codec
look into the packed offsets and come with an standard Blosc-c-blosc2-d3bee7c/WHATS-NEW.rst 0000664 0000000 0000000 00000020034 15255167342 0017014 0 ustar 00root root 0000000 0000000 New features in C-Blosc2
========================
The following features are already implemented:
* **64-bit containers:** the first-class container in C-Blosc2 is the `super-chunk` or, for brevity, `schunk`, that is made by smaller chunks which are essentially C-Blosc1 32-bit containers. The super-chunk can be backed or not by another container which is called a `frame` (see later).
* **More filters:** besides `shuffle` and `bitshuffle` already present in C-Blosc1, C-Blosc2 already implements:
- `delta`: the stored blocks inside a chunk are diff'ed with respect to first block in the chunk. The idea is that, in some situations, the diff will have more zeros than the original data, leading to better compression.
- `trunc_prec`: it zeroes the least significant bits of the mantissa of float32 and float64 types. When combined with the `shuffle` or `bitshuffle` filter, this leads to more contiguous zeros, which are compressed better.
* **A filter pipeline:** the different filters can be pipelined so that the output of one can be the input for the other. A possible example is a `delta` followed by `shuffle`, or as described above, `trunc_prec` followed by `bitshuffle`.
* **Prefilters:** allows applying user-defined C callbacks **prior to** the filter pipeline during compression. See `test_prefilter.c `_ for an example of use.
* **Postfilters:** allows applying user-defined C callbacks **after** the filter pipeline during decompression. The combination of prefilters and postfilters could be interesting for supporting e.g. encryption (via prefilters) and decryption (via postfilters). Also, a postfilter alone can be used to produce on-the-fly computation based on existing data (or other metadata, like e.g. coordinates). See `test_postfilter.c `_ for an example of use.
* **SIMD support for ARM (NEON):** this allows for faster operation on ARM architectures. Both `shuffle` and `bitshuffle` are supported. Thanks to Lucian Marc.
* **SIMD support for PowerPC (ALTIVEC):** this allows for faster operation on PowerPC architectures. Both `shuffle` and `bitshuffle` are supported; however, this has been done via a transparent mapping from SSE2 into ALTIVEC emulation in GCC 8, so performance could be better (but still, it is already a nice improvement over native C code; see PR https://github.com/Blosc/c-blosc2/pull/59 for details). Thanks to Jerome Kieffer and `ESRF `_ for sponsoring the Blosc team in doing this task.
* **Dictionaries:** when a block is going to be compressed, C-Blosc2 can use a previously made dictionary (stored in the header of the super-chunk) for compressing all the blocks that are part of the chunks. This usually improves the compression ratio, as well as the decompression speed, at the expense of a (small) overhead in compression speed. Currently, it is only supported in the `zstd` codec, but would be nice to extend it to `lz4` and `blosclz` at least.
* **Contiguous frames:** allow to store super-chunks contiguously, either on-disk or in-memory. When a super-chunk is backed by a frame, instead of storing all the chunks sparsely in-memory, they are serialized inside the frame container. The frame can be stored on-disk too, meaning that persistence of super-chunks is supported.
* **Sparse frames:** each chunk in a super-chunk, as well as the metadata, are stored separately. This allows for more efficient updates than in frames (i.e. avoiding 'holes' in monolithic files).
* **Partial chunk reads:** there is support for reading just part of a chunk, so avoiding reading the whole thing and then discarding the unnecessary data (which is a waste of resources).
* **Parallel chunk reads:** when several blocks of a chunk are to be read, this is done in parallel by the decompressing machinery. That means that every thread is responsible to read, post-filter and decompress a block by itself, leading to an efficient overlap of I/O and CPU usage that optimizes reads to a maximum.
* **Meta-layers:** optionally, the user can add meta-data for different uses and in different layers. For example, one may think on providing a meta-layer for `NumPy `_ so that most of the meta-data for it is stored in a meta-layer; then, one can place another meta-layer on top of the latter for adding more high-level info if desired (e.g. geo-spatial, meteorological...).
* **Variable length meta-layers:** the user may want to add variable-length meta information that can be potentially very large (up to 2 GB). The regular meta-layer described above is very quick to read, but meant to store fixed-length and relatively small meta information. Variable length metalayers are stored in the trailer of a frame, whereas regular meta-layers are in the header.
* **Efficient support for special values:** large sequences of repeated values can be represented with an efficient, simple and fast run-length representation, without the need to use regular codecs. With that, chunks or super-chunks with values that are the same (zeros, NaNs or any value in general) can be built in constant time, regardless of the size. This can be useful in situations where a lot of zeros (or NaNs) need to be stored (e.g. sparse matrices).
* **Nice markup for documentation:** we are currently using a combination of Sphinx + Doxygen + Breathe for documenting the C-API. See https://www.blosc.org/c-blosc2/c-blosc2.html. Thanks to Alberto Sabater and Aleix Alcacer for contributing the support for this.
* **Plugin capabilities for filters and codecs:** we have a plugin register capability inplace so that the info about the new filters and codecs can be persisted and transmitted to different machines. Thanks to the NumFOCUS foundation for providing a grant for doing this.
* **Centralized plugin repository:** we have implemented a centralized repository so that people can send their plugins (using the existing machinery) to the Blosc2 team. If the plugins fulfill a series of requirements, they will be officially accepted, and distributed within the library. Thanks to NumFOCUS foundation for providing a grant for doing this. See https://www.blosc.org/posts/registering-plugins/.
* **Support for lossy codecs:** besides supporting the `trunc_prec` filter (described above), we also offer support for `zfp `_, a codec that is specifically meant for lossy compression of *multidimensional* floating point data. For details on how use it, see https://github.com/Blosc/c-blosc2/tree/main/plugins/codecs/zfp. Support for more lossy codecs may come in the future.
* **Pluggable tuning capabilities:** this will allow users with different needs to define an interface so as to better tune different parameters like the codec, the compression level, the filters to use, the blocksize or the shuffle size. Thanks to ironArray for sponsoring us in doing this.
* **Support for I/O plugins:** so that users can extend the I/O capabilities beyond the current filesystem support. Things like use databases or S3 interfaces should be possible by implementing these interfaces. Thanks to ironArray for sponsoring us in doing this.
* **Python wrapper:** the `python-blosc2 `_ package is a full-featured, officially supported wrapper of C-Blosc2 (and it comes with binary wheels of the C-Blosc2 libraries too). Thanks to the Python Software Foundation for providing a grant for kickstarting this.
* **Concurrent access to persistent data:** growth-SWMR (single writer, multiple readers) support allows one process to append to a persistent super-chunk while others read from it, and an opt-in file locking mechanism (via the ``locking`` member of ``blosc2_stdio_params`` or the ``BLOSC_LOCKING`` environment variable) serializes accesses from several processes.
* **Security:** we are actively using the `OSS-Fuzz `_ and `ClusterFuzz `_ for uncovering programming errors in C-Blosc2. Thanks to Google for sponsoring us in doing this.
Blosc-c-blosc2-d3bee7c/bench/ 0000775 0000000 0000000 00000000000 15255167342 0016025 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/bench/CMakeLists.txt 0000664 0000000 0000000 00000020135 15255167342 0020566 0 ustar 00root root 0000000 0000000 # Blosc - Blocked Shuffling and Compression Library
#
# Copyright (c) 2021 Blosc Development Team
# https://blosc.org
# License: BSD 3-Clause (see LICENSE.txt)
#
# See LICENSE.txt for details about copyright and rights to use.
# sources for main bench
set(SOURCES b2bench.c)
# other benchmarks
set(SOURCES_DELTA delta_schunk.c)
set(SOURCES_TRUNC_PREC trunc_prec_schunk.c)
set(SOURCES_SUM_OPENMP sum_openmp.c)
set(SOURCES_ZERO_RUNLEN zero_runlen.c)
set(SOURCES_CFRAME create_frame.c)
set(SOURCES_SFRAME sframe_bench.c)
set(SOURCES_GET_SPARSE get_sparse.c)
set(SOURCES_FRAME_LOCK frame_lock_bench.c)
add_subdirectory(b2nd)
# targets
add_executable(b2bench ${SOURCES})
add_executable(delta_schunk ${SOURCES_DELTA})
add_executable(trunc_prec_schunk ${SOURCES_TRUNC_PREC})
add_executable(sum_openmp ${SOURCES_SUM_OPENMP})
add_executable(zero_runlen ${SOURCES_ZERO_RUNLEN})
add_executable(create_frame ${SOURCES_CFRAME})
add_executable(sframe_bench ${SOURCES_SFRAME})
add_executable(get_sparse ${SOURCES_GET_SPARSE})
add_executable(frame_lock_bench ${SOURCES_FRAME_LOCK})
if(UNIX AND NOT APPLE)
# cmake is complaining about LINK_PRIVATE in original PR
# and removing it does not seem to hurt, so be it.
# target_link_libraries(b2bench LINK_PRIVATE rt)
target_link_libraries(b2bench rt)
target_link_libraries(delta_schunk rt)
target_link_libraries(trunc_prec_schunk rt)
target_link_libraries(sum_openmp rt)
target_link_libraries(zero_runlen rt)
target_link_libraries(create_frame rt)
target_link_libraries(sframe_bench rt)
target_link_libraries(get_sparse rt)
target_link_libraries(frame_lock_bench rt)
endif()
if(UNIX)
if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
# Avoid a warning when using gcc without -fopenmp
target_compile_options(sum_openmp PRIVATE "-Wno-unknown-pragmas")
endif()
endif()
target_link_libraries(b2bench blosc_testing)
target_link_libraries(delta_schunk blosc_testing)
target_link_libraries(trunc_prec_schunk blosc_testing)
target_link_libraries(sum_openmp blosc_testing)
target_link_libraries(zero_runlen blosc_testing)
target_link_libraries(create_frame blosc_testing)
target_link_libraries(sframe_bench blosc_testing)
target_link_libraries(get_sparse blosc_testing)
target_link_libraries(frame_lock_bench blosc_testing)
add_executable(bench_compact_tail bench_compact_tail.c)
target_link_libraries(bench_compact_tail blosc_testing)
if(UNIX AND NOT APPLE)
target_link_libraries(bench_compact_tail rt)
endif()
# tests
if(BUILD_TESTS)
# The commented tests below take too much time to complete
option(TEST_INCLUDE_BENCH_SHUFFLE_1 "Include b2bench shuffle (1 thread) in the tests" ON)
if(TEST_INCLUDE_BENCH_SHUFFLE_1)
set(SHUFFLE_1_OPTS shuffle test 1)
add_test(NAME test_blosclz_shuffle_1
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ blosclz ${SHUFFLE_1_OPTS})
add_test(NAME test_lz4_shuffle_1
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4 ${SHUFFLE_1_OPTS})
# add_test(NAME test_lz4hc_shuffle_1
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4hc ${SHUFFLE_1_OPTS})
if(HAVE_ZLIB)
# add_test(NAME test_zlib_shuffle_1
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zlib ${SHUFFLE_1_OPTS})
endif()
if(HAVE_ZSTD)
# add_test(NAME test_zstd_shuffle_1
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zstd ${SHUFFLE_1_OPTS})
endif()
endif()
option(TEST_INCLUDE_BENCH_SHUFFLE_N "Include b2bench shuffle (multithread) in the tests" ON)
if(TEST_INCLUDE_BENCH_SHUFFLE_N)
set(SHUFFLE_N_OPTS shuffle test)
add_test(NAME test_blosclz_shuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ blosclz ${SHUFFLE_N_OPTS})
add_test(NAME test_lz4_shuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4 ${SHUFFLE_N_OPTS})
add_test(NAME test_lz4hc_shuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4hc ${SHUFFLE_N_OPTS})
if(HAVE_ZLIB)
add_test(NAME test_zlib_shuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zlib ${SHUFFLE_N_OPTS})
endif()
if(HAVE_ZSTD)
add_test(NAME test_zstd_shuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zstd ${SHUFFLE_N_OPTS})
endif()
endif()
option(TEST_INCLUDE_BENCH_BITSHUFFLE_1 "Include b2bench bitshuffle (1 thread) in the tests" ON)
if(TEST_INCLUDE_BENCH_BITSHUFFLE_1)
set(BITSHUFFLE_1_OPTS bitshuffle test 1)
add_test(NAME test_blosclz_bitshuffle_1
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ blosclz ${BITSHUFFLE_1_OPTS})
add_test(NAME test_lz4_bitshuffle_1
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4 ${BITSHUFFLE_1_OPTS})
# add_test(NAME test_lz4hc_bitshuffle_1
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4hc ${BITSHUFFLE_1_OPTS})
if(HAVE_ZLIB)
# add_test(NAME test_zlib_bitshuffle_1
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zlib ${BITSHUFFLE_1_OPTS})
endif()
if(HAVE_ZSTD)
# add_test(NAME test_zstd_bitshuffle_1
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zstd ${BITSHUFFLE_1_OPTS})
endif()
endif()
option(TEST_INCLUDE_BENCH_BITSHUFFLE_N "Include b2bench bitshuffle (multithread) in the tests" ON)
if(TEST_INCLUDE_BENCH_BITSHUFFLE_N)
set(BITSHUFFLE_N_OPTS bitshuffle test)
add_test(NAME test_blosclz_bitshuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ blosclz ${BITSHUFFLE_N_OPTS})
add_test(NAME test_lz4_bitshuffle_n
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4 ${BITSHUFFLE_N_OPTS})
# add_test(NAME test_lz4hc_bitshuffle_n
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ lz4hc ${BITSHUFFLE_N_OPTS})
if(HAVE_ZLIB)
# add_test(NAME test_zlib_bitshuffle_n
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zlib ${BITSHUFFLE_N_OPTS})
endif()
if(HAVE_ZSTD)
# add_test(NAME test_zstd_bitshuffle_n
# COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ zstd ${BITSHUFFLE_N_OPTS})
endif()
endif()
option(TEST_INCLUDE_BENCH_SUITE "Include b2bench suite in the tests" OFF)
if(TEST_INCLUDE_BENCH_SUITE)
add_test(NAME test_hardsuite
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ blosclz shuffle suite)
endif()
option(TEST_INCLUDE_BENCH_DEBUGSUITE "Include b2bench debugsuite in the tests" OFF)
if(TEST_INCLUDE_BENCH_DEBUGSUITE)
add_test(NAME test_debugsuite
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ blosclz shuffle debugsuite)
endif()
option(TEST_INCLUDE_BENCH_DELTA "Include delta bench in the tests" ON)
if(TEST_INCLUDE_BENCH_DELTA)
add_test(NAME test_bench_delta
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $)
endif()
option(TEST_INCLUDE_BENCH_TRUNC_PREC "Include trunc bench in the tests" ON)
if(TEST_INCLUDE_BENCH_TRUNC_PREC)
add_test(NAME test_bench_trunc_prec
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $)
endif()
option(TEST_INCLUDE_BENCH_SUM_OPENMP "Include sum_openmp in the tests" OFF)
if(TEST_INCLUDE_BENCH_SUM_OPENMP)
add_test(NAME test_bench_sum_openmp
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $)
endif()
option(TEST_INCLUDE_BENCH_ZERO_RUNLEN "Include zero_runlen in the tests" ON)
if(TEST_INCLUDE_BENCH_ZERO_RUNLEN)
add_test(NAME test_bench_zero_runlen
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $)
endif()
endif()
Blosc-c-blosc2-d3bee7c/bench/Makefile.mingw 0000664 0000000 0000000 00000001504 15255167342 0020605 0 ustar 00root root 0000000 0000000 # Makefile for the MinGW suite for Windows
CC = g++ # or clang
CFLAGS = -O3 -g -msse2 -Wall
#LDFLAGS = -lpthread # for UNIX or for Windows with pthread installed
LDFLAGS = -static # for mingw
SOURCES = $(wildcard ../blosc/*.c)
EXECUTABLE = bench
# Support for external LZ4
#LDFLAGS += -llz4
# Support for the msvc zlib:
ZLIB_ROOT=/libs/zlib128
LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lzdll -L$(ZLIB_ROOT)/lib
# Support for the mingw zlib:
#ZLIB_ROOT=/libs/libz-1.2.8
#LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lz -L$(ZLIB_ROOT)/lib
# Support for internal Zlib
#ZLIB_DIR = ../internal-complibs/zlib-1.2.8
#CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR)
#SOURCES += $(wildcard $(ZLIB_DIR)/*.c)
all: $(SOURCES) $(EXECUTABLE)
$(EXECUTABLE): $(SOURCES)
$(CC) $(CFLAGS) bench.c $(SOURCES) -o $@ $(LDFLAGS)
clean:
rm -rf $(EXECUTABLE)
Blosc-c-blosc2-d3bee7c/bench/aggregate_multiple.py 0000664 0000000 0000000 00000005757 15255167342 0022256 0 ustar 00root root 0000000 0000000 from pathlib import Path
import re
import os
import subprocess
from typing import Union
import numpy as np
import argparse
def run_command(command: str, return_output: bool = False) -> Union[tuple[list[float], str], tuple[list[float]]]:
bench_folder = Path(__file__).parent.parent / "build" / "bench"
if os.name == "nt":
bench_folder /= "Release"
res = subprocess.run(f"{bench_folder}/{command}", shell=True, capture_output=True)
assert res.returncode == 0, f"\nstdout: {res.stdout.decode('utf-8')}\nstderr: {res.stderr.decode('utf-8')}"
out = res.stdout.decode("utf-8")
numbers = []
for number in re.findall(r"-?\d+(?:\.\d+)?", out):
numbers.append(float(number))
assert len(numbers) > 0, f"Could not find any numbers in the output:\n{out}"
if return_output:
return numbers, out
else:
return numbers
class ReplaceNumbers:
def __init__(self, numbers: np.ndarray):
self.numbers = numbers
self.index = 0
def __call__(self, match: re.Match) -> str:
assert match.group() == "NUMBER"
number = self.numbers[self.index]
self.index += 1
if number.is_integer():
return str(int(number))
else:
return f"{number:.3f}"
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description=(
"Repeat multiple runs of a benchmark and aggregate all numbers in the output (the outputs from all runs must "
"have the same format)."
),
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--command",
required=True,
type=str,
help="The command to execute (e.g. sframe_bench 1000 insert 1000 io_mmap).",
)
parser.add_argument(
"--warmups",
default=3,
required=False,
type=int,
help="Number of warmup runs.",
)
parser.add_argument(
"--runs",
default=30,
required=False,
type=int,
help="Number of runs to perform and aggregate the output from.",
)
args = parser.parse_args()
for i in range(args.warmups):
print(f"Warmup {i+1}/{args.warmups}", end="\r")
run_command(args.command)
print()
# Collect numbers from outputs of multiple runs
example_output = None
numbers = []
for i in range(args.runs):
print(f"Run {i+1}/{args.runs}", end="\r")
if example_output is None:
values, text = run_command(args.command, return_output=True)
example_output = text
numbers.append(values)
else:
numbers.append(run_command(args.command))
print()
numbers = np.asarray(numbers)
numbers = np.min(numbers, axis=0)
assert example_output is not None, "No output was collected."
example_output = re.sub(r"-?\d+(?:\.\d+)?", "NUMBER", example_output)
aggregated_output = re.sub("NUMBER", ReplaceNumbers(numbers), example_output)
print(aggregated_output)
Blosc-c-blosc2-d3bee7c/bench/b2bench.c 0000664 0000000 0000000 00000040613 15255167342 0017500 0 ustar 00root root 0000000 0000000 /*********************************************************************
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Small benchmark for testing basic capabilities of Blosc.
You can select different degrees of 'randomness' in input buffer, as
well as external datafiles (uncomment the lines after "For data
coming from a file" comment).
For usage instructions of this benchmark, please see:
https://www.blosc.org/pages/synthetic-benchmarks/
I'm collecting speeds for different machines, so the output of your
benchmarks and your processor specifications are welcome!
Note: Compiling this with VS2008 does not work well with cmake. Here
it is a way to compile the benchmark (with added support for LZ4):
> cl /arch:SSE2 /Ox /Febench.exe /Iblosc /Iinternal-complibs\lz4-1.7.0 bench\bench.c blosc\blosc.c blosc\blosclz.c blosc\shuffle.c blosc\shuffle-sse2.c blosc\shuffle-generic.c blosc\bitshuffle-generic.c blosc\bitshuffle-sse2.c internal-complibs\lz4-1.7.0\*.c
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
#include
#include
#include
#include
#include
#include "blosc2.h"
#define KB 1024u
#define MB (1024*KB)
#define GB (1024*MB)
#define NCHUNKS (32*1024) /* maximum number of chunks */
int nchunks = NCHUNKS;
int niter = 1;
int niter_c = 1;
int niter_d = 1;
/* default number of iterations */
double totalsize = 0.; /* total compressed/decompressed size */
/* Define posix_memalign for Windows */
#if defined(_WIN32)
#include
int posix_memalign(void **memptr, size_t alignment, size_t size)
{
*memptr = _aligned_malloc(size, alignment);
return 0;
}
/* Buffers allocated with _aligned_malloc need to be freed with _aligned_free. */
#define aligned_free(memptr) _aligned_free(memptr)
#else
/* If not using MSVC, aligned memory can be freed in the usual way. */
#define aligned_free(memptr) free(memptr)
#endif /* defined(_WIN32) && !defined(__MINGW32__) */
/* Given two timeval stamps, return the time per chunk in usec */
double get_usec_chunk(blosc_timestamp_t last, blosc_timestamp_t current,
int niter_, int nchunks_) {
double elapsed_usecs = 1e-3 * blosc_elapsed_nsecs(last, current);
return elapsed_usecs / (double)(niter_ * nchunks_);
}
int get_value(int i, int rshift) {
int v;
v = (i << 26) ^ (i << 18) ^ (i << 11) ^ (i << 3) ^ i;
if (rshift < 32) {
v &= (1 << rshift) - 1;
}
return v;
}
void init_buffer(void* src, size_t size, int rshift) {
int i;
int* _src = (int*)src;
/* To have reproducible results */
srand(1);
/* Initialize the original buffer */
for (i = 0; i < (int) (size / sizeof(int)); ++i) {
/* Choose one below */
/* _src[i] = 0;
* _src[i] = 0x01010101;
* _src[i] = 0x01020304;
* _src[i] = i * 1/.3;
* _src[i] = i; */
//_src[i] = rand() >> (32 - rshift);
_src[i] = get_value(i, rshift);
}
}
void do_bench(char* compressor, char* shuffle, int nthreads, int size_, int elsize,
int rshift, FILE* ofile) {
size_t size = (size_t)size_;
void* src = NULL, *srccpy;
void* dest[NCHUNKS], *dest2 = NULL;
int nbytes = 0, cbytes = 0;
int i, j, retcode;
unsigned char* orig, * round;
blosc_timestamp_t last, current;
double tmemcpy, tshuf, tunshuf;
int clevel, doshuffle = BLOSC_NOFILTER;
if (strcmp(shuffle, "shuffle") == 0) {
doshuffle = BLOSC_SHUFFLE;
}
else if (strcmp(shuffle, "bitshuffle") == 0) {
doshuffle = BLOSC_BITSHUFFLE;
}
else if (strcmp(shuffle, "noshuffle") == 0) {
doshuffle = BLOSC_NOSHUFFLE;
}
blosc2_set_nthreads((int16_t) nthreads);
if (blosc1_set_compressor(compressor) < 0) {
printf("Compiled w/o support for compressor: '%s', so sorry.\n",
compressor);
exit(1);
}
/* Initialize buffers */
srccpy = malloc(size);
retcode = posix_memalign(&src, 32, size);
if (retcode != 0) {
printf("Error in allocating memory!");
free(srccpy);
return;
}
retcode = posix_memalign(&dest2, 32, size);
if (retcode != 0) {
printf("Error in allocating memory!");
aligned_free(src);
free(srccpy);
return;
}
/* zero src to initialize all bytes on it, and not only multiples of 4 */
memset(src, 0, size);
init_buffer(src, size, rshift);
memcpy(srccpy, src, size);
for (j = 0; j < nchunks; j++) {
retcode = posix_memalign(&dest[j], 32, size + BLOSC2_MAX_OVERHEAD);
if (retcode != 0) {
printf("Error in allocating memory!");
}
}
memset(dest2, 0, size); // just to avoid some GCC compiler warnings
fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, (int)size, elsize, rshift, compressor, shuffle);
fprintf(ofile, "********************** Run info ******************************\n");
fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE);
fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift);
fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", (int)size, elsize);
fprintf(ofile, "Working set: %.1f MB\t\t", (float)(size * nchunks) / (float)MB);
fprintf(ofile, "Number of threads: %d\n", nthreads);
fprintf(ofile, "********************** Running benchmarks *********************\n");
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, ((float)size * 1e6) / (tmemcpy * MB));
blosc_set_timestamp(&last);
for (i = 0; i < niter; i++) {
for (j = 0; j < nchunks; j++) {
memcpy(dest2, dest[j], size);
}
}
blosc_set_timestamp(¤t);
tmemcpy = get_usec_chunk(last, current, niter, nchunks);
fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n",
tmemcpy, ((float)size * 1e6) / (tmemcpy * MB));
for (clevel = 0; clevel < 10; clevel++) {
fprintf(ofile, "Compression level: %d\n", clevel);
blosc_set_timestamp(&last);
for (i = 0; i < niter_c; i++) {
for (j = 0; j < nchunks; j++) {
cbytes = blosc1_compress(clevel, doshuffle, (size_t)elsize, size, src,
dest[j], size + BLOSC2_MAX_OVERHEAD);
}
}
blosc_set_timestamp(¤t);
tshuf = get_usec_chunk(last, current, niter_c, nchunks);
fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ",
tshuf, ((float)size * 1e6) / (tshuf * MB));
fprintf(ofile, "Final bytes: %d ", cbytes);
if (cbytes > 0) {
fprintf(ofile, "Ratio: %3.2f", (float)size / (float)cbytes);
}
fprintf(ofile, "\n");
/* Compressor was unable to compress. Copy the buffer manually. */
if (cbytes == 0) {
for (j = 0; j < nchunks; j++) {
memcpy(dest[j], src, size);
}
}
blosc_set_timestamp(&last);
for (i = 0; i < niter_d; i++) {
for (j = 0; j < nchunks; j++) {
if (cbytes == 0) {
memcpy(dest2, dest[j], size);
nbytes = (int)size;
}
else {
nbytes = blosc1_decompress(dest[j], dest2, size);
}
}
}
blosc_set_timestamp(¤t);
tunshuf = get_usec_chunk(last, current, niter_d, nchunks);
fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ",
tunshuf, (nbytes * 1e6) / (tunshuf * MB));
if (nbytes < 0) {
fprintf(ofile, "FAILED. Error code: %d\n", nbytes);
}
/* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */
/* Check if data has had a good roundtrip.
Byte-by-byte comparison is slow, so use 'memcmp' to check whether the
roundtripped data is correct. If not, fall back to the slow path to
print diagnostic messages. */
orig = (unsigned char*)srccpy;
round = (unsigned char*)dest2;
if (memcmp(orig, round, size) != 0) {
for (i = 0; i < (int)size; ++i) {
if (orig[i] != round[i]) {
fprintf(ofile, "\nError: Original data and round-trip do not match in pos %d\n", i);
fprintf(ofile, "Orig--> %x, round-trip--> %x\n", orig[i], round[i]);
break;
}
}
}
else {
i = (int)size;
}
if (i == (int)size) fprintf(ofile, "OK\n");
} /* End clevel loop */
/* To compute the totalsize, we should take into account the 10
compression levels */
totalsize += ((double)size * nchunks * niter * 10.);
aligned_free(src);
free(srccpy);
aligned_free(dest2);
for (i = 0; i < nchunks; i++) {
aligned_free(dest[i]);
}
}
/* Compute a sensible value for nchunks */
int get_nchunks(int size_, int ws) {
int nchunks_;
nchunks_ = ws / size_;
if (nchunks_ > NCHUNKS) nchunks_ = NCHUNKS;
if (nchunks_ < 1) nchunks_ = 1;
return nchunks_;
}
void print_compress_info(void) {
char* name = NULL, * version = NULL;
int ret;
printf("Blosc version: %s (%s)\n", BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE);
printf("List of supported compressors in this build: %s\n",
blosc2_list_compressors());
printf("Supported compression libraries:\n");
ret = blosc2_get_complib_info("blosclz", &name, &version);
if (ret >= 0) printf(" %s: %s\n", name, version);
free(name); free(version);
ret = blosc2_get_complib_info("lz4", &name, &version);
if (ret >= 0) printf(" %s: %s\n", name, version);
free(name); free(version);
ret = blosc2_get_complib_info("zlib", &name, &version);
if (ret >= 0) printf(" %s: %s\n", name, version);
free(name); free(version);
ret = blosc2_get_complib_info("zstd", &name, &version);
if (ret >= 0) printf(" %s: %s\n", name, version);
free(name); free(version);
}
int main(int argc, char* argv[]) {
char compressor[32];
char shuffle[32] = "shuffle";
char bsuite[32];
int single = 1;
int suite = 0;
int hard_suite = 0;
int extreme_suite = 0;
int debug_suite = 0;
int nthreads = 8; /* The number of threads */
int size = 8 * MB; /* Buffer size */
int elsize = 4; /* Datatype size */
int rshift = 19; /* Significant bits */
int workingset = 256 * MB; /* The maximum allocated memory */
int nthreads_, size_, elsize_, rshift_, i;
FILE* output_file = stdout;
blosc_timestamp_t last, current;
double totaltime;
char usage[256];
print_compress_info();
strncpy(usage, "Usage: bench [blosclz | lz4 | lz4hc | zlib | zstd] "
"[noshuffle | shuffle | bitshuffle] "
"[single | suite | hardsuite | extremesuite | debugsuite] "
"[nthreads] [bufsize(bytes)] [typesize] [sbits]", 255);
if (argc < 1) {
printf("%s\n", usage);
exit(1);
}
if (argc >= 2) {
strcpy(compressor, argv[1]);
}
else {
strcpy(compressor, "blosclz");
}
if (strcmp(compressor, "blosclz") != 0 &&
strcmp(compressor, "lz4") != 0 &&
strcmp(compressor, "lz4hc") != 0 &&
strcmp(compressor, "zlib") != 0 &&
strcmp(compressor, "zstd") != 0) {
printf("No such compressor: '%s'\n", compressor);
printf("%s\n", usage);
exit(2);
}
if (argc >= 3) {
strcpy(shuffle, argv[2]);
if (strcmp(shuffle, "shuffle") != 0 &&
strcmp(shuffle, "bitshuffle") != 0 &&
strcmp(shuffle, "noshuffle") != 0) {
printf("No such shuffler: '%s'\n", shuffle);
printf("%s\n", usage);
exit(2);
}
}
if (argc < 4)
strcpy(bsuite, "single");
else
strcpy(bsuite, argv[3]);
if (strcmp(bsuite, "single") == 0) {
single = 1;
}
else if (strcmp(bsuite, "test") == 0) {
single = 1;
workingset /= 2;
}
else if (strcmp(bsuite, "suite") == 0) {
suite = 1;
}
else if (strcmp(bsuite, "hardsuite") == 0) {
hard_suite = 1;
workingset /= 4;
/* Values here are ending points for loops */
nthreads = 2;
size = 8 * MB;
elsize = 32;
rshift = 32;
}
else if (strcmp(bsuite, "extremesuite") == 0) {
extreme_suite = 1;
workingset /= 8;
niter = 1;
/* Values here are ending points for loops */
nthreads = 4;
size = 16 * MB;
elsize = 32;
rshift = 32;
}
else if (strcmp(bsuite, "debugsuite") == 0) {
debug_suite = 1;
workingset /= 8;
niter = 1;
/* Warning: values here are starting points for loops. This is
useful for debugging. */
nthreads = 1;
size = 16 * KB;
elsize = 1;
rshift = 0;
}
else {
printf("%s\n", usage);
exit(1);
}
printf("Using compressor: %s\n", compressor);
printf("Using shuffle type: %s\n", shuffle);
printf("Running suite: %s\n", bsuite);
if (argc >= 5) {
nthreads = (int)strtol(argv[4], NULL, 10);
}
if (argc >= 6) {
size = (int)strtol(argv[5], NULL, 10);
}
if (argc >= 7) {
elsize = (int)strtol(argv[6], NULL, 10);
}
if (argc >= 8) {
rshift = (int)strtol(argv[7], NULL, 10);
}
if ((argc >= 9) || !(single || suite || hard_suite || extreme_suite)) {
printf("%s\n", usage);
exit(1);
}
nchunks = get_nchunks(size, workingset);
blosc_set_timestamp(&last);
blosc2_init();
if (suite) {
for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, output_file);
}
}
else if (hard_suite) {
/* Let's start the rshift loop by 4 so that 19 is visited. This
is to allow a direct comparison with the plain suite, that runs
precisely at 19 significant bits. */
for (rshift_ = 4; rshift_ <= rshift; rshift_ += 5) {
for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) {
/* The next loop is for getting sizes that are not power of 2 */
for (i = -elsize_; i <= elsize_; i += elsize_) {
for (size_ = 32 * KB; size_ <= size; size_ *= 2) {
nchunks = get_nchunks(size_ + i, workingset);
niter = 1;
for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
do_bench(compressor, shuffle, nthreads_, size_ + i, elsize_, rshift_, output_file);
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n",
totaltime, totalsize / GB);
}
}
}
}
}
}
else if (extreme_suite) {
for (rshift_ = 0; rshift_ <= rshift; rshift_++) {
for (elsize_ = 1; elsize_ <= elsize; elsize_++) {
/* The next loop is for getting sizes that are not power of 2 */
for (i = -elsize_ * 2; i <= elsize_ * 2; i += elsize_) {
for (size_ = 32 * KB; size_ <= size; size_ *= 2) {
nchunks = get_nchunks(size_ + i, workingset);
for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) {
do_bench(compressor, shuffle, nthreads_, size_ + i, elsize_, rshift_, output_file);
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n",
totaltime, totalsize / GB);
}
}
}
}
}
}
else if (debug_suite) {
for (rshift_ = rshift; rshift_ <= 32; rshift_++) {
for (elsize_ = elsize; elsize_ <= 32; elsize_++) {
/* The next loop is for getting sizes that are not power of 2 */
for (i = -elsize_ * 2; i <= elsize_ * 2; i += elsize_) {
for (size_ = size; size_ <= (int) (16 * MB); size_ *= 2) {
nchunks = get_nchunks(size_ + i, workingset);
for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) {
do_bench(compressor, shuffle, nthreads_, size_ + i, elsize_, rshift_, output_file);
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n",
totaltime, totalsize / GB);
}
}
}
}
}
}
/* Single mode */
else {
do_bench(compressor, shuffle, nthreads, size, elsize, rshift, output_file);
}
/* Print out some statistics */
blosc_set_timestamp(¤t);
totaltime = (float)blosc_elapsed_secs(last, current);
printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB);
printf("Elapsed time:\t %6.1f s, %.1f MB/s\n",
totaltime, totalsize * 2 * 1.1 / (MB * totaltime));
/* Free blosc resources */
blosc2_free_resources();
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/b2nd/ 0000775 0000000 0000000 00000000000 15255167342 0016652 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/bench/b2nd/CMakeLists.txt 0000664 0000000 0000000 00000001024 15255167342 0021407 0 ustar 00root root 0000000 0000000 # Blosc - Blocked Shuffling and Compression Library
#
# Copyright (c) 2021 Blosc Development Team
# https://blosc.org
# License: BSD 3-Clause (see LICENSE.txt)
#
# See LICENSE.txt for details about copyright and rights to use.
file(GLOB SOURCES bench_*.c)
foreach (source ${SOURCES})
get_filename_component(target_name ${source} NAME_WE)
set(target b2nd_${target_name})
add_executable(${target} ${target_name}.c)
target_link_libraries(${target} PUBLIC blosc_testing ${LIBS})
endforeach (source)
Blosc-c-blosc2-d3bee7c/bench/b2nd/bench_concatenate.c 0000664 0000000 0000000 00000007542 15255167342 0022451 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
// Benchmark for concatenating b2nd arrays. This is to check the fast path
// that has been added to b2nd_concat() that allows for faster concatenation of
// b2nd arrays when there are no partial (or zero-padded) chunks in the arrays being
// concatenated.
#include
#include "blosc2.h"
#include "b2nd.h"
int main() {
blosc2_init();
const int width = 1000;
const int height = 1000;
const int nimages_inbuf = 10;
int64_t N_images = 1000;
bool copy = false; // whether to copy the data or expand src1
// Shapes of the b2nd array
// int64_t shape[] = {N_images, height, width};
// The initial shape of the array before concatenation
int64_t shape[] = {nimages_inbuf, height, width};
int32_t chunkshape[] = {nimages_inbuf, height, width};
int32_t blockshape[] = {1, height, width};
// Determine the buffer size of the image (in bytes)
const int64_t buffersize = nimages_inbuf * height * width * (int64_t)sizeof(uint16_t);
uint16_t* image = malloc(buffersize);
// Generate data
for (int j = 0; j < nimages_inbuf * height * width; j++) {
image[j] = j;
}
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.typesize = sizeof(image[0]);
blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS;
char *urlpath = "bench_concat.b2nd";
//storage.urlpath = urlpath; // for storing in a file
storage.cparams = &cparams;
char *accel_str;
double t, t_accel;
for (int accel=0; accel <= 1; accel++) {
int32_t new_chunkshape[3] = {chunkshape[0], chunkshape[1], chunkshape[2]};
if (!accel) {
new_chunkshape[0] = chunkshape[0] + 1; // to avoid the fast path
accel_str = "non-fast path";
}
else {
accel_str = "fast path";
}
blosc2_remove_urlpath(urlpath);
b2nd_context_t *ctx = b2nd_create_ctx(&storage, 3,
shape, new_chunkshape, blockshape,
"|u2", DTYPE_NUMPY_FORMAT,
NULL, 0);
// The first array
b2nd_array_t *src1;
if (b2nd_empty(ctx, &src1) < 0) {
printf("Error in b2nd_empty\n");
return -1;
}
// The second array, with the data in buffersize
b2nd_array_t *src2;
int ret = b2nd_from_cbuffer(ctx, &src2, image, buffersize);
if (ret < 0) {
printf("Error in b2nd_from_cbuffer\n");
return -1;
}
// Concatenate all images
b2nd_array_t *array;
blosc_timestamp_t t0, t1;
blosc_set_timestamp(&t0);
for (int i = 1; i < N_images / nimages_inbuf; i++) {
if (b2nd_concatenate(ctx, src1, src2, 0, copy, &array) < 0) {
printf("Error in b2nd_concatenate\n");
return -1;
}
if (copy) {
// If we copy, then we need to free the src1 array
b2nd_free(src1);
}
src1 = array;
}
blosc_set_timestamp(&t1);
if (!accel) {
t = blosc_elapsed_secs(t0, t1);
printf("Time to append (%s): %.4f s\n", accel_str, t);
}
else {
t_accel = blosc_elapsed_secs(t0, t1);
printf("Time to append (%s): %.4f s\n", accel_str, t_accel);
}
printf("Number of chunks: %" PRId64 "\n", array->sc->nchunks);
// printf("Shape of array: (%" PRId64 ", %" PRId64 ", %" PRId64 ")\n",
// array->shape[0], array->shape[1], array->shape[2]);
b2nd_free(src2);
if (copy) {
b2nd_free(array);
}
b2nd_free_ctx(ctx);
}
free(image);
blosc2_remove_urlpath(urlpath);
printf("Spedup: %.2fx\n", t / t_accel);
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/b2nd/bench_get_slice.c 0000664 0000000 0000000 00000004437 15255167342 0022123 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
#define DATA_TYPE int64_t
# include
int main() {
blosc_timestamp_t t0, t1;
blosc2_init();
int nslices = 10;
int8_t ndim = 3;
int32_t itemsize = sizeof(DATA_TYPE);
int64_t shape[] = {1250, 745, 400};
int32_t chunkshape[] = {50, 150, 100};
int32_t blockshape[] = {13, 21, 30};
int64_t nbytes = itemsize;
for (int i = 0; i < ndim; ++i) {
nbytes *= shape[i];
}
DATA_TYPE *src = malloc(nbytes);
for (int i = 0; i < nbytes / itemsize; ++i) {
src[i] = i;
}
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.nthreads = 4;
cparams.typesize = itemsize;
blosc2_storage b2_storage = {.cparams=&cparams};
b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape, chunkshape, blockshape, NULL, 0,
NULL, 0);
b2nd_array_t *arr;
blosc_set_timestamp(&t0);
BLOSC_ERROR(b2nd_from_cbuffer(ctx, &arr, src, nbytes));
blosc_set_timestamp(&t1);
printf("from_buffer: %.4f s\n", blosc_elapsed_secs(t0, t1));
blosc_set_timestamp(&t0);
for (int dim = 0; dim < ndim; ++dim) {
int64_t slice_start[B2ND_MAX_DIM], slice_stop[B2ND_MAX_DIM], slice_shape[B2ND_MAX_DIM];
int64_t buffersize = itemsize;
for (int j = 0; j < ndim; ++j) {
slice_start[j] = 0;
slice_stop[j] = j == dim ? 1 : shape[j];
slice_shape[j] = slice_stop[j] - slice_start[j];
buffersize *= slice_shape[j];
}
DATA_TYPE *buffer = malloc(buffersize);
for (int slice = 0; slice < nslices; ++slice) {
slice_start[dim] = rand() % shape[dim];
slice_stop[dim] = slice_start[dim] + 1;
BLOSC_ERROR(b2nd_get_slice_cbuffer(arr, slice_start, slice_stop, buffer, slice_shape, buffersize));
}
free(buffer);
}
blosc_set_timestamp(&t1);
printf("get_slice: %.4f s\n", blosc_elapsed_secs(t0, t1));
free(src);
BLOSC_ERROR(b2nd_free(arr));
BLOSC_ERROR(b2nd_free_ctx(ctx));
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/b2nd/bench_stack_append.c 0000664 0000000 0000000 00000006356 15255167342 0022623 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
// Benchmark for appending data to a b2nd array. A new accelerated path has been
// added to b2nd_append() that allows for faster appending of data to a b2nd array
// when data to append is of the same size as the chunkshape. This benchmark
// compares the performance of the new accelerated path with the old one.
#include
#include "blosc2.h"
#include "b2nd.h"
int main() {
blosc2_init();
const int width = 512;
const int height = 256;
const int nimages_inbuf = 10;
const int64_t buffershape[] = {nimages_inbuf, height, width};
int64_t N_images = 1000;
// Shapes of the b2nd array
int64_t shape[] = {N_images, height, width};
int32_t chunkshape[] = {nimages_inbuf, height, width};
int32_t blockshape[] = {1, height, width};
// Determine the buffer size of the image (in bytes)
const int64_t buffersize = nimages_inbuf * height * width * (int64_t)sizeof(uint16_t);
uint16_t* image = malloc(buffersize);
// Generate data
for (int j = 0; j < nimages_inbuf * height * width; j++) {
image[j] = j;
}
char *urlpath = "bench_stack_append.b2nd";
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.typesize = sizeof(image[0]);
blosc2_storage storage = BLOSC2_STORAGE_DEFAULTS;
//storage.contiguous = true; // for a single file in output
//storage.urlpath = urlpath;
storage.cparams = &cparams;
char *accel_str = "non-accel";
for (int accel=0; accel < 2; accel++) {
if (accel) {
shape[0] = 0;
accel_str = "accel";
}
blosc2_remove_urlpath(urlpath);
b2nd_context_t *ctx = b2nd_create_ctx(&storage, 3,
shape, chunkshape, blockshape,
"|u2", DTYPE_NUMPY_FORMAT,
NULL, 0);
b2nd_array_t *src;
if (b2nd_empty(ctx, &src) < 0) {
printf("Error in b2nd_empty\n");
return -1;
}
// loop through all images
blosc_timestamp_t t0, t1;
blosc_set_timestamp(&t0);
for (int i = 0; i < N_images / nimages_inbuf; i++) {
if (accel) {
if (b2nd_append(src, image, buffersize, 0) < 0) {
printf("Error in b2nd_append\n");
return -1;
}
} else {
int64_t start[] = {i, 0, 0};
int64_t stop[] = {i + nimages_inbuf, height, width};
if (b2nd_set_slice_cbuffer(image, buffershape, buffersize, start, stop, src) < 0) {
printf("Error in b2nd_append\n");
return -1;
}
}
}
blosc_set_timestamp(&t1);
printf("Time to append (%s): %.4f s\n", accel_str, blosc_elapsed_secs(t0, t1));
printf("Number of chunks: %" PRId64 "\n", src->sc->nchunks);
printf("Shape of array: (%" PRId64 ", %" PRId64 ", %" PRId64 ")\n",
src->shape[0], src->shape[1], src->shape[2]);
b2nd_free(src);
b2nd_free_ctx(ctx);
}
free(image);
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/b2nd/bench_zfp_getitem.c 0000664 0000000 0000000 00000017344 15255167342 0022503 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
/*
* Benchmark to measure the retrieval time of a number of elements in random positions
* in b2nd arrays. To get the necessary arrays (air1.cat, precip1.cat, snow1.cat...)
* you can use the following script, changing the dataset by your preference (see
* https://docs.digitalearthafrica.org/fr/latest/sandbox/notebooks/Datasets/Climate_Data_ERA5_AWS.html):
8<---snip---- "fetch_data.py"
#!/usr/bin/env python
import numpy as np
import s3fs
import xarray as xr
import blosc2
def open_zarr(year, month, datestart, dateend):
fs = s3fs.S3FileSystem(anon=True)
datestring = "era5-pds/zarr/{year}/{month:02d}/data/".format(year=year, month=month)
s3map = s3fs.S3Map(datestring + "precipitation_amount_1hour_Accumulation.zarr/", s3=fs)
precip_zarr = xr.open_dataset(s3map, engine="zarr")
precip_zarr = precip_zarr.sel(time1=slice(np.datetime64(datestart), np.datetime64(dateend)))
return precip_zarr.precipitation_amount_1hour_Accumulation
print("Fetching data from S3 (era5-pds)...")
precip_m0 = open_zarr(1987, 10, "1987-10-01", "1987-10-30 23:59")
precip0 = blosc2.empty(shape=precip_m0.shape, dtype=precip_m0.dtype, urlpath="precip1.b2nd")
print("Fetching and storing 1st month...")
values = precip_m0.values
precip0[:] = values
8<---snip----
* To call this script, you can run the following commands:
* $ pip install blosc2
* $ python fetch_data.py
*
*/
#include "../plugins/codecs/zfp/zfp-private.h"
#include "../../plugins/codecs/zfp/blosc2-zfp.h"
#include "context.h"
#include "blosc2/codecs-registry.h"
#include "b2nd.h"
#include "blosc2.h"
int comp(const char *urlpath) {
blosc2_init();
blosc2_schunk *schunk = blosc2_schunk_open(urlpath);
if (schunk->typesize != 4) {
printf("Error: This test is only for floats.\n");
return -1;
}
blosc2_remove_urlpath("schunk_rate.cat");
blosc2_remove_urlpath("schunk.cat");
// Get multidimensional parameters and configure Blosc2 NDim array
int8_t ndim;
int64_t shape[4];
int64_t shape_aux[4];
int32_t chunkshape[4];
int32_t blockshape[4];
uint8_t *smeta;
int32_t smeta_len;
if (blosc2_meta_get(schunk, "b2nd", &smeta, &smeta_len) < 0) {
printf("This benchmark only supports b2nd arrays");
return -1;
}
char *dtype;
int8_t dtype_format;
b2nd_deserialize_meta(smeta, smeta_len, &ndim, shape_aux, chunkshape, blockshape, &dtype, &dtype_format);
free(smeta);
free(dtype);
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.nthreads = 6;
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
blosc2_storage b2_storage = {.cparams=&cparams, .dparams=&dparams};
b2_storage.urlpath = "schunk_rate.cat";
b2nd_context_t *ctx = b2nd_create_ctx(&b2_storage, ndim, shape_aux, chunkshape, blockshape, NULL, 0,
NULL, 0);
b2nd_array_t *arr;
b2nd_from_schunk(schunk, &arr);
int copied;
printf("LZ4 comp ratio: %f \n", (float) arr->sc->nbytes / (float) arr->sc->cbytes);
/* Use BLOSC_CODEC_ZFP_FIXED_RATE */
b2nd_array_t *arr_rate;
blosc2_context *ctx_zfp = blosc2_create_cctx(cparams);
ctx_zfp->compcode = BLOSC_CODEC_ZFP_FIXED_RATE;
ctx_zfp->splitmode = BLOSC_NEVER_SPLIT;
ctx_zfp->compcode_meta = (uint8_t) (100.0 * (float) arr->sc->cbytes / (float) arr->sc->nbytes);
ctx_zfp->filters[BLOSC2_MAX_FILTERS - 1] = 0;
ctx_zfp->filters_meta[BLOSC2_MAX_FILTERS - 1] = 0;
copied = b2nd_copy(ctx, arr, &arr_rate);
if (copied != 0) {
printf("Error BLOSC_CODEC_ZFP_FIXED_RATE \n");
b2nd_free(arr);
return -1;
}
printf("ZFP_FIXED_RATE comp ratio: %f \n", (float) arr_rate->sc->nbytes / (float) arr_rate->sc->cbytes);
int64_t nelems = arr_rate->nitems;
int dsize_zfp, dsize_blosc;
int64_t index;
float item_zfp, item_blosc;
blosc_timestamp_t t0, t1;
double zfp_time, blosc_time;
zfp_time = blosc_time = 0;
int64_t index_ndim[ZFP_MAX_DIM];
int64_t index_chunk_ndim[ZFP_MAX_DIM];
int64_t ind_ndim[ZFP_MAX_DIM];
int32_t stride_chunk, ind_chunk;
int64_t nchunk;
bool needs_free_blosc, needs_free_zfp;
uint8_t *chunk_blosc, *chunk_zfp;
int32_t chunk_nbytes_zfp, chunk_cbytes_zfp, chunk_nbytes_lossy, chunk_cbytes_lossy;
double ntests = 500.0;
for (int i = 0; i < ntests; ++i) {
srand(i);
index = rand() % nelems;
blosc2_unidim_to_multidim(ndim, shape, index, index_ndim);
for (int j = 0; j < ndim; ++j) {
index_chunk_ndim[j] = index_ndim[j] / chunkshape[j];
ind_ndim[j] = index_ndim[j] % chunkshape[j];
}
stride_chunk = (int32_t)(shape[1] - 1) / chunkshape[1] + 1;
nchunk = index_chunk_ndim[0] * stride_chunk + index_chunk_ndim[1];
ind_chunk = (int32_t) (ind_ndim[0] * chunkshape[1] + ind_ndim[1]);
blosc2_schunk_get_lazychunk(arr->sc, nchunk, &chunk_blosc, &needs_free_blosc);
blosc2_cbuffer_sizes(chunk_blosc, &chunk_nbytes_lossy, &chunk_cbytes_lossy, NULL);
blosc_set_timestamp(&t0);
dsize_blosc = blosc2_getitem_ctx(arr->sc->dctx, chunk_blosc, chunk_cbytes_lossy,
ind_chunk, 1, &item_blosc, sizeof(item_blosc));
blosc_set_timestamp(&t1);
blosc_time += blosc_elapsed_secs(t0, t1);
blosc2_schunk_get_lazychunk(arr_rate->sc, nchunk, &chunk_zfp, &needs_free_zfp);
blosc2_cbuffer_sizes(chunk_zfp, &chunk_nbytes_zfp, &chunk_cbytes_zfp, NULL);
blosc_set_timestamp(&t0);
dsize_zfp = blosc2_getitem_ctx(arr_rate->sc->dctx, chunk_zfp, chunk_cbytes_zfp,
ind_chunk, 1, &item_zfp, sizeof(item_zfp));
blosc_set_timestamp(&t1);
zfp_time += blosc_elapsed_secs(t0, t1);
if (dsize_blosc != dsize_zfp) {
printf("Different amount of items gotten");
return -1;
}
}
printf("ZFP_FIXED_RATE time: %.5f microseconds\n", (zfp_time * 1000000.0 / ntests));
printf("Blosc2 time: %.5f microseconds\n", (blosc_time * 1000000.0 / ntests));
b2nd_free(arr);
b2nd_free(arr_rate);
BLOSC_ERROR(b2nd_free_ctx(ctx));
blosc2_free_ctx(ctx_zfp);
if (needs_free_blosc) {
free(chunk_blosc);
}
if (needs_free_zfp) {
free(chunk_zfp);
}
blosc2_destroy();
return BLOSC2_ERROR_SUCCESS;
}
int solar1(void) {
const char *urlpath = "../../bench/solar1.cat";
int result = comp(urlpath);
return result;
}
int air1(void) {
const char *urlpath = "../../bench/air1.cat";
int result = comp(urlpath);
return result;
}
int snow1(void) {
const char *urlpath = "../../bench/snow1.cat";
int result = comp(urlpath);
return result;
}
int wind1(void) {
const char *urlpath = "../../bench/wind1.cat";
int result = comp(urlpath);
return result;
}
int precip1(void) {
const char *urlpath = "../../bench/precip1.cat";
int result = comp(urlpath);
return result;
}
int precip2(void) {
const char *urlpath = "../../bench/precip2.cat";
int result = comp(urlpath);
return result;
}
int precip3(void) {
const char *urlpath = "../../bench/precip3.cat";
int result = comp(urlpath);
return result;
}
int precip3m(void) {
const char *urlpath = "../../bench/precip-3m.cat";
int result = comp(urlpath);
return result;
}
int main() {
printf("wind1 \n");
BLOSC_ERROR(wind1());
printf("air1 \n");
BLOSC_ERROR(air1());
printf("solar1 \n");
BLOSC_ERROR(solar1());
printf("snow1 \n");
BLOSC_ERROR(snow1());
printf("precip1 \n");
BLOSC_ERROR(precip1());
printf("precip2 \n");
BLOSC_ERROR(precip2());
printf("precip3 \n");
BLOSC_ERROR(precip3());
// printf("precip3m \n");
// BLOSC_ERROR(precip3m());
return BLOSC2_ERROR_SUCCESS;
}
Blosc-c-blosc2-d3bee7c/bench/bench_compact_tail.c 0000664 0000000 0000000 00000011237 15255167342 0021773 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark for bounded-buffer contiguous-frame tail updates.
*/
#include
#include
#include
#include
#include "blosc2.h"
#include "frame.h"
#define BENCH_FRAME "bench_compact_tail.b2frame"
#define CHUNK_ITEMS (250 * 1000) /* 1 MB of int32 per chunk */
#define NUM_CHUNKS (20) /* 20 MB total uncompressed */
static void bench_move_range_caps(void) {
printf("=== Microbenchmark: frame_move_range throughput across buffer caps ===\n");
const char *tmpfile = "bench_move_raw.bin";
FILE *f = fopen(tmpfile, "w+b");
if (!f) return;
uint8_t *dummy = malloc(1024 * 1024);
memset(dummy, 0x5A, 1024 * 1024);
for (int i = 0; i < 50; ++i) {
fwrite(dummy, 1, 1024 * 1024, f);
}
free(dummy);
fflush(f);
blosc2_io_cb *io_cb = blosc2_get_io_cb(BLOSC2_IO_FILESYSTEM);
void *fp = io_cb->open(tmpfile, "rb+", NULL);
int64_t move_len = 30 * 1024 * 1024; /* 30 MB tail */
int64_t caps[] = {64 * 1024, 256 * 1024, 1024 * 1024, 4 * 1024 * 1024};
int ncaps = (int)(sizeof(caps) / sizeof(caps[0]));
for (int i = 0; i < ncaps; ++i) {
int64_t cap = caps[i];
/* Test shrink (forward) */
blosc_timestamp_t t0, t1;
blosc_set_timestamp(&t0);
int rc = frame_move_range(io_cb, fp, 15 * 1024 * 1024, 10 * 1024 * 1024, move_len, cap);
blosc_set_timestamp(&t1);
double dt_shrink = blosc_elapsed_secs(t0, t1);
double speed_shrink = ((double)move_len / (1024.0 * 1024.0)) / dt_shrink;
/* Test growth (backward) */
blosc_set_timestamp(&t0);
rc = frame_move_range(io_cb, fp, 10 * 1024 * 1024, 15 * 1024 * 1024, move_len, cap);
blosc_set_timestamp(&t1);
double dt_growth = blosc_elapsed_secs(t0, t1);
double speed_growth = ((double)move_len / (1024.0 * 1024.0)) / dt_growth;
printf(" Cap: %6" PRId64 " KB | Shrink (fwd): %6.2f MB/s (%6.4f s) | Growth (bwd): %6.2f MB/s (%6.4f s) [rc=%d]\n",
cap / 1024, speed_shrink, dt_shrink, speed_growth, dt_growth, rc);
}
io_cb->close(fp);
remove(tmpfile);
}
static void bench_frame_update_positions(void) {
printf("\n=== Macrobenchmark: contiguous frame_update_chunk early/mid/late ===\n");
blosc2_remove_urlpath(BENCH_FRAME);
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
cparams.typesize = sizeof(int32_t);
cparams.compcode = BLOSC_BLOSCLZ;
cparams.clevel = 1; /* Fast compression so tail I/O is dominant */
blosc2_storage storage = {
.cparams = &cparams,
.dparams = &dparams,
.urlpath = BENCH_FRAME,
.contiguous = true,
};
blosc2_schunk *schunk = blosc2_schunk_new(&storage);
int32_t *data = malloc(CHUNK_ITEMS * sizeof(int32_t));
int32_t buf_bytes = CHUNK_ITEMS * sizeof(int32_t);
for (int i = 0; i < NUM_CHUNKS; ++i) {
for (int j = 0; j < CHUNK_ITEMS; ++j) {
data[j] = i * 10000 + j;
}
blosc2_schunk_append_buffer(schunk, data, buf_bytes);
}
int64_t positions[] = {0, NUM_CHUNKS / 2, NUM_CHUNKS - 1};
const char *pos_names[] = {"Early (chunk 0)", "Middle (chunk 10)", "Late (chunk 19)"};
for (int p = 0; p < 3; ++p) {
int64_t pos = positions[p];
/* Growth update: less compressible data */
for (int j = 0; j < CHUNK_ITEMS; ++j) {
data[j] = rand();
}
uint8_t *chunk_grow = malloc(buf_bytes + BLOSC2_MAX_OVERHEAD);
int csize_grow = blosc2_compress_ctx(schunk->cctx, data, buf_bytes, chunk_grow, buf_bytes + BLOSC2_MAX_OVERHEAD);
blosc_timestamp_t t0, t1;
blosc_set_timestamp(&t0);
blosc2_schunk_update_chunk(schunk, pos, chunk_grow, true);
blosc_set_timestamp(&t1);
double dt_grow = blosc_elapsed_secs(t0, t1);
free(chunk_grow);
/* Shrink update: highly compressible data */
for (int j = 0; j < CHUNK_ITEMS; ++j) {
data[j] = 42;
}
uint8_t *chunk_shrink = malloc(buf_bytes + BLOSC2_MAX_OVERHEAD);
int csize_shrink = blosc2_compress_ctx(schunk->cctx, data, buf_bytes, chunk_shrink, buf_bytes + BLOSC2_MAX_OVERHEAD);
blosc_set_timestamp(&t0);
blosc2_schunk_update_chunk(schunk, pos, chunk_shrink, true);
blosc_set_timestamp(&t1);
double dt_shrink = blosc_elapsed_secs(t0, t1);
free(chunk_shrink);
printf(" Position: %-20s | Growth delta (%d -> %d): %6.4f s | Shrink delta (%d -> %d): %6.4f s\n",
pos_names[p], csize_shrink, csize_grow, dt_grow, csize_grow, csize_shrink, dt_shrink);
}
free(data);
blosc2_schunk_free(schunk);
blosc2_remove_urlpath(BENCH_FRAME);
}
int main(void) {
blosc2_init();
bench_move_range_caps();
bench_frame_update_positions();
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/create_frame.c 0000664 0000000 0000000 00000014143 15255167342 0020611 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Simple benchmark for frame creation.
To run:
$ ./create_frame
*** Creating uninitialized ***
*** Using fill method! ***
*** Creating *contiguous* super-chunk for *blosclz*
Compression ratio: 186.26 GB -> 0.17 KB (1162790697.7x)
Compression time: 8.5e-06 s, 21399.9 TB/s
Decompression time: 0.0628 s, 2965.1 GB/s
*** Creating *sparse* super-chunk for *blosclz*
Compression ratio: 186.26 GB -> 3906.29 KB (49999.5x)
Compression time: 0.00294 s, 61.8 TB/s
Decompression time: 0.00424 s, 43959.2 GB/s
*** Creating *contiguous* super-chunk for *lz4*
Compression ratio: 186.26 GB -> 0.17 KB (1162790697.7x)
Compression time: 4.71e-06 s, 38636.1 TB/s
Decompression time: 0.0701 s, 2656.7 GB/s
*** Creating *sparse* super-chunk for *lz4*
Compression ratio: 186.26 GB -> 3906.29 KB (49999.5x)
Compression time: 0.00311 s, 58.5 TB/s
Decompression time: 0.0101 s, 18516.4 GB/s
Process finished with exit code 0
*/
#include
#include
#include
#define KB (1024.)
#define MB (1024 * KB)
#define GB (1024 * MB)
#define TB (1024 * GB)
#define CHUNKSHAPE (500 * 1000)
#define NCHUNKS 100000
#define NTHREADS 1 // curiously, using 1 single thread is better for the uninitialized values
// For exercising the optimized chunk creators (un)comment the lines below as you please
//#define CREATE_ZEROS
#define CREATE_FILL
//#define CREATE_LOOP
int create_cframe(const char* compname, bool contiguous) {
int32_t isize = CHUNKSHAPE * sizeof(int32_t);
int32_t* data = malloc(isize);
int32_t* data_dest = malloc(isize);
int32_t* data_dest2 = malloc(isize);
int64_t nbytes, cbytes;
int nchunk;
blosc_timestamp_t last, current;
double ttotal;
int compcode = blosc2_compname_to_compcode(compname);
printf("\n*** Creating *%s* super-chunk for *%s*\n",
contiguous ? "contiguous" : "sparse", compname);
/* Create a super-chunk container */
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.typesize = sizeof(int32_t);
cparams.compcode = compcode;
cparams.clevel = 9;
cparams.nthreads = NTHREADS;
//cparams.blocksize = 1024;
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
dparams.nthreads = NTHREADS;
char filename[64];
sprintf(filename, "frame_simple-%s.b2frame", compname);
blosc2_storage storage = {.cparams=&cparams, .dparams=&dparams,
.urlpath=NULL, .contiguous=contiguous};
blosc2_schunk* schunk = blosc2_schunk_new(&storage);
#ifdef CREATE_ZEROS
// Precompute chunk of zeros
int ret = blosc2_chunk_zeros(cparams, isize, data_dest, isize);
#else
int ret = blosc2_chunk_uninit(cparams, isize, data_dest, isize);
#endif
if (ret < 0) {
printf("Creation error in special chunk. Error code: %d\n", ret);
return ret;
}
// Add some data
blosc_set_timestamp(&last);
int64_t nitems;
#ifdef CREATE_FILL
// Make nitems a non-divisible number of CHUNKSHAPE
nitems = (int64_t)NCHUNKS * CHUNKSHAPE + 1;
#ifdef CREATE_ZEROS
// Precompute chunk of zeros
int special_value = BLOSC2_SPECIAL_ZERO;
#else
int special_value = BLOSC2_SPECIAL_UNINIT;
#endif
int64_t rc = blosc2_schunk_fill_special(schunk, nitems, special_value, isize);
if (rc < 0) {
printf("Error in fill special. Error code: %" PRId64 "\n", rc);
return (int)rc;
}
#else
// In these methods, nitems can only be an actual multiple of CHUNKSHAPE
nitems = (int64_t)NCHUNKS * CHUNKSHAPE;
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
#ifdef CREATE_LOOP
int nchunks = blosc2_schunk_append_chunk(schunk, (uint8_t *) data_dest, true);
if (nchunks != nchunk + 1) {
printf("Compression error in append chunk. Error code: %d\n", nchunks);
return nchunk;
}
#else
for (int i = 0; i < CHUNKSHAPE; i++) {
// Different data patterns
// data[i] = i * nchunk;
// data[i] = nchunk;
data[i] = 0;
}
int nchunks = blosc2_schunk_append_buffer(schunk, data, isize);
if (nchunks != nchunk + 1) {
printf("Compression error appending in schunk. Error code: %d\n", nchunks);
return nchunk;
}
#endif
}
#endif
blosc_set_timestamp(¤t);
/* Gather some info */
nbytes = schunk->nbytes;
cbytes = blosc2_schunk_frame_len(schunk);
ttotal = blosc_elapsed_secs(last, current);
printf("Compression ratio: %.2f GB -> %.2f KB (%4.1fx)\n",
(double)nbytes / GB, (double)cbytes / KB, (double)nbytes / (double)cbytes);
printf("Compression time: %.3g s, %.1f TB/s\n",
ttotal, (double)nbytes / (ttotal * TB));
/* Retrieve and decompress the chunks from the super-chunks and compare values */
blosc_set_timestamp(&last);
int32_t leftover_bytes = (int32_t)((nitems % CHUNKSHAPE) * sizeof(int32_t));
int32_t nchunks = leftover_bytes ? NCHUNKS + 1 : NCHUNKS;
for (nchunk = 0; nchunk < nchunks; nchunk++) {
int32_t dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, data_dest, isize);
if (dsize < 0) {
printf("Decompression error in schunk. Error code: %d\n", dsize);
return dsize;
}
if ((nchunk == nchunks - 1) && (leftover_bytes > 0)) {
if (dsize != leftover_bytes) {
printf("Wrong size for last chunk. It is %d and should be: %d\n", dsize, leftover_bytes);
return dsize;
}
}
}
blosc_set_timestamp(¤t);
ttotal = blosc_elapsed_secs(last, current);
printf("Decompression time: %.3g s, %.1f GB/s\n",
ttotal, (double)nbytes / (ttotal * GB));
/* Free resources */
blosc2_schunk_free(schunk);
free(data);
free(data_dest);
free(data_dest2);
return 0;
}
int main(void) {
blosc2_init();
#ifdef CREATE_ZEROS
printf("\n *** Creating zeros ***\n");
#else
printf("\n *** Creating uninitialized ***\n");
#endif
#ifdef CREATE_FILL
printf(" *** Using fill method! ***\n");
#elif defined(CREATE_LOOP)
printf(" *** Using loop method! ***\n");
#else
printf(" *** Using not optimized method! ***\n");
#endif
create_cframe("blosclz", true);
create_cframe("blosclz", false);
create_cframe("lz4", true);
create_cframe("lz4", false);
blosc2_destroy();
}
Blosc-c-blosc2-d3bee7c/bench/delta_schunk.c 0000664 0000000 0000000 00000006511 15255167342 0020640 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark showing Blosc filter from C code.
To compile this program:
$ gcc -O3 delta_schunk.c -o delta_schunk -lblosc2
*/
#include
#include
#include
#include
#define KB 1024
#define MB (1024*KB)
#define GB (1024*MB)
#define CHUNKSIZE (50 * 1000)
#define NCHUNKS 100
// Setting NTHREADS > 1 increases the likelihood of a crash. See #112.
#define NTHREADS 1
int main(void) {
int32_t *data, *data_dest;
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
blosc2_schunk *schunk;
int32_t isize = CHUNKSIZE * sizeof(int32_t);
int dsize;
int64_t nbytes, cbytes;
int nchunk;
int64_t nchunks = 0;
blosc_timestamp_t last, current;
double totaltime;
float totalsize = (float)(isize * NCHUNKS);
data = malloc(CHUNKSIZE * sizeof(int32_t));
data_dest = malloc(CHUNKSIZE * sizeof(int32_t));
for (int i = 0; i < CHUNKSIZE; i++) {
data[i] = i;
}
printf("Blosc version info: %s (%s)\n", BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc2_init();
/* Create a super-chunk container */
cparams.filters[0] = BLOSC_DELTA;
//cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_BITSHUFFLE;
cparams.typesize = sizeof(int32_t);
cparams.compcode = BLOSC_BLOSCLZ;
cparams.clevel = 1;
cparams.nthreads = NTHREADS;
blosc2_storage storage = {.cparams=&cparams, .dparams=&dparams};
schunk = blosc2_schunk_new(&storage);
/* Append chunks (the first will be taken as reference for delta) */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
nchunks = blosc2_schunk_append_buffer(schunk, data, isize);
}
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
printf("[Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
totaltime, totalsize / GB, totalsize / (GB * totaltime));
/* Gather some info */
nbytes = schunk->nbytes;
cbytes = schunk->cbytes;
printf("Compression super-chunk: %ld -> %ld (%.1fx)\n",
(long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes);
/* Retrieve and decompress the chunks */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, data_dest, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
assert (dsize == (int)isize);
}
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
totalsize = (float)(isize * nchunks);
printf("[Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
totaltime, totalsize / GB, totalsize / (GB * totaltime));
printf("Decompression successful!\n");
for (int i = 0; i < CHUNKSIZE; i++) {
if (data[i] != data_dest[i]) {
printf("Decompressed data differs from original %d, %d, %d!\n",
i, data[i], data_dest[i]);
return -1;
}
}
printf("Successful roundtrip!\n");
/* Free resources */
free(data);
free(data_dest);
/* Destroy the super-chunk */
blosc2_schunk_free(schunk);
/* Destroy the Blosc environment */
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/frame_lock_bench.c 0000664 0000000 0000000 00000010516 15255167342 0021435 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark for the opt-in cross-process frame locking
(blosc2_stdio_params.locking): measures the per-operation overhead of the
sidecar lock by running the same workload on a disk-based sparse frame with
locking disabled and enabled.
To run:
$ ./frame_lock_bench
locking=off get_lazychunk: 23731 ns decompress: 64382 ns update: 264792 ns
locking=on get_lazychunk: 24303 ns decompress: 65273 ns update: 265234 ns
overhead: get_lazychunk: +2.4 % decompress: +1.4 % update: +0.2 %
*/
#include
#include
#include
#define CHUNKSIZE (100 * 1000) /* items per chunk (int64_t) */
#define NCHUNKS (50)
#define NREADS (20 * 1000)
#define NUPDATES (500)
#define URLPATH "frame_lock_bench.b2frame"
static double elapsed_ns_per_op(blosc_timestamp_t t0, int nops) {
blosc_timestamp_t t1;
blosc_set_timestamp(&t1);
return blosc_elapsed_nsecs(t0, t1) / nops;
}
/* Run the workload with or without locking; fills results[3] with ns/op for
get_lazychunk, decompress_chunk and update_chunk (with a special chunk). */
static int run(bool locking, double results[3]) {
blosc2_stdio_params ioparams = {.locking = true};
blosc2_io io = {.id = BLOSC2_IO_FILESYSTEM, .name = "filesystem", .params = &ioparams};
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.typesize = sizeof(int64_t);
blosc2_storage storage = {.contiguous = false, .urlpath = URLPATH, .cparams = &cparams};
if (locking) {
storage.io = &io;
}
blosc2_remove_urlpath(URLPATH);
blosc2_schunk *schunk = blosc2_schunk_new(&storage);
if (schunk == NULL) {
return -1;
}
int64_t *data = malloc(CHUNKSIZE * sizeof(int64_t));
for (int nchunk = 0; nchunk < NCHUNKS; nchunk++) {
for (int i = 0; i < CHUNKSIZE; i++) {
data[i] = nchunk * CHUNKSIZE + i;
}
if (blosc2_schunk_append_buffer(schunk, data, CHUNKSIZE * sizeof(int64_t)) < 0) {
return -1;
}
}
blosc2_schunk_free(schunk);
schunk = locking ? blosc2_schunk_open_udio(URLPATH, &io) : blosc2_schunk_open(URLPATH);
if (schunk == NULL) {
return -1;
}
uint8_t *chunk;
bool needs_free;
blosc_timestamp_t t0;
// get_lazychunk (one untimed warm-up sweep first)
for (int pass = 0; pass < 2; pass++) {
blosc_set_timestamp(&t0);
for (int i = 0; i < NREADS; i++) {
int csize = blosc2_schunk_get_lazychunk(schunk, i % NCHUNKS, &chunk, &needs_free);
if (csize < 0) {
return -1;
}
if (needs_free) {
free(chunk);
}
}
results[0] = elapsed_ns_per_op(t0, NREADS);
}
// decompress_chunk
int64_t *dest = malloc(CHUNKSIZE * sizeof(int64_t));
const int ndecs = NREADS / 10;
blosc_set_timestamp(&t0);
for (int i = 0; i < ndecs; i++) {
if (blosc2_schunk_decompress_chunk(schunk, i % NCHUNKS, dest,
CHUNKSIZE * sizeof(int64_t)) < 0) {
return -1;
}
}
results[1] = elapsed_ns_per_op(t0, ndecs);
// update_chunk with an UNINIT special chunk (the cache-eviction primitive)
uint8_t uninit[BLOSC_EXTENDED_HEADER_LENGTH];
if (blosc2_chunk_uninit(*schunk->storage->cparams, CHUNKSIZE * sizeof(int64_t),
uninit, sizeof(uninit)) < 0) {
return -1;
}
blosc_set_timestamp(&t0);
for (int i = 0; i < NUPDATES; i++) {
if (blosc2_schunk_update_chunk(schunk, i % NCHUNKS, uninit, true) < 0) {
return -1;
}
}
results[2] = elapsed_ns_per_op(t0, NUPDATES);
blosc2_schunk_free(schunk);
blosc2_remove_urlpath(URLPATH);
free(data);
free(dest);
return 0;
}
int main(void) {
blosc2_init();
double off[3], on[3];
if (run(false, off) < 0 || run(true, on) < 0) {
printf("Error running the frame lock benchmark!\n");
return -1;
}
printf("locking=off get_lazychunk: %8.0f ns decompress: %8.0f ns update: %8.0f ns\n",
off[0], off[1], off[2]);
printf("locking=on get_lazychunk: %8.0f ns decompress: %8.0f ns update: %8.0f ns\n",
on[0], on[1], on[2]);
printf("overhead: get_lazychunk: %+7.1f %% decompress: %+7.1f %% update: %+7.1f %%\n",
100.0 * (on[0] - off[0]) / off[0],
100.0 * (on[1] - off[1]) / off[1],
100.0 * (on[2] - off[2]) / off[2]);
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/get_sparse.c 0000664 0000000 0000000 00000015563 15255167342 0020337 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2026 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark for reading individual coordinates from a 1-D b2nd array.
To run from the repository root:
$ ./build/bench/read-coords bench/sin-1d.b2nd [ncoords]
*/
/*
Use:
import numpy as np
import blosc2
N = 24_000_000
a = blosc2.linspace(0., 1., N, dtype=np.float32)
b = blosc2.sin(a).compute(urlpath="sin-1d.b2nd", mode="w")
print(b[:])
for generating the data file.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#define DEFAULT_NCOORDS 100
#define NPRINT 10
#define NTHREADS 4
static void fill_random_coords(int64_t *coords, int64_t ncoords, int64_t nitems) {
uint64_t state = UINT64_C(0x20260522);
for (int64_t i = 0; i < ncoords; ++i) {
/* xorshift64*: deterministic and good enough for benchmark coordinates. */
state ^= state >> 12;
state ^= state << 25;
state ^= state >> 27;
uint64_t r = state * UINT64_C(2685821657736338717);
coords[i] = (int64_t)(r % (uint64_t)nitems);
}
}
static int open_array(const char **urlpath, bool use_fallbacks, b2nd_array_t **array) {
int rc = b2nd_open(*urlpath, array);
if (rc >= 0 || *array != NULL || !use_fallbacks) {
if (rc < 0) {
fprintf(stderr, "Cannot open %s (error %d)\n", *urlpath, rc);
}
return rc;
}
const char *fallbacks[] = {"../bench/sin-1d.b2nd", "../../bench/sin-1d.b2nd"};
for (size_t i = 0; i < sizeof(fallbacks) / sizeof(fallbacks[0]); ++i) {
*urlpath = fallbacks[i];
rc = b2nd_open(*urlpath, array);
if (rc >= 0) {
return rc;
}
}
fprintf(stderr, "Cannot open sin-1d.b2nd (last error %d)\n", rc);
return rc;
}
static int getitem_sparse(const b2nd_array_t *array, const int64_t *coords, int64_t ncoords, float *data) {
int64_t chunk_nitems = array->sc->chunksize / array->sc->typesize;
for (int64_t i = 0; i < ncoords; ++i) {
int64_t coord = coords[i];
int64_t nchunk = coord / chunk_nitems;
int start = (int)(coord % chunk_nitems);
uint8_t *chunk = NULL;
bool needs_free = false;
int cbytes = blosc2_schunk_get_lazychunk(array->sc, nchunk, &chunk, &needs_free);
if (cbytes <= 0) {
fprintf(stderr, "Cannot get chunk %" PRId64 " for coordinate %" PRId64 " (error %d)\n",
nchunk, coord, cbytes);
return cbytes < 0 ? cbytes : -1;
}
int nbytes = blosc2_getitem_ctx(array->sc->dctx, chunk, cbytes, start, 1, &data[i], sizeof(float));
if (needs_free) {
free(chunk);
}
if (nbytes != (int)sizeof(float)) {
fprintf(stderr, "Cannot get coordinate %" PRId64 " from chunk %" PRId64 " (error %d)\n",
coord, nchunk, nbytes);
return nbytes < 0 ? nbytes : -1;
}
}
return 0;
}
int main(int argc, char **argv) {
const char *urlpath = (argc > 1) ? argv[1] : "bench/sin-1d.b2nd";
int64_t ncoords = (argc > 2) ? strtoll(argv[2], NULL, 10) : DEFAULT_NCOORDS;
bool new_first = (argc > 3) && (strcmp(argv[3], "new-first") == 0);
bool getitem_only = (argc > 3) && (strcmp(argv[3], "getitem-only") == 0);
bool new_only = (argc > 3) && (strcmp(argv[3], "new-only") == 0);
b2nd_array_t *array = NULL;
int64_t *coords = NULL;
float *data_getitem = NULL;
float *data_new = NULL;
blosc_timestamp_t t0, t1;
double getitem_time;
double new_time;
blosc2_init();
blosc2_set_nthreads(NTHREADS);
if (ncoords <= 0) {
fprintf(stderr, "ncoords must be positive\n");
blosc2_destroy();
return EXIT_FAILURE;
}
if (open_array(&urlpath, argc == 1, &array) < 0) {
blosc2_destroy();
return EXIT_FAILURE;
}
if (array->ndim != 1) {
fprintf(stderr, "Expected a 1-D array, got %d dimensions\n", array->ndim);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
if (array->sc->typesize != (int32_t)sizeof(float)) {
fprintf(stderr, "Expected float elements (typesize %zu), got typesize %d\n",
sizeof(float), array->sc->typesize);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
coords = malloc((size_t)ncoords * sizeof(int64_t));
data_getitem = malloc((size_t)ncoords * sizeof(float));
data_new = malloc((size_t)ncoords * sizeof(float));
if (coords == NULL || data_getitem == NULL || data_new == NULL) {
fprintf(stderr, "Cannot allocate benchmark buffers\n");
free(coords);
free(data_getitem);
free(data_new);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
fill_random_coords(coords, ncoords, array->shape[0]);
if (new_first || new_only) {
blosc_set_timestamp(&t0);
int rc = blosc2_schunk_get_sparse_buffer(array->sc, ncoords, coords, data_new);
blosc_set_timestamp(&t1);
new_time = blosc_elapsed_secs(t0, t1);
if (rc < 0) {
fprintf(stderr, "blosc2_schunk_get_sparse failed (error %d)\n", rc);
free(coords);
free(data_getitem);
free(data_new);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
}
if (!new_only) {
blosc_set_timestamp(&t0);
if (getitem_sparse(array, coords, ncoords, data_getitem) < 0) {
free(coords);
free(data_getitem);
free(data_new);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
blosc_set_timestamp(&t1);
getitem_time = blosc_elapsed_secs(t0, t1);
}
if (!new_first && !getitem_only && !new_only) {
blosc_set_timestamp(&t0);
int rc = blosc2_schunk_get_sparse_buffer(array->sc, ncoords, coords, data_new);
blosc_set_timestamp(&t1);
new_time = blosc_elapsed_secs(t0, t1);
if (rc < 0) {
fprintf(stderr, "blosc2_schunk_get_sparse failed (error %d)\n", rc);
free(coords);
free(data_getitem);
free(data_new);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
}
if (!getitem_only && !new_only) {
for (int64_t i = 0; i < ncoords; ++i) {
if (data_getitem[i] != data_new[i]) {
fprintf(stderr, "Mismatched result at %" PRId64 ": getitem_sparse=%.9g, new_sparse=%.9g\n",
i, data_getitem[i], data_new[i]);
free(coords);
free(data_getitem);
free(data_new);
b2nd_free(array);
blosc2_destroy();
return EXIT_FAILURE;
}
}
}
printf("Read %" PRId64 " random coordinates from %s (%d threads)\n", ncoords, urlpath, NTHREADS);
int64_t nprint = ncoords < NPRINT ? ncoords : NPRINT;
printf("First %" PRId64 " retrieved elements:\n", nprint);
for (int64_t i = 0; i < nprint; ++i) {
printf(" coord[%" PRId64 "] = %" PRId64 ", value = %.9g\n", i, coords[i], data_new[i]);
}
if (!new_only) {
printf("getitem_sparse: %.9f s\n", getitem_time);
}
if (!getitem_only) {
printf("new_sparse: %.9f s\n", new_time);
}
free(coords);
free(data_getitem);
free(data_new);
b2nd_free(array);
blosc2_destroy();
return EXIT_SUCCESS;
}
Blosc-c-blosc2-d3bee7c/bench/plot-speeds.py 0000664 0000000 0000000 00000016421 15255167342 0020642 0 ustar 00root root 0000000 0000000 """
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Script for plotting the results of the 'suite' benchmark.
Invoke without parameters for usage hints.
"""
import matplotlib as mpl
from pylab import *
KB_ = 1024
MB_ = 1024*KB_
GB_ = 1024*MB_
NCHUNKS = 128 # keep in sync with bench.c
linewidth=2
#markers= ['+', ',', 'o', '.', 's', 'v', 'x', '>', '<', '^']
#markers= [ 'x', '+', 'o', 's', 'v', '^', '>', '<', ]
markers= [ 's', 'o', 'v', '^', '+', 'x', '>', '<', '.', ',' ]
markersize = 8
def get_values(filename):
f = open(filename)
values = {"memcpyw": [], "memcpyr": []}
for line in f:
if line.startswith('-->'):
tmp = line.split('-->')[1]
nthreads, size, elsize, sbits, codec, shuffle = [i for i in tmp.split(', ')]
nthreads, size, elsize, sbits = map(int, (nthreads, size, elsize, sbits))
values["size"] = size / MB_
values["elsize"] = elsize
values["sbits"] = sbits
values["codec"] = codec
values["shuffle"] = shuffle
# New run for nthreads
(ratios, speedsw, speedsr) = ([], [], [])
# Add a new entry for (ratios, speedw, speedr)
values[nthreads] = (ratios, speedsw, speedsr)
#print("-->", nthreads, size, elsize, sbits)
elif line.startswith('memcpy(write):'):
tmp = line.split(',')[1]
memcpyw = float(tmp.split(' ')[1]) / 1024.
values["memcpyw"].append(memcpyw)
elif line.startswith('memcpy(read):'):
tmp = line.split(',')[1]
memcpyr = float(tmp.split(' ')[1]) / 1024.
values["memcpyr"].append(memcpyr)
elif line.startswith('comp(write):'):
tmp = line.split(',')[1]
speedw = float(tmp.split(' ')[1]) / 1024.
ratio = float(line.split(':')[-1])
speedsw.append(speedw)
ratios.append(ratio)
elif line.startswith('decomp(read):'):
tmp = line.split(',')[1]
speedr = float(tmp.split(' ')[1]) / 1024.
speedsr.append(speedr)
if "OK" not in line:
print("WARNING! OK not found in decomp line!")
f.close()
return nthreads, values
def show_plot(plots, yaxis, legends, gtitle, xmax=None, ymax=None):
xlabel('Compression ratio')
ylabel('Speed (GB/s)')
title(gtitle)
xlim(0, xmax)
ylim(0, ymax)
grid(True)
# legends = [f[f.find('-'):f.index('.out')] for f in filenames]
# legends = [l.replace('-', ' ') for l in legends]
#legend([p[0] for p in plots], legends, loc = "upper left")
legend([p[0] for p in plots
if not isinstance(p, mpl.lines.Line2D)],
legends, loc = "best")
#subplots_adjust(bottom=0.2, top=None, wspace=0.2, hspace=0.2)
if outfile:
print("Saving plot to:", outfile)
savefig(outfile, dpi=100)
else:
show()
if __name__ == '__main__':
from optparse import OptionParser
usage = "usage: %prog [-r] [-o outfile] [-t title ] [-d|-c] filename"
compress_title = 'Compression speed'
decompress_title = 'Decompression speed'
yaxis = 'No axis name'
parser = OptionParser(usage=usage)
parser.add_option('-o',
'--outfile',
dest='outfile',
help=('filename for output (many extensions '
'supported, e.g. .png, .jpg, .pdf)'))
parser.add_option('-t',
'--title',
dest='title',
help='title of the plot',)
parser.add_option('-l',
'--limit',
dest='limit',
help='expression to limit number of threads shown',)
parser.add_option('-x',
'--xmax',
dest='xmax',
help='limit the x-axis',
default=None)
parser.add_option('-y',
'--ymax',
dest='ymax',
help='limit the y-axis',
default=None)
parser.add_option('-r', '--report', action='store_true',
dest='report',
help='generate file for reporting ',
default=False)
parser.add_option('-d', '--decompress', action='store_true',
dest='dspeed',
help='plot decompression data',
default=False)
parser.add_option('-c', '--compress', action='store_true',
dest='cspeed',
help='plot compression data',
default=False)
(options, args) = parser.parse_args()
if len(args) == 0:
parser.error("No input arguments")
elif len(args) > 1:
parser.error("Too many input arguments")
else:
pass
if options.report and options.outfile:
parser.error("Can only select one of [-r, -o]")
if options.dspeed and options.cspeed:
parser.error("Can only select one of [-d, -c]")
elif options.cspeed:
options.dspeed = False
plot_title = compress_title
else: # either neither or dspeed
options.dspeed = True
plot_title = decompress_title
filename = args[0]
cspeed = options.cspeed
dspeed = options.dspeed
if options.outfile:
outfile = options.outfile
elif options.report:
if cspeed:
outfile = filename[:filename.rindex('.')] + '-compr.png'
else:
outfile = filename[:filename.rindex('.')] + '-decompr.png'
else:
outfile = None
plots = []
legends = []
nthreads, values = get_values(filename)
#print("Values:", values)
if options.limit:
thread_range = eval(options.limit)
else:
thread_range = range(1, nthreads+1)
if options.title:
plot_title = options.title
else:
plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits), %(codec)s, %(shuffle)s" % values
gtitle = plot_title
for nt in thread_range:
#print("Values for %s threads --> %s" % (nt, values[nt]))
(ratios, speedw, speedr) = values[nt]
if cspeed:
speed = speedw
else:
speed = speedr
#plot_ = semilogx(ratios, speed, linewidth=2)
plot_ = plot(ratios, speed, linewidth=2)
plots.append(plot_)
nmarker = nt
if nt >= len(markers):
nmarker = nt%len(markers)
setp(plot_, marker=markers[nmarker], markersize=markersize,
linewidth=linewidth)
legends.append("%d threads" % nt)
# Add memcpy lines
# Take the first value found in order to avoid cache effects
if cspeed:
# memcpy_speed = np.mean(values["memcpyw"])
memcpy_speed = values["memcpyw"][0]
message = "memcpy (write to memory)"
else:
# memcpy_speed = np.mean(values["memcpyr"])
memcpy_speed = values["memcpyr"][0]
message = "memcpy (read from memory)"
plot_ = axhline(memcpy_speed, linewidth=3, linestyle='-.', color='black')
text(4.0, memcpy_speed + .5, message)
plots.append(plot_)
show_plot(plots, yaxis, legends, gtitle,
xmax=int(options.xmax) if options.xmax else None,
ymax=int(options.ymax) if options.ymax else None)
Blosc-c-blosc2-d3bee7c/bench/plot-sum_openmp-results.py 0000664 0000000 0000000 00000003377 15255167342 0023246 0 ustar 00root root 0000000 0000000 import matplotlib.pyplot as plt
import numpy as np
labels = ['1', '2', '4', '8', '12', '14', '16', '20', '24', '28', '32']
uncompressed = [
11909.0, 21645.4, 40004.2, 56435.1, 57749.5, 58024.4, 49663.5, 55813.8, 55357.2, 56296.0, 46849.2]
blosclz_cl1 = [
6016.4, 11062.0, 21284.2, 41452.4, 57542.6, 65535.6, 54084.9, 65367.7, 74655.7, 86628.0, 53853.5]
lz4_cl1 = [
5194.0, 9465.5, 18262.0, 35698.7, 50247.9, 57009.0, 42268.3, 52207.0, 59755.1, 67353.5, 36658.2]
lz4hc_cl1 = [
6109.7, 11114.1, 21368.7, 41932.4, 57268.5, 66780.0, 50380.2, 61220.5, 69262.0, 82380.5, 37305.5]
zstd_cl1 = [
1593.8, 2955.5, 5669.4, 11244.7, 16363.2, 18377.6, 12865.2, 15830.7, 18736.9, 21474.3, 10466.3]
# Use GB/s
uncompressed = np.array(uncompressed) / 1024.
blosclz_cl1 = np.array(blosclz_cl1) / 1024.
lz4_cl1 = np.array(lz4_cl1) / 1024.
lz4hc_cl1 = np.array(lz4hc_cl1) / 1024.
zstd_cl1 = np.array(zstd_cl1) / 1024.
x = np.arange(len(labels)) # the label locations
width = 0.30 / 2 # the width of the bars
fig, ax = plt.subplots()
rects1 = ax.bar(x, uncompressed, width, label='Uncompressed')
rects2 = ax.bar(x + width, blosclz_cl1, width, label='blosclz cl-1 (3.7x cr)')
rects3 = ax.bar(x + 2 * width, lz4_cl1, width, label='lz4 cl-1 (4.5x cr)')
rects4 = ax.bar(x + 3 * width, lz4hc_cl1, width, label='lz4hc cl-1 (5.0x cr)')
rects5 = ax.bar(x + 4 * width, zstd_cl1, width, label='zstd cl-1 (5.9x cr)')
# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylabel('Speed (GB/s)')
ax.set_xlabel('Threads')
ax.set_title('Summing up precipitation data (381.5 MB, float32)')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend()
#ax.bar_label(rects1, padding=3)
#ax.bar_label(rects2, padding=3)
#ax.bar_label(rects2, padding=3)
fig.tight_layout()
plt.show() Blosc-c-blosc2-d3bee7c/bench/rainfall-grid-150x150.bin 0000664 0000000 0000000 00000044317 15255167342 0022164 0 ustar 00root root 0000000 0000000 _ _ ĎH g ! ŕ˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙) 2 ! ŕ^ €ŔŔ `€ŕ Ŕ @ @€ @Ŕ@€@Ŕ €€ € ŕp @Ŕ Ť @ Ŕ ‡ € ŔŔ€€€Ŕ ŕs @…@€@ŕ †@ €€ €@@€ ŕr
€@@@Ŕ`@@đ ŕŠŕu•
€@ Ŕ€Ŕ€ ŔŔ €Ŕ Á Ŕŕp – @"î ` Ŕ€ € ŕ ŕ
ŕk €@Ŕ ŕ ŕv†ŕ "@@€ €` Ŕ ŕ| € — @ŕ`#‹€€ ŕz „€Ŕ€@`Ŕ ŔbW ŕ}— • @ @@
€€ ŕd ŕ n „@€#|Ŕ ŕ €Ŕ ŕy `€ đ`Ŕŕ€@ @ŕw
€ €@@@ŕŔ@ @@#… € ŕz € H/€P°@€ŕ@€! ŕz „@ Ŕ"RPŕ@Ŕ@Ŕ Ŕ€ š ŕx Ŕ€&n ` Đ ś ŔŔ @ŕ{– €@ @đ
€ —Ŕ@ € ŕu €@ $Ŕ0°@
Ž@ Ŕ@ŕt € €@Ŕ`ŔĐ€`@ŕ€€ @@ŕy– Ŕ ` đĐ ŔbR Ŕ @ťŕv Ŕ@ ` €ŕ`ŕ
@€ @€ŔŔŔ€ ŕt €@ @@Đ`@ D@@ Ŕ@ € ŕr €€Ŕ@€ŕ Ŕ €Ŕ Ŕ€€€ Ŕ @ € ŕg ŕq @(-Ŕ°`` !(
Ŕ€€€ €€Ŕ@@€@ ŕd € ŕ `@€0`ŔB\@ @ @ € • ŕe ŕ o€ @@ŕ`ĐĐŕŔ@ ŔŔŔ ™ € ™€€Ŕ €`ŕl € @€@€ Ŕ @ Ŕ@@ €€ ŕm €€Ŕ €€€p` @€€@› @ @@ŕo’ € ŕ`$ °"^Ŕ ŚŔŕŔ€Ŕ@ ť€ €`ŕi s@@€%Ú@`€@Ŕ.ŕ@ @ €€€ ŕi € €@ Ŕ€°Đ¨°€Ŕŕ Ŕ ŕ$!
€ @€€ŔŔ Ŕ€ ŕn Ŕ€ `0` ! Ŕ`€€ $& Ŕ!D @ ŕj
€ @€ Đđđp € !Ă@@ š Ŕ Ŕ@€€€ ŕi €€@ @ŔđđHŕ@đ``!ż @@` Ŕ€Ŕ Ŕ ŕk• t @ đP``€`€°p p € @ŕ €Ŕ€`° @ ŕl ŕ@` 8°đp @€@` — Ŕ€@@€Ŕ@Ŕ@€€ ŕh @€`@đ0ŔM| °Ŕ@@`€@ŕ@€!2 C…@Ŕ€ € ŕg €@ @ Đ P @P Đ€ŕ Ŕ€+,€`€ ŔŔŔ€@ ’ ŕe €
@0Pp`P00 Ź Ŕ ``ŕ "ń!)@ €€ ŕg @"Ď@@ČŘ$đŕ €@ŕ ‹ ŕ€ €`ŕ @ A/ŕe €€ `€PŔp€đ°đ ˘ !)€@ ‹
`` @ŔŔ ŕf @` pđ@`PĐ €@€ŕ@ @ `@@@€€€` @ € ŕd €Ŕ@€Đ P¸ ŕp`Đŕ `"â@ŕ€ Ŕ@€ §Ŕ !Ň ŕg Ŕ pPđp@ { ŕŔ € Ŕ@@@ŕ ' ` "Z ŕ"ý © ŕc € ŕ@ĐPđA) € @Ŕ ŔŔ€€€ €`Ŕ`ŕ@ € @ @ € Ŕ ŕb @ Ŕpp€ 0P ŕŕ@`€€ @€ ` ž ’ŕ ŕŕŔ@ ŕc `@ŕŕ€@)] @ € €€@ @@‡ €€ Ŕ@ ŕ!(Ŕ € ŕa €ŕ °ŕĐ @‘ @ € € @Ŕŕ@!É ŕŔ`@ € ŕ_
€€` 0Ŕ ŕ`€ ` € Ŕ@!ľ @€ŕ€ @Ŕ ŕb @ đ 00 ` € Ŕ € €Ŕ@€ ` `ŕ •€ ŕ` @ŕPŔŕ°0ŕs €@ €ŔŔŔ` @ !/€@€€€ Ŕ ŕ_ €#f `°!,ŕ @ „@ "T Ŕ ŕŕŔ€ ŕ`
€ €ŕ ŕ đŔ€@Ŕ€ ŕ
@€Ŕ€@ Ŕŕ ŕŔBs"íaÓŕ^ €`€@p&o @ `– € @@ŔŔ@`@Ŕŕ"é ` €Ŕ ŕ_ € `@ p • €€ € €€€@@ `ŕ€ `@` ŕ``Ŕ € ŕ_ € €€Đ°°ĐŔ@€Ŕ€€ @ € Ŕ @ €ŕ` @ Ŕ@Ŕ ŕ_ €!ť€ ``@@ € €€Ŕ
``€ ŔŔ€ @ ťŕ` €aŕŕ € Ŕ €€€Ŕ@ ' ŕ@ `$" ` ĽŔ ŕa ŔŔ Ŕ ``€!-ŕ• €€`@ !#
€ € ŕ`€ ŕa €@`!
!ž€€€ ŕ
© $Ň"ô Ŕŕ ŕd @Ŕ `@@€ ŕ € ‹€ŕ ` @ € !`@ ŕc "Î €@ á* €€@ š Ŕ ™ ł @@ ŕe €@ ™ŕ €"U`ŔŕŕŔ ŕ źŔ ŕd €€ŕ “ € @ ` @ @`@`BaŔ ŕh € ŕ € @€€@` d@€ áf+ŕ
€ €€ŕŕ` ``` ŔŔ@”@ ŕz €€Ŕŕ`$Ł€ ŕ!»@@áw*€ ` Ŕ ŔŕŔ € ŕ `¤Ľŕv @™%Ň Ŕ€@€Bî€ €ázÁ€@€Ŕ` ŕ€ `@€ €ŕ ŕw €!$€€ Ŕŕ Ŕ@` @@ ŕv €@@ @ Ŕ @ @€ € ˘ Ŕâxî @€€ ŔŔ @ ŕŔ !/ ŕ@Ŕăw„ €@‚€€€ŕ Ŕ &y Ŕ Bđŕx €Ŕ € ‰ ` € Ŕ€$ş€ ŕy ‰ Ŕ"č%× €ŕ €AË ŕy @€@"ć ŔŔ€€` ŕx “ Ŕ! `Ŕ@ŕŔ€"ó @ ŕz €@@ŕ@€ `ŕ`Ŕ`(8€ ŕ{•€ !ÄŔ@Đ $ŕ@@"úŕz !ą€P@ ŕ€`"ňŕ{ €€€@` Đ`@%ă € ŕz € – P° ŕŔ@Ŕ@ ŕ€á|Á@•Ŕ:r ŕ ŕ|• €Bă °ŕ@ Ŕ #‰ ŕz €€€@Ŕpŕ@@@`ŔŔŔ` ` ŕ{ AÁ @pĐ `ŕ€@@ "[ŕz €€€3pP0`€@ŕ@&|á|,€ @@đ @@ŕŕŔ @` @Ŕ ŕ{ €=` `@— €p° ŕ} Ŕ `0ŕ@ €€` ŔP ŕ}
Ŕ€ Đ `€ €` P á+ ŕp@` @€Đ€Đ ŕ| Ŕđ@Aľ(: €đ€` ŕ} `Đ $ € ŕ PĐ`đ ŕ} Đ Ŕ ŕ €P ŕ} ŕP €` € @ ° â}íŔŕ `Ŕ€@ŕ'@ ›ŕ{ ŔP ”'—€0Ŕ@€` ŕ| ŕ@ € Ŕŕ€`0@ŕ€€ŕ|•€@`€€ ŕ€ŕŔĐ`p `â}îĐ@€ €``€€ŕ`PŔ```Ŕ ŕy €€ 0@€€€ ° ŕ` °PŔ€!7ŕx Ŕ€Đ€ !ĹŔ ŕ$ @$¸ŕx @"GŔ €đP+ ŕŕ`F{ ŕx €€`!ł€Ŕ@°` €#‹€ŕŔŔ ŕx Ŕ€@ Ŕ@ŕđ&ü@@@`Ŕĺ~C Ŕ€€%Ôp€€!5 `#Ť @á Áŕ ŕR n €Ŕ `đ@€Ŕ @`ŕ%LŔŔ ŕw € Ŕ@ ŕ Pŕ@€ "ř @ŕz• Ŕ@ ŹŔŔ `€`aľ€ ŕx €€ Ŕ $Ą €#|p`@Ŕ@@€@ ŕx €A+đ ‹
Ŕ @ `Ŕ@ €€ ŕ ŕ ŕV @ Ŕ @€ Ŕŕ@@ŔŔ ŕ •ŕf–€ )]Ŕ€Ŕ€@ | ŕ
ŕ
ŕQ €`ŕŕ@ŕ€€€!* !( €€ ŕ ŕ ŕQ €`€°ŕ Ŕ€ "S`– ŕ ŕŕA €Ŕ )Q @€ ŕ @@ ŕz€ ) ŕU €€`ŕđ€@`*‡`Ŕ` Ŕ@ r ŕ €€ € ŕT %9 @@ŕ Ŕ` ` Ŕ! ŕ €€ $@ŕO @Ŕ€ `"V @Ŕ@Ŕ Ŕ ŕ”ŕX €@ŕŕŕ` ö@ŕ €ŔŔ ŕz
ŕŕŕŔŕ Ŕ€Ŕ@—á|*A" ` Ŕ!'ŕ€
ŔŔá|*Ŕ€ @ @ ŕ `€ €Ŕ € @ €€€ŕo €Ŕŕ @Ŕ Ŕ `@ @d ŕv•Ŕ @ `ŔŔ`Ŕ€@ `‘áu+@ ŕ€Ŕ!!@ ŕ – Ž ŕo
@ Ŕ€ŔŔ€€` € !Ë€€ ‘ ŕr € Ŕ ‹ @@ ` !5€ @ €ŕr
Ŕ€ €€Ŕ @ @“ € @ŕt Ŕ€ Ź @#€ Ŕ€ €@ ŕs € Š€ŔŔ&nŔ@Ŕ
€ŕx @ Ź€@!) •€€ € ŕu ŔŔ € € ž ś €ŕt €~€Ŕ@ €Ŕ @ ŕt @€ €!+ € ŕ !'€ !Ä € ŕt ŕ~Ŕ@%G @ €€ŕŕt € € @@ŕs ŕ €@@€!'€@€ á€) € !, Ŕ • € źŕ{ €€@ ‰ €`’ €
ŕ~
@Ŕ €ŔŔ Ŕ €€€Ŕ ŕ} € !˝ Ŕ@ Ŕ€€ ŕ} € !(@ Ŕ ŕ€• €ŔŔ€+/Ŕ +ÁŔ ŕ€ €@@+ş Ŕ€ŕ‚• @@ €€ŔŔ`@ €=& ! ŕS `Ř:đ
r÷b–…«5" â°„‰žĽÔ×DZˇ–lE#۲‘l<ĐpŔ€ ŕI ` ŕ @ŕPüđ
VŘU ¤ˇé/;w â«‚TBIo”«ł ’€Q 缧›’‰xV*ě„ŔŔ ŕG đX0ŕS €Ŕ‡JŘ D·;ˇ·ť˘Ü#9v[ďŮŞz> +[„‘”ŹR°Z26Hbtvb2čp°€ ŕE @XŕÔđŕT ť“äô<ť —¨¤×RC€ ĚŁq4
âĂÉő Eexx]$ľ2´śĽô @VL Č0@ ŕF X:8 đá‚ « @’‹Ę¶«Ů \IvAľ•f0
Ň Ť•±á +EK.çb´0 X”Ä řđ ŕF ÁJČ á- `HäP¶Ü‡H ŕ €ŔnJ\Á pŠ€SňÂşŁŤj? ň©Tě@@€ € Ľŕ4 @ ` âńđŹÎĚ ŕ €Đŕđlqd·`™Ťl Űᤳ¨ś‰d7Ë„ Ŕ @ä* ŕ C*ŕ ŔĚj’§|Ś#ę `ÄĘb„s±
0%YśŹŚ±š‰• ˘š‡\+÷ŁLĐ0€ĐĐA.ŕ8 bˇŕ NThĐ!€ 0´©T‹ŻD;d¬ł ®ffzŚś¤šMČz ŕ ` *ŕ2 ŕ nŕ
€pťś6¤lá Ä hĽG‘Ť±đJQt›ąĄ ĐWATxŹ˘§x8÷–2Đ€00H(Dŕ á<9ŕ @Ś2¶¦|Ú¨#€ „ä7—ś·äIb ą™Äw_mh~—¨¨’b˝` Ä”pP0 ŕ‰ €ŕ˘ŕA
°śF–Üů»X€€€B] z%«ÄîApŽ¦¶ Ş‘Ś•źŤŠź§ŠEř‘4
ŕ´X ŕäk`3ŕK ĐĽ:ÜĚ„ ”Hđ@• „$~®Ů:wšŻł—!Ξ‹ĄŽ†—Şł˘x%Čr2ܨ„X(…O@ ŕQ– ¸ PŹ—n$čH @4… eĄë:{˘¸±‘'Ë©Ž’–}|“®Ă¶Y ·źŽ…(čŔŚä a÷ŕK ``ŕpŘůĹŹ( °4”
Tťő$A‚©Â±Î®•vyŹ®É̲Š<
×̸¨DÄp€ŕPŤŕ ß3$ pČ)Ě ŞNšţ'GłĚ»]Ë´ťťŁ‹•ŻŇŕÍĄl2
ő™HŚĺŕ< Ŕhž(8~¬đ@ ľ L— *LŹşÖżBóÝŔ°Ż±¨ŻŻ±ÉŢÜÁŽN$ ş|>Č( ŔáY- € Ó
őŹČŔ€ěŇ/N•/W“ľÝ¦5ęăÎżľÂČÔËŔľŃŮËťh/(Ř–rđ` Ŕ€ ŕT bĄ¤xčŔ ŔěđIY”5gšĂěžüěÜÓĚÍÓíćÓ˝ľĹľ¤|B 8$$ŐŤHěx€# Ŕ ŕR Ŕ´ Ě ŕbg”;vˇÇóöčáŢáńîâÔ˝·˛ O,,E
â„(Ř °` çúŕ: ŔŔ(Xhđ€( xy@©Ëő™"
˙ů÷ îčÚò˘ŠaC5g5% şvä¤Pđ @áUĂ € @ Đ p4† D…®ŇúĄ)8,&
ýü˙ëĹš€e[a_:ĐŚD Č!Ĺ@7ŕU € ĐđČV°
H˛Ú±0OF?9.&"
řĘ«“‚††‚cóů´h@.ÜxĐ B]ŕU €`đp `ÝZŹŔNŠ´ŕ·Ck__XRE=9' "ôÁ¦žšŁ’‚8íŇĐXB6Ä(`ăW… €`ŕ´6Ł/ÄU‹łâU€{€|si`OA6-%' áÁ˛´ľşš](÷ϡrF6&ř€ě8Xŕ °Ś<wĹ [Ťłá g†‹Ś‰€yk^N>74)öĐÂÂŇĐÇšj*ôŔŹT. ń×ŕM °đŕÄřpÉ!a‘µáš{Ť’Ť‚zom]Q@<="çÉÄÔŘÚĽ“Wŕ«Bě¨ âWí@ `0ÜíuÓ&g•»äł‡“’”‘‰‚yook_OC=*űÜËĐĎѶ˘<ůĆ @ ¤0„®ŕWš ¤@„ŕ,lšÄíÇŚˇŤ†{nh`hdZK?7&óăŰŃŃʢŚeŃ·¨ŽFŘ8` €Ŕ`ŕQ ŕČŤ Źě1pźÎř Ͷ˘Š‡‚s_XVTa`MC:5 ň˙ŰÖƦŹ€4𶹺›<” @ €@ ŕR ŔĐȦ/™ő5t˘ÔÜ»¸„‚{jUHEHWXN:2,#
ýüě߯”Š]§©ÂĂ”€Ž"^ŕ ŕQ €ŔĚ: ý9zĄÖ馶µŻ}qaJ957FNF:/% ţÝŁŽQጡĽ±vôOy @ ęžŕH $€ ¦=¦>€§Őř˛¬™ˇ‚bU>& &5>=7,& ŢDzŽA‰x“Ą–X •€ Ŕ ‚řŕL €ś•8ŞC©Ô ¸Ż¤«WB. /4.* ëܵ’*|hŹŠd Ŕ`` €°óÄŕ4 p€,© H…«Őţą–śŹJ(çí $"
$ ţŘ»‚ú†lx‹Ź P` Đ(( ŕ ä@0 Ŕ8V ¤ J†¬Ő˙ ·“p‚p,üŢŔ¶Ţ
## " ęÓ°Ś
“v€ż)ů `đ Xx„P/ń 8 € `€ŕç;ż €8ž
K†«Ô އVTO Ňş¬—ŽžÇç
" % őćËŻ€™€© V‹Đ„”ĸŔ”8ŕŔĐŘŔ
đXŚX Đ đ 6ŕ( €*™
L‡©Đţđ—p*I2ťtoaXxťŔŰíř %! őÓŔ©zűťÁ,ś´ @Ŕ@p°Ř ¬„`Ś”Üŕ”·Ś
€X°h8Ŕ@€°"~Ŕ€ ŕ+ €& –
N§É÷ۉHmÇ‘Y C‚ĄÄŐßéö %* Ţ»ŞśbÓˇ”Ś !«ĐhĽĐřČäদ¨Ô¤ŕÔ¸Peń@@¤ě
ŚŔ ` @ `€ ` @ @Đ!üŕ € ő’N‰§Ăí Ăq*ţ3Oz$äžĹč ^±ÄĚŐâ ţ17) çȨ—~&ËčČ ŕ €Ŕ ŕ`”Ŕ°Ľ¸ĐđŔ”””ô
.2Ě`aČŚ2. @ @ @Ŕ @@ 1 €7®đ@`@ ŕ ŕĺ‹L¦żäó±Z4dë,„Dź´ľÇĐë /?> őúüܶ’•Iý (Đ `ŔHxh´Ôěě°ÔÔ”¤Ěü.6@ Đ` č6jhôđ``€ °Ŕ !" › @ŔŔĐâ‚ŕ €č΂üF…¤˝Ü嫉c |p4Ą@„ŐVj‘Ą˛ŔĂŮ
+FD1ř˙őǤ…SĐŕX°`Ŕ! xĽČ°Ŕ´p hś ě Nx0Ä° €xn\~°đ ‚°Ř`đ `„( € HP0€ ŕ °°jë<źąÖŮŁënŢśHB˘ F˘R„•¦»ĽË(AO= üó÷ý÷Řşn1„8 0 “ € `PĚp`PH„h(@ Ŕ(0Ŕô4X€b$Ä`06—
Łvj `Ŕ(ÄŔĐŚ8X †@xŔ0ŕŔá €€ŤIÔ0v™´ŃÎťö{őŤ¨¸´´ é8n‡™¶şÄţ&ARD: řű˙ěŃ°ŚR Đ° 82e @ Xś@ 8h0Ŕ`@ŔHśđ8Jh\ ”X¨ nH"śŕ ”ĽxÜ0¤@ Ŕ Đ ŕP € ŕ 0X$¸ e‘¬ĘĤLűŽ ľbŕ´ ®%?Vz‹ŻąÂú&COH7 ˙ţçɤz °! . PŚ¬HŔ°@Ŕ€@@€đŕH¨8TTZDÄśpPŔ
4pŠVś0¤„ XřP€Pŕŕ Đ( ŕ” ŕ P‡˘ľ»”I¤MĄ<ä‡fŵ`Łj€¤¶Äű)HSM;*
űăż” Ŕ"9 ` ”0´ś„°Ŕ €@@`€đŚX„"Bxv:0@¸ đśřH6"¸P $`ÜH X đ(Đ€ ŕ €Đľsë7u•ŻŽH·v&ŢŠ, ĂŔ9^~r°Ç -NXQ7# řŰŻ 1 Ŕ đ °¸°p Ŕ €€!/ ŕ@0¸&Bz†ZFP ¸P 0`ČČÔÄŔ”hĐ„0ś€X(`$˝ ŕ Ŕp:Ŕ X†šž‡G Ĺ‹E ´T ůGçNhŽ§Ç 2T\T:%
ôË €Ŕŕ 80ŕxěô„ŕ%Ĺ €Ŕ €€° @°ČTŚŤj ÄŔđhŚŔđ¸ŕĽ ĚĐŚx8X`ś¬¸°ŕ “ ŕ&:mŚ}EŇ™^ Ö…,×L¦‘µ3LY‡›Á7Y_U="
ć !^ 8``H xpxp` h € @°ŕ€Ŕr|—6ě¬x¤Čśś
đx2(ŚĽp¨¬¨@â U ĽĽhŇ Mmve>ןj'ć“,Á’t®12E}Žµ<]`*
ű Ŕ€` đ H8P} @ @ 8hĚ ,F‡"đŔś¸ŔĚ ¬´¨ĚÄ ¬PŕÜŕpB„ŕ 8p)¤ -KVL1 Ň›b ŰŽ,‰ĘŤ 2c„¤@_ _R@*
J‹ ŕh”*đŕđŔ€@Ŕ Ŕ €€ Đ(0 €¸ĚŕĽ<$ 6"Ř0€xx”ŕÜ0 ¬Đ°`` ŕ
ăsÉ *72 ĆŹK Ľr¨|…–Ĺ« :g–C_\OA0
@ °€(Ř” E¦ ”@ € ĚäŘŚHśŕ J086< ”” –
„”śŘ謨ČÔ ŕ Ŕ‰%’Őĺ˛3ďš@ä€Ë‰°ę‹ 8‡öE]WJ@5$ @ `@ŚP”x`€Ŕ#‚ – č*2 ü Z€Šv2Ä´Ü XđHô4T~F, ,$Vŕ! đĘA“ÂâęŰ˝—_Ę|°@´V÷"Ěţćę qčF[P F@:, ţ €• € `Ěh¤ %MŔ€ € ŕ ä8Ŕ¬:0x~‰€ĄR:řŘĚ”hp¨JbxXH<ô€Há"+ ^ç:š§Łm4ţ¦Lđ„Pé4Ç$´ŘSâDYHBA?4 •űř @ €Đ`¬´ Ŕđ Ŕ`' ÄÄ°.ěŕ@pt„>
ä„€Ľ€ŚPđ$Š\JÔdśx@%ŕ" ŕŘ|ŮB[_M- ¸HĐ@¨=Dŕ™?‰µFăCV@>
BD<(
úďţ)h@ ° ”h 0đŔŔ!6@Đ@ €”¤„Ř´H¬Ě6h.4ěh8H@„ŘÔřě¸ë Ţŕ 8 płč đŔŤ@đ (C·ĹJĺŞJéAO9:B GC0 űčó€S
X`Ŕ Ŕ Ŕ Ŕ @pŚ” p€Ľ:h üxhhŚśČÄxřś° ŕ( püH‡¤´´Ł†H
¬HĐ` `Çţ˙Ŕc´[ńUSK6 üç ŕ €@"ÝŔ Ŕâ ~ŕ# €Ŕ@ ć ĐŔ €@ € xŔ¸”`»ćËá
AXTM9=X ř ŕ[ €Ŕ ` ° €'j •°ĚŔśx¨»ßĐę#G[VP>*'%,P -zŕ\ Ŕ " `@ @” ĐĚĤh‡¶ŃĂń 2R_YSC221 ¶ŕ\ • C0`” đĚČ°9š˛Öó:E_c^XH:==( ŕ_ €$ˇ $o€€ €Čȸ¨Qg»öGZpke^LAHK1 ! ŕa €Ŕ €‘` ”ĽÄŔ´đÍ;Âĺ˙Oo€xocRFNU9%&*€ ŕ^ €€a%€ @¬¸ŔĽČąĄ‚şŮî.zfWIO[A,-3 ŕp `x¬ŔÄȲ%'”ĆŰ3‹ŹŚ‚jZIM\F36; ŕo €€`Ś ¸ČĐ ÷ qĂ)•™–†m[HFYI9>D€"ŕk Ŕ `Ś¤ÄŘŘ`Ŕ^•Ż4 Ąťp\F@RKAGM€€áęŕc (ŐŚ¨ĐŕÔ”ĚV‚›@Ş°˘‰oYC8KKHPY ŕn €`Đ0h ĽČ¨lPgŚO˛·ĄlS=3CLM\f ŕo @xŚś°°N KKd¶şĄ†fJ70>LRetŕn•€ ŔP€0¤€üŁ E+}¸· ‚_@1.á§ýk›‰X@QvwtJ ŕ €°,‚ť–\Ô`6ŕD € xx!, Ŕ`Ĺ—
jO¶ż-’Ą{0%=cjZJ é ŕ
f26fŞŘîŢHř'ŕ €€8¬äÜ ] /G Ŕ\/LŇ·űh¨›J
6^^A%řÄ ŕ
4r˘F¨ă ˙â°b¤`ŕ ŕ4
@ ¬ôŘ”0Ŕ` €” ěĽ, ęÄŰ)©~ě8bX0 Ńł ŕ Ľé鼾µĂŢçÖµśr@ŕ ŕ6 ŔĐŘüč¬`°` –ÇŕőÍŐ Yź”3ĚÝ DjU ó¸§ ŕ ˇ7/䌏°Ĺż©‚hNéG~@(¤ŘÜ´xĐ$Ż a,€*ÎŢËŃř'€™fí¨ű:YqKŐ˘ŕv zuýľŚ„®Ă˛’T řaúŕ `Hŕ1 €°X¨Ŕ°„0ŕ%Ú@ @\8ź¨ŔđE‹‡&®Ŕ!]jm4ň¸Ź’ ŕ
Hť.Ř^–¨˘*ě%}ŕ € 'ě@ €€€"±ŕ!ŕ đhś HŔ@•cŕřb“Ů.hŽ^é©MxnYĘź‚ŠŕG
.Ţđx >4ŕäâV€8€¬Ěě ä €Ŕ€ŔŔ ŔŔŕ!—
pŚ„X đŔ°€@ `7đŇ;« .P…‡'łÝ3r€c:q ŕ
:Ŕ\°(”&d üŔô`8ŕ C°p¸Ôü ÔpĐ Ŕ@ ŕ$ŕ €0x€`8)ď(Ç€•Ŕ¸Şkő1P|‘`ä¶]sLĂ™{`w ŕ N@€ĐhZ^ Ě<` Ŕ Ŕ ´8€ PŚ°¸¬ŔĚŕôä€ŕ"’ ŔŕP€pH8( 5©G ŔřĽ'Wz’Ś ˇëAsyV1ü©‹\Ke ŕ ś Đ€ Lŕ *¸ŔŕHÄ°ś¬´Äŕč¸XŔŕ˛ŕ €(€hH@@80ŕ b pO{Ź—iĚš\o\:Ř}<2Q ŕ @—K€@€„đ` Đŕpś°€ŕxŔđ` đ €xpp`0Đ ŕ" hŚ„hXP. €•€áÂ-tŤŠ*•Ć9aU:' ÄŠb < ŕ 1Ľ”ŚôŹL$ŁtŁF‡Śŕ” €F8Đ7Ę°€@ á$ľ € 0€Ś?Ţ!* Á+^S‰’fî‘MT6! ą|F ő„—Đ"ĽŢť#/sßŃşĘz8 ¸¤Öđŕ€&ł8$ ŕ- ŕHxpP0 ŕŔÂVä#wŚŚ~6¸Ć-R@ ý°c*ńü ŕ x>T^Ň( [rLĄ‘~Y{RĺÚNŘ 7 ŕ¤$bPp ŕ+ €`đ0@0đŕ˙đ ‚%ßK€wX˝
HM/÷˘D
Ôř ŕ xŐ|Ůĺç‹HTQ‚K5$/˙*Šřŕx¤üÄž ŕ 3ŕ Ŕ€ŕ!$YĹĹÖ °~}`mf[=ôí4RB( ç‰Ň˝úY1ŕ ´‚Ç Ćdŕćĺ/â:
&ąůÓÍ D,6ÜČĚ :ŕ) Ŕ` °€@ Ŕ DÜş1PNPN,ęIN9., ÇKŇś´ý ŕ ”¨Ä¨)[ŘĎĽ<" ŹÜxôlLPh\Rá-'@ ‹DJ ŔŔ” /ď12:NM :MH;:1ď— Ľ˙ ŕ 8„ #ÄXŠD ŕ) Ŕ €€ ny<&@WA!HLJD:%ţ´Kź\“ĐůŔ!ŕX €Ŕŕ ŤŔ ”µ™,Y4RS!2LSUE%
Ę‚
‡€°ëůŕ "ŕW €€ŕ•`ĹAMDU6
:WcW3ć¨Z˙ ŞÚ ŕ` ŕ•0„^/*E8 CimEňŇźY Đßă ŕM á ŕĺžJ) ÷'Qsh/ňÓźc"
&( ŕ` €Ŕ…9Ŕ H ą đëČÓ2YnX&úŐśg9 -13˙6˙7ŕ$ €ŕ•hl"¬Ëź—ŤŽÂ9V^C# řË’iJ.(137űc‰âX €„HŤ‚Č«qŽń(>GB/
趆kS4%(+4@ ŕp P"
EŞÓ©›ă21*ţÎź{iQ. ,Aŕq• ŕîH’ĐžŁĎÚ Ý°ŚueF"#? ŕp `śÓ„Ľ»łÁËćÝ·‡~f; = ŕb BíĺáŔ jĺvŐŕ«ŠŔë áż±•ŽŠr>âbí €4x ÉX` <Ň™¤şś_©ŰÚ±‰›ŁśO ŕb• ŔĐ°€‹®€ €($v„)2~¨şś8\žąş°Śd ábÁ đ ŔMĘŕ ńů*ýď=d_śč†ÇÝÎÁp( 1$#B ŕ_ 0(Ŕß˙Î` "ľÉ»Ĺř
ë‡j‰%®íüßĚšv/'B)*I ŕ] "ä8P@Đŕ•łÎłŽłŚ0D§eÎ ęÓĄ7,H.7S ŕV € @Đ(XhX0 îöxµ˘„r ¸„"’ę
íÓ®‹B+?8L_ ŕS €€€ŔŔŔ
@ŔHx€X Ŕ@ ŁŔ (H˝p,Ü€đ€h0¬ô ěÍąI"0Ljmŕ "ŕI €Ŕ@ đ P€ś¨śx8ŕŕ•äh”ľH” XÎ`´é čÉÉĄI-oŕąŕH • @ 8h¬ÄŔ¨„8Ŕ ŕ .†ĐŔŕ¨xr§ŃóâËÜB >źŽŕ%ŕJ €Ŕ €ŕP ÄŘääÔ°x`á)E (!K°°Đ¦!hşă˙ŢŰěŞ<c®¶ź ŕW € €ŚÔôĐ0€ ŕ € p:` @ ŻEx¤ÓěŢőő A7ŽÎͱ ŕW € ŚÄčđčĚś8 áŔ <µ° ¬–ß6‰ÁŮăđ•Xp±ďăSľŕS € ` P„”pż˙ŐŔ ` Ŕ°Ŕ@pj”ÇřM¬Éć
ă—„ťŘöź˙
´ŕN €'Ź `€°đ5â ŔâS Ŕ,2 ”`@ŕ2ü6@PŚ–¶ŢŢ«§Äţâ ŕ €€ ŕ) € @€Ŕđđŕ ``€ ç—` Ŕ( Ŕ° Ü6 ,Ň}šÁńĎĚč
ŕŐ ! ŕS <=>=>` ?@ @` `
ŕ €"==<;ŕHŽ;<<ŕU • l@t€†`–€
ŕ @ ` ŕK–<< Uŕë ;@‘@ŕ•ŕŹ€ @áG- ”ŕ T@‘a,€á+@€
9@(@ŕG–aŕT€l } ŕ• @Ś@ •ánŕ ŕ6 =á‚ < i m @”ŕ• @• ÍAi` ŕ×ŕ ŕ•a, _ ŕ ;@j @} á+Ŕ Ś < @šă 2ŕ ; ŕ&`ŕ•c`|@”ŕ•!·@ ´@@Ś #Ćŕ,• 9ŕ–á, ;Bo €”Ł…@ “` @”@\` Íŕ$ @Ëá
ŕ "F aŕU`• ”ŕ• * A¶ @Ź@ň@ŕSŕ c Öŕ6ŹŕI@1 ’ ŕ
• ‹ S •`![ ŕ?ŕ Ë Cŕ7!,!áM •` !˝@ € 'A, Ž@R / ŕ Šŕ @fŕ–! mŕ–@sa( `“ @ •Ax "@ +€@ŕ( A, Úŕ
!, ŕµ sŕ
•@ľ@I€–ŕPŕ
ŕ! A,ŕEŕ ŕ–ŕ •A%` ` ż I!` ŕFŕ( f ŕE€–ŕ "6a,€’ŕ”€–@K ™ŕ)†ŕ @•ŕEfy mŕ "!ńŕ• –ŕdŕ( •ŕDd á-ŕ•ˇĂ#=ŕQá,9ŕ4,ŕ–' a+Á Ŕ % ’ ŕ•Beá)*ŕ "G@ j fŕ•ŕ –€4ŕJ ŕ –ë ¤ €á +@•- *@ %V`ŕG $; m@oa™ €ÁŔáÁ€–Ŕ—!W@ !,`ŕH Ŕ– o kŕ•a-@' )Á"ŕO•AÄ ‹@ (Ů$Á ~á
+ `& ż@Ŕ€ ŔŕI AÄ s •;;` ŕ• • –!Ă€Ź€ŕL $ p •'@ "Q` ŕ ŕ • áRÁ @Ś! •`ź€“ŕ•`)@• @>ŕN !, ŕ • ľ!Â@+`ŕO’@ !#@k!Á vá +á- – `>ŕN $ ˇŔŕ•`—`ŕ `Á -@ ŕO‘` 'ÇL~bV †ŕ ŕ •á .€>ŕO &< ”ŕ• Aŕšŕ— ’@ A-ćNqŕ –B] }`Źŕ•ŕ–`“ –âUí` €–ŕ•ŕ–ŕ]•K a á
+ŕ•Á-á\+A,!Äŕ•ŕ–C‡äTŻ !+ŕ"• –€•ŕVš •!*áÁá
,d#ľŔÖŕL Ŕŕ•áĂ!) Á+äU¶%âÁ+ Aä` ł€ @”@–A "Ž`ŕN ŕ• €– ’ =QÔ “ ŕO“Ŕ•@zá+` `—@” 5@“!ÄâPó+`z AáÂ` ŕ– 3ŕ ˇŕK ŕ• – € ¨ŕ @–A) •#@ $S ŕ•á@<)ţŔ‚ŕ •@?ŕ €— Ša- “@ < BŔš`á;Aŕ•á Á }` ŕ“Ŕ!+5ż yŕ • Ł@Âd ` Ŕ®ŕ' ŕ•č 3€“` ŕ ` • —bď z(q @Ł €•(bá,Őŕ•`{ŕ ŕ•€@‡ l€•ŕ AA Š@‚@® % !ŕ a-Je@!ŕ • > zBě`!"l Ž € }9ŕ– F Š`— ŕ”@ `1€ ŕ ;MŔw`y ] €‚ @ @
@o 7Aŕ @5`—@č @&! + ‰aÚ ` ŕ fr`x@ u€^ŕ –` “ @j@hŕ’` 5 Ś €`% ”@
@ Łŕ B p ” V [ŕ•@Ž`??!Á g! _ŕ €
7@@ @— @ g!5 7
˘ŕ !o p @•@– € €@€ ů Yŕ• ŔŚ A€ź –
@ €4 ŕ*•,ŕ“@Ť` @ť
?? ő!`!Ă@a ca7` Ŕ‹@• @ @á) O=€x u J •á *ŕ ? 1 l g ;!7€–
’` € ŕ Îŕ • „ŕ z u@V"ó ` @Śŕ @•€^`o ˘#ő€— @=@
ŕ @ .ŕ âX –á Ăŕ’@ ? •!@c @fŕ•Ŕ— ` ŕ ŕ “ Qá ”ˇ,!0` `‹ŕ ŕ•Ŕ ü!Ĺ!3€† 9 ŕ ŕ ,ŕ áÂ@‰ B! ` ŕ‘@ •"X l a =ľ`
9 @ ` @ @ ŕ ”áÂ@‰@T @
`Šŕ @1@‚#ö~;DaÁ@‚` € € ¤€ ŕ!•#… ‚ @` @‡ R€V@ ŕ 0Á, — ŹM†@ €{ @€ ŕ ŕ"•bď`~ `”€X @ŕ ” – n • FA€• ‰ `#@)Ŕ ;ŕ "ď @ `< H@@ @ŕ @ `UŻ “!0€!* w 5 @ @ @ á"+@O@` B J@§ @ @ŕ @
/@Č $ BZ@ E @{ B@ @@ @ @á#(@ c@` C@l%E ŕ• ?€• ‰`q!* =D˛ `€ E@G` ` Tá"˝€ @•@ †`CuÄ ?@ŕ ! • " •@A"[ i@` ź G@€ ` á#Ľ€ !,`G` @ @ ™ †ŕ •DŻ@Ć eA$@ŽŔ @Ť@ @>`O`
@ ľá() ŕ– F@Vŕ• =D –ŕ• Š @`
@L Tŕ.•@F >@G k !ľ!) ?á+A, ’@ BP`–ŕ— ŔM@N W á)Ŕ A, čŕ•`Š •"ë`
` Ŕ@`“@ €” ŕ-“ a! ”@Ł čá+aą ”ŕ’ ŕŤŔK@– ŕ1 ŕ –A*@• R!ż@ !ąŔ `caŹ ”ŕ• 7ŔŹ@G@Bŕ5”ŕ&•€ ŕ”@
@•!1!H
`żŕ0 ć€E T ;!Ť !+A á +ŕ
€‹ ”` € ŕ. `ć@ç Ŕ•%ßAłá ľ@ ?Á ŕ @Fŕ ŕ ŕ, ĺ€ ŕ •OU ?ŕ• @ •ä{ŕ5‡ŕ ŕ”ˇÁ!, ”ŕ•Fqá5–ŕ !kŕ @”cA+@ @ŕ UŔ•áBŕ ŕ• ”ŕ•ŕ^ • @ !( á+ŕ^ `ŚŔ td ŕ – ŕ_ Ŕ’ pŕ •`– @ŕ "ŕ\ @–ŕ i˘ěáx+ŕ ŕ•á
+%ýŕżŕb "ű!'@ ŕ •!UŕcŚŕ $Ľ`“ŕ•ŕ)ŕc ŕ•ŕdŠŕ ŕ”á Áŕ)ŕc ŕ•„° @ŕ ŕd‹ŕ ŕŚ•ŕ”A+HÎŕ ŕżŕb ä $>>@ŕ ŕc‹ŕ ŕ”.Şŕ‰•°} @ŕ ŕŔŕb Ĺ@ ŕ•áŔ%á.¨á|+ •!1+3 • =Âěá Čác¸Ŕ Š€Ŕ•ŔŤŔŕk•$¬€ @” šLćŕ ŕ Ŕŕa ŕ •` !0âVŕw•If <‚Vá0áb# ŕ @ ‚Vç Ŕ+ŕa @Ŕ *s< Ŕá*ŕ•@Ńę% bŕ0 ŕ 1;;`Źá*ŕ •€X# ŕ%– `€<ŕ ŕ” naŔ/Ö @ŕ ŕ B ;$Ř@ ň&® dŕ !(@îdA) 0bëŕ”1“ŕ `‘` ŕ&–ŕ ”@•!Y@”aŔá* ?ŕ
•€ŹH^ŕI•`”6ŕ•AŔŔ”ˇĹŕ
•» áG+`•@ !żA,!]#ŕ ”€• ŕ .ÉŔ“ŕH•ŕ”€• < @ @@ŕ“<óŕ”=Xŕ " `ß@ @ ` ŕ Ŕ• ’@ @=5 ‹@Ť @ ?ŕ•+ŰŁ€!)äŔU@ @T€ ŕ ”, `“`”€• ‹@Ť ”ŕ• Ś7ő @”ŕ•Ŕ @ \ŕ „ ” 9 ç!ŕ Ť@“`”ŕ •IŽ€w >$ĺ :Ŕ =ŔV & ]@— ŕ !ŕ á j ’ @6ŕ• ?€” ŕH ş > 1@w Ô8‹ tŕ ""f í ` €–ŕ ŕ ŕ= c`7@• Š ?€@“ ?ŕ
• ? 1-=;%á\5ů% ŕ €Zŕ ŕ; b€•ŕ”ŕ
•#‚ ?!(ô "kŕ%@ A€Ŕ “€Ňá * }`1ŕ w0Śŕ
™>> B€“@
ŕ* • Ź` €Ö h! a* €ŕ ` {ŕ
—` @ž#ŕ* $Ů Ő€‘ €”á*@• aŕ
• — € š` šCŕ+ @•ŕ  $‡ ‰ `kŕ • .@ `Ž !!!&` ŕ- •@ “béA*ŕ Ž€ A ŕ•ŕ › ˘ŕ. ‰ŕ €Eá *Ŕ Ŕ”ŕ0eŕ' ŕ “ĺ×Ŕ ŕn• ‹ŕ "ëáŔŔ” ?áÁŕ> +ŕMáŔ ŤBě ŕ`•€– ŕ (*€ ŕ ŕ%ŕV a,ŕd ” Ź@Š ‘@ @
ŕ'ŕT ŕ•HĂ `“ŕ˙•áŤá+Á9@ ŕ(ŕd ˙˙ !˝ `Ž ŕdŠb[Á1C€ ŕ“ ?ŕc•Gçá, ‰ŕ ŕ^ (Ę@ á + ;-ĺ Šŕ”ŕa• ”ŕ•$± ŕ
ŕ\ @”iőÂV -I(¸` Ŕ @ ŕZ ”Ił ”ŕ•@ ’ ŕ ŕW’#s` `”ŕ•&s Ź€ ?#‰ŕ™ŕU•€‘ *`–ŕq ‹`“*č ŕ^•@Ť Ŕ”A,ŕ• Ś • ŕ_• ‘#v ` "Xŕ •!"@“ !4ŕ^• ”`•@y€ ŕ•,¦o‡ • ” – ŕ ąŕH ‘` !* ’ á* •,Ş Ŕ•ŕ ŕS @v!Ł"?ŕ”1ě-5(; ŕ‘@ ŕT“ •€ á)@•
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Script to extract a small grid of precipitation out of this file:
ftp://ftp-cdc.dwd.de/pub/REA/COSMO_REA6/hourly/2D/TOT_PRECIP/TOT_PRECIP.2D.201512.grb.bz2
After downloading it and uncompressing it, just run this script for extracting a small grid.
For more info on these datasets, see https://reanalysis.meteo.uni-bonn.de.
"""
from osgeo import gdal
import numpy as np
import blosc
# Read a 150x150 grid from the first band
dataset = gdal.Open("TOT_PRECIP.2D.201512.grb", gdal.GA_ReadOnly)
band = dataset.GetRasterBand(1)
precip_data = band.ReadAsArray(0, 0, 150, 150).astype(np.float32)
# Compress the chunk and write it to a file
cdata = blosc.compress(precip_data, cname="blosclz", clevel=9, typesize=4)
open("rainfall-band-150x150.bin", "wb").write(cdata)
Blosc-c-blosc2-d3bee7c/bench/results-corex/ 0000775 0000000 0000000 00000000000 15255167342 0020644 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/bench/results-corex/blosclz-cl1-sum_openmp-corex.out 0000664 0000000 0000000 00000026024 15255167342 0027024 0 ustar 00root root 0000000 0000000 faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11810.5 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.4 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.0634 s, 6016.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.032 s, 11909.0 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.731 s, 522.0 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.0637 s, 5988.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0176 s, 21623.0 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.5 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.0347 s, 10988.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0176 s, 21645.4 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.729 s, 523.0 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.0345 s, 11062.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00954 s, 40004.2 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.728 s, 524.1 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0181 s, 21039.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00963 s, 39614.4 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.731 s, 521.6 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0179 s, 21284.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.00692 s, 55139.5 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.731 s, 521.9 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.00924 s, 41271.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.00676 s, 56435.1 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.728 s, 524.0 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0092 s, 41452.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285240
Sum time for uncompressed data: 0.00661 s, 57749.5 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.3 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00663 s, 57542.6 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285248
Sum time for uncompressed data: 0.00669 s, 57012.0 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.3 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00664 s, 57450.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290212
Sum time for uncompressed data: 0.00657 s, 58024.4 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.729 s, 523.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00586 s, 65044.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290216
Sum time for uncompressed data: 0.00675 s, 56531.6 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.5 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00582 s, 65535.6 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.0078 s, 48889.8 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.8 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00705 s, 54084.9 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.00768 s, 49663.5 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.729 s, 523.1 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00716 s, 53282.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00683 s, 55813.8 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.7 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.00584 s, 65367.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00688 s, 55473.4 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.73 s, 522.2 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.00589 s, 64789.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310868
Sum time for uncompressed data: 0.00689 s, 55357.2 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.729 s, 523.0 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.00511 s, 74655.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310868
Sum time for uncompressed data: 0.00695 s, 54890.5 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.728 s, 523.7 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.00514 s, 74246.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311968
Sum time for uncompressed data: 0.00678 s, 56296.0 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.733 s, 520.5 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.00448 s, 85203.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311968
Sum time for uncompressed data: 0.00679 s, 56221.8 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.732 s, 521.1 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.0044 s, 86628.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00815 s, 46808.0 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.732 s, 521.1 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.00942 s, 40495.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=blosclz ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313064
Sum time for uncompressed data: 0.00814 s, 46849.2 MB/s
Compression ratio: 381.5 MB -> 103.3 MB (3.7x)
Compression time: 0.731 s, 521.9 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.00708 s, 53853.5 MB/s
Blosc-c-blosc2-d3bee7c/bench/results-corex/blosclz-suite-corex.out 0000664 0000000 0000000 00000072153 15255167342 0025322 0 ustar 00root root 0000000 0000000 Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
List of supported compressors in this build: blosclz,lz4,lz4hc,zlib,zstd
Supported compression libraries:
BloscLZ: 2.3.0
LZ4: 1.9.3
Zlib: 1.2.11.zlib-ng
Zstd: 1.4.9
Using compressor: blosclz
Using shuffle type: shuffle
Running suite: suite
--> 1, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 1
********************** Running benchmarks *********************
memcpy(write): 864.2 us, 4628.8 MB/s
memcpy(read): 482.6 us, 8289.1 MB/s
Compression level: 0
comp(write): 314.9 us, 12704.0 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 470.0 us, 8510.9 MB/s OK
Compression level: 1
comp(write): 848.4 us, 4714.6 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 375.0 us, 10667.6 MB/s OK
Compression level: 2
comp(write): 808.3 us, 4948.9 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 393.9 us, 10153.7 MB/s OK
Compression level: 3
comp(write): 835.3 us, 4788.5 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 392.9 us, 10180.4 MB/s OK
Compression level: 4
comp(write): 1251.7 us, 3195.8 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 430.9 us, 9282.2 MB/s OK
Compression level: 5
comp(write): 1279.2 us, 3127.0 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 431.5 us, 9270.3 MB/s OK
Compression level: 6
comp(write): 1031.9 us, 3876.2 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 456.7 us, 8759.2 MB/s OK
Compression level: 7
comp(write): 999.2 us, 4003.0 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 462.4 us, 8651.0 MB/s OK
Compression level: 8
comp(write): 993.2 us, 4027.4 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 464.6 us, 8608.8 MB/s OK
Compression level: 9
comp(write): 997.4 us, 4010.4 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 526.2 us, 7601.8 MB/s OK
--> 2, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 2
********************** Running benchmarks *********************
memcpy(write): 614.6 us, 6508.1 MB/s
memcpy(read): 471.8 us, 8478.1 MB/s
Compression level: 0
comp(write): 169.1 us, 23655.5 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 240.4 us, 16639.8 MB/s OK
Compression level: 1
comp(write): 480.9 us, 8317.5 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 208.9 us, 19150.1 MB/s OK
Compression level: 2
comp(write): 446.3 us, 8963.0 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 211.8 us, 18884.9 MB/s OK
Compression level: 3
comp(write): 452.1 us, 8848.1 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 211.6 us, 18905.9 MB/s OK
Compression level: 4
comp(write): 648.8 us, 6165.7 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 232.4 us, 17211.8 MB/s OK
Compression level: 5
comp(write): 661.0 us, 6051.4 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 230.7 us, 17339.2 MB/s OK
Compression level: 6
comp(write): 537.1 us, 7447.8 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 244.7 us, 16347.8 MB/s OK
Compression level: 7
comp(write): 521.6 us, 7668.0 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 247.8 us, 16141.0 MB/s OK
Compression level: 8
comp(write): 517.7 us, 7727.2 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 248.4 us, 16101.4 MB/s OK
Compression level: 9
comp(write): 523.8 us, 7635.9 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 273.0 us, 14654.1 MB/s OK
--> 3, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 3
********************** Running benchmarks *********************
memcpy(write): 604.8 us, 6613.7 MB/s
memcpy(read): 437.5 us, 9142.3 MB/s
Compression level: 0
comp(write): 137.2 us, 29160.2 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 165.6 us, 24155.8 MB/s OK
Compression level: 1
comp(write): 329.0 us, 12157.6 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 143.6 us, 27861.0 MB/s OK
Compression level: 2
comp(write): 309.6 us, 12918.6 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 157.0 us, 25470.7 MB/s OK
Compression level: 3
comp(write): 325.4 us, 12291.3 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 159.1 us, 25146.7 MB/s OK
Compression level: 4
comp(write): 512.1 us, 7810.9 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 193.6 us, 20656.9 MB/s OK
Compression level: 5
comp(write): 502.4 us, 7961.5 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 191.9 us, 20848.0 MB/s OK
Compression level: 6
comp(write): 413.5 us, 9674.0 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 202.7 us, 19734.3 MB/s OK
Compression level: 7
comp(write): 416.0 us, 9615.9 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 207.7 us, 19255.3 MB/s OK
Compression level: 8
comp(write): 416.5 us, 9604.2 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 206.4 us, 19381.7 MB/s OK
Compression level: 9
comp(write): 501.9 us, 7970.4 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 277.7 us, 14403.0 MB/s OK
--> 4, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 4
********************** Running benchmarks *********************
memcpy(write): 604.7 us, 6614.6 MB/s
memcpy(read): 435.9 us, 9175.9 MB/s
Compression level: 0
comp(write): 122.9 us, 32542.3 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 133.5 us, 29959.6 MB/s OK
Compression level: 1
comp(write): 264.3 us, 15131.5 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 118.9 us, 33643.4 MB/s OK
Compression level: 2
comp(write): 249.7 us, 16020.5 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 123.7 us, 32335.0 MB/s OK
Compression level: 3
comp(write): 256.2 us, 15613.0 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 124.4 us, 32149.5 MB/s OK
Compression level: 4
comp(write): 366.4 us, 10916.5 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 135.8 us, 29446.4 MB/s OK
Compression level: 5
comp(write): 367.2 us, 10891.9 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 135.7 us, 29479.1 MB/s OK
Compression level: 6
comp(write): 303.3 us, 13187.0 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 144.7 us, 27643.8 MB/s OK
Compression level: 7
comp(write): 294.2 us, 13595.6 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 145.4 us, 27519.2 MB/s OK
Compression level: 8
comp(write): 293.9 us, 13611.9 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 146.3 us, 27332.3 MB/s OK
Compression level: 9
comp(write): 286.4 us, 13968.9 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 156.2 us, 25609.2 MB/s OK
--> 5, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 5
********************** Running benchmarks *********************
memcpy(write): 618.5 us, 6467.6 MB/s
memcpy(read): 449.1 us, 8907.6 MB/s
Compression level: 0
comp(write): 121.1 us, 33034.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 115.2 us, 34720.6 MB/s OK
Compression level: 1
comp(write): 218.1 us, 18340.8 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 100.2 us, 39921.7 MB/s OK
Compression level: 2
comp(write): 211.7 us, 18891.2 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 103.6 us, 38624.4 MB/s OK
Compression level: 3
comp(write): 212.4 us, 18829.6 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 104.4 us, 38311.7 MB/s OK
Compression level: 4
comp(write): 364.6 us, 10969.5 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 138.2 us, 28944.0 MB/s OK
Compression level: 5
comp(write): 365.8 us, 10936.1 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 137.4 us, 29112.1 MB/s OK
Compression level: 6
comp(write): 302.1 us, 13238.5 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 146.5 us, 27306.3 MB/s OK
Compression level: 7
comp(write): 293.2 us, 13643.8 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 147.9 us, 27054.3 MB/s OK
Compression level: 8
comp(write): 291.6 us, 13716.5 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 148.3 us, 26974.3 MB/s OK
Compression level: 9
comp(write): 282.8 us, 14146.5 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 158.7 us, 25202.7 MB/s OK
--> 6, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 6
********************** Running benchmarks *********************
memcpy(write): 605.1 us, 6610.7 MB/s
memcpy(read): 439.0 us, 9112.1 MB/s
Compression level: 0
comp(write): 123.6 us, 32352.6 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 102.9 us, 38863.8 MB/s OK
Compression level: 1
comp(write): 189.6 us, 21095.9 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 86.4 us, 46313.4 MB/s OK
Compression level: 2
comp(write): 179.8 us, 22242.3 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 89.5 us, 44689.7 MB/s OK
Compression level: 3
comp(write): 182.8 us, 21878.1 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 89.5 us, 44701.8 MB/s OK
Compression level: 4
comp(write): 366.6 us, 10910.9 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 139.9 us, 28599.8 MB/s OK
Compression level: 5
comp(write): 364.4 us, 10978.4 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 141.1 us, 28348.8 MB/s OK
Compression level: 6
comp(write): 304.0 us, 13156.3 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 148.8 us, 26874.6 MB/s OK
Compression level: 7
comp(write): 295.2 us, 13548.2 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 149.2 us, 26814.4 MB/s OK
Compression level: 8
comp(write): 293.2 us, 13640.9 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 148.3 us, 26980.6 MB/s OK
Compression level: 9
comp(write): 282.2 us, 14173.4 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 161.0 us, 24839.3 MB/s OK
--> 7, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 7
********************** Running benchmarks *********************
memcpy(write): 626.5 us, 6384.3 MB/s
memcpy(read): 467.4 us, 8557.9 MB/s
Compression level: 0
comp(write): 117.6 us, 34011.1 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 96.1 us, 41631.2 MB/s OK
Compression level: 1
comp(write): 166.3 us, 24049.4 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 77.0 us, 51930.5 MB/s OK
Compression level: 2
comp(write): 158.7 us, 25199.1 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 84.8 us, 47144.3 MB/s OK
Compression level: 3
comp(write): 162.1 us, 24675.2 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 85.0 us, 47052.6 MB/s OK
Compression level: 4
comp(write): 365.4 us, 10946.1 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 142.3 us, 28108.5 MB/s OK
Compression level: 5
comp(write): 364.5 us, 10975.3 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 142.7 us, 28029.3 MB/s OK
Compression level: 6
comp(write): 305.4 us, 13096.0 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 151.2 us, 26461.6 MB/s OK
Compression level: 7
comp(write): 293.2 us, 13642.2 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 150.3 us, 26604.6 MB/s OK
Compression level: 8
comp(write): 291.8 us, 13707.1 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 152.5 us, 26227.3 MB/s OK
Compression level: 9
comp(write): 287.2 us, 13928.3 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 162.5 us, 24609.8 MB/s OK
--> 8, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 8
********************** Running benchmarks *********************
memcpy(write): 595.5 us, 6716.6 MB/s
memcpy(read): 438.6 us, 9119.7 MB/s
Compression level: 0
comp(write): 121.8 us, 32828.1 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 95.0 us, 42121.4 MB/s OK
Compression level: 1
comp(write): 148.6 us, 26913.6 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 70.9 us, 56396.2 MB/s OK
Compression level: 2
comp(write): 140.9 us, 28382.8 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 75.1 us, 53291.3 MB/s OK
Compression level: 3
comp(write): 146.7 us, 27271.8 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 74.0 us, 54037.5 MB/s OK
Compression level: 4
comp(write): 202.4 us, 19763.2 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 81.6 us, 49013.8 MB/s OK
Compression level: 5
comp(write): 196.9 us, 20311.2 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 81.4 us, 49155.5 MB/s OK
Compression level: 6
comp(write): 169.4 us, 23608.3 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 85.1 us, 47006.9 MB/s OK
Compression level: 7
comp(write): 162.1 us, 24677.1 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 88.5 us, 45215.7 MB/s OK
Compression level: 8
comp(write): 162.5 us, 24621.3 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 87.6 us, 45687.2 MB/s OK
Compression level: 9
comp(write): 290.9 us, 13751.6 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 164.8 us, 24268.1 MB/s OK
--> 9, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 9
********************** Running benchmarks *********************
memcpy(write): 621.9 us, 6431.7 MB/s
memcpy(read): 448.0 us, 8928.4 MB/s
Compression level: 0
comp(write): 119.8 us, 33402.2 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 97.4 us, 41068.8 MB/s OK
Compression level: 1
comp(write): 138.3 us, 28915.2 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 70.4 us, 56823.8 MB/s OK
Compression level: 2
comp(write): 132.8 us, 30126.2 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 78.1 us, 51191.9 MB/s OK
Compression level: 3
comp(write): 135.9 us, 29427.5 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 75.6 us, 52901.2 MB/s OK
Compression level: 4
comp(write): 202.4 us, 19761.5 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 84.2 us, 47505.0 MB/s OK
Compression level: 5
comp(write): 200.7 us, 19931.0 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 82.9 us, 48228.4 MB/s OK
Compression level: 6
comp(write): 168.3 us, 23767.0 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 86.9 us, 46031.6 MB/s OK
Compression level: 7
comp(write): 164.0 us, 24382.9 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 90.2 us, 44330.1 MB/s OK
Compression level: 8
comp(write): 164.6 us, 24300.9 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 88.2 us, 45342.9 MB/s OK
Compression level: 9
comp(write): 294.8 us, 13569.2 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 166.4 us, 24032.9 MB/s OK
--> 10, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 10
********************** Running benchmarks *********************
memcpy(write): 605.1 us, 6610.7 MB/s
memcpy(read): 440.6 us, 9078.4 MB/s
Compression level: 0
comp(write): 123.1 us, 32494.3 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 99.3 us, 40289.4 MB/s OK
Compression level: 1
comp(write): 131.2 us, 30494.9 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 66.8 us, 59850.3 MB/s OK
Compression level: 2
comp(write): 125.6 us, 31859.7 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 70.2 us, 56943.8 MB/s OK
Compression level: 3
comp(write): 126.8 us, 31545.8 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 70.5 us, 56742.3 MB/s OK
Compression level: 4
comp(write): 206.1 us, 19405.9 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 91.2 us, 43883.1 MB/s OK
Compression level: 5
comp(write): 201.3 us, 19869.3 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 86.8 us, 46105.1 MB/s OK
Compression level: 6
comp(write): 170.5 us, 23462.4 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 92.8 us, 43100.2 MB/s OK
Compression level: 7
comp(write): 166.6 us, 24013.0 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 89.4 us, 44744.3 MB/s OK
Compression level: 8
comp(write): 164.2 us, 24367.9 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 90.8 us, 44063.7 MB/s OK
Compression level: 9
comp(write): 311.3 us, 12850.5 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 165.7 us, 24141.9 MB/s OK
--> 11, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 11
********************** Running benchmarks *********************
memcpy(write): 625.2 us, 6398.0 MB/s
memcpy(read): 446.5 us, 8958.8 MB/s
Compression level: 0
comp(write): 121.0 us, 33051.6 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 103.2 us, 38759.9 MB/s OK
Compression level: 1
comp(write): 128.8 us, 31064.4 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 66.5 us, 60116.1 MB/s OK
Compression level: 2
comp(write): 119.5 us, 33459.0 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 67.5 us, 59269.1 MB/s OK
Compression level: 3
comp(write): 121.7 us, 32859.0 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 67.4 us, 59380.2 MB/s OK
Compression level: 4
comp(write): 206.1 us, 19403.5 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 88.9 us, 44990.4 MB/s OK
Compression level: 5
comp(write): 202.9 us, 19709.8 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 87.3 us, 45844.0 MB/s OK
Compression level: 6
comp(write): 171.0 us, 23388.4 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 90.8 us, 44074.1 MB/s OK
Compression level: 7
comp(write): 166.0 us, 24093.2 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 93.7 us, 42693.0 MB/s OK
Compression level: 8
comp(write): 167.0 us, 23950.3 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 91.9 us, 43547.5 MB/s OK
Compression level: 9
comp(write): 318.9 us, 12541.2 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 172.1 us, 23243.3 MB/s OK
--> 12, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 12
********************** Running benchmarks *********************
memcpy(write): 608.8 us, 6570.8 MB/s
memcpy(read): 454.6 us, 8798.6 MB/s
Compression level: 0
comp(write): 139.6 us, 28660.4 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 101.3 us, 39474.1 MB/s OK
Compression level: 1
comp(write): 120.2 us, 33285.9 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 67.1 us, 59624.8 MB/s OK
Compression level: 2
comp(write): 115.9 us, 34510.4 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 70.0 us, 57144.2 MB/s OK
Compression level: 3
comp(write): 119.4 us, 33508.2 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 71.4 us, 56023.5 MB/s OK
Compression level: 4
comp(write): 216.3 us, 18490.8 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 95.1 us, 42076.8 MB/s OK
Compression level: 5
comp(write): 213.0 us, 18783.7 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 93.0 us, 42989.9 MB/s OK
Compression level: 6
comp(write): 180.4 us, 22178.5 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 95.9 us, 41720.2 MB/s OK
Compression level: 7
comp(write): 175.5 us, 22795.0 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 100.3 us, 39861.9 MB/s OK
Compression level: 8
comp(write): 176.4 us, 22673.7 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 97.8 us, 40916.8 MB/s OK
Compression level: 9
comp(write): 336.9 us, 11873.7 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 176.3 us, 22691.4 MB/s OK
--> 13, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 13
********************** Running benchmarks *********************
memcpy(write): 623.0 us, 6420.3 MB/s
memcpy(read): 470.6 us, 8499.6 MB/s
Compression level: 0
comp(write): 140.9 us, 28398.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 102.0 us, 39220.4 MB/s OK
Compression level: 1
comp(write): 115.8 us, 34550.1 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 64.5 us, 61988.2 MB/s OK
Compression level: 2
comp(write): 110.7 us, 36147.2 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 65.6 us, 60986.6 MB/s OK
Compression level: 3
comp(write): 111.7 us, 35808.4 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 66.7 us, 59973.3 MB/s OK
Compression level: 4
comp(write): 217.6 us, 18384.2 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 95.7 us, 41805.5 MB/s OK
Compression level: 5
comp(write): 216.6 us, 18470.4 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 95.0 us, 42116.9 MB/s OK
Compression level: 6
comp(write): 190.5 us, 20999.1 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 98.9 us, 40449.0 MB/s OK
Compression level: 7
comp(write): 177.4 us, 22551.5 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 101.7 us, 39345.5 MB/s OK
Compression level: 8
comp(write): 180.8 us, 22120.3 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 100.3 us, 39865.5 MB/s OK
Compression level: 9
comp(write): 344.3 us, 11619.4 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 177.3 us, 22558.0 MB/s OK
--> 14, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 14
********************** Running benchmarks *********************
memcpy(write): 615.4 us, 6500.2 MB/s
memcpy(read): 456.9 us, 8754.2 MB/s
Compression level: 0
comp(write): 136.9 us, 29227.6 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 105.2 us, 38010.4 MB/s OK
Compression level: 1
comp(write): 118.3 us, 33813.9 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 88.6 us, 45134.6 MB/s OK
Compression level: 2
comp(write): 113.6 us, 35217.8 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 82.3 us, 48587.4 MB/s OK
Compression level: 3
comp(write): 118.8 us, 33668.4 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 84.2 us, 47507.0 MB/s OK
Compression level: 4
comp(write): 222.3 us, 17993.9 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 98.2 us, 40739.2 MB/s OK
Compression level: 5
comp(write): 224.4 us, 17826.9 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 158.6 us, 25215.6 MB/s OK
Compression level: 6
comp(write): 193.1 us, 20710.5 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 102.4 us, 39049.4 MB/s OK
Compression level: 7
comp(write): 183.5 us, 21799.4 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 102.2 us, 39151.0 MB/s OK
Compression level: 8
comp(write): 184.8 us, 21639.5 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 147.1 us, 27189.2 MB/s OK
Compression level: 9
comp(write): 351.1 us, 11392.3 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 180.3 us, 22180.2 MB/s OK
--> 15, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 15
********************** Running benchmarks *********************
memcpy(write): 624.5 us, 6405.6 MB/s
memcpy(read): 452.9 us, 8831.1 MB/s
Compression level: 0
comp(write): 148.5 us, 26940.2 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 118.3 us, 33820.0 MB/s OK
Compression level: 1
comp(write): 117.8 us, 33966.0 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 77.6 us, 51513.4 MB/s OK
Compression level: 2
comp(write): 118.9 us, 33645.2 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 75.3 us, 53143.6 MB/s OK
Compression level: 3
comp(write): 116.6 us, 34293.6 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 71.0 us, 56333.9 MB/s OK
Compression level: 4
comp(write): 239.0 us, 16736.1 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 114.3 us, 34983.7 MB/s OK
Compression level: 5
comp(write): 247.7 us, 16150.8 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 100.5 us, 39796.4 MB/s OK
Compression level: 6
comp(write): 200.0 us, 19999.5 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 105.0 us, 38078.9 MB/s OK
Compression level: 7
comp(write): 198.8 us, 20120.3 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 118.8 us, 33682.4 MB/s OK
Compression level: 8
comp(write): 195.3 us, 20483.8 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 112.3 us, 35617.2 MB/s OK
Compression level: 9
comp(write): 356.6 us, 11217.3 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 186.3 us, 21469.5 MB/s OK
--> 16, 4194304, 4, 19, blosclz, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 16
********************** Running benchmarks *********************
memcpy(write): 606.4 us, 6596.0 MB/s
memcpy(read): 438.9 us, 9112.8 MB/s
Compression level: 0
comp(write): 145.0 us, 27589.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 116.1 us, 34462.3 MB/s OK
Compression level: 1
comp(write): 123.9 us, 32272.2 MB/s Final bytes: 1102592 Ratio: 3.80
decomp(read): 83.6 us, 47828.2 MB/s OK
Compression level: 2
comp(write): 113.5 us, 35234.6 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 84.3 us, 47424.5 MB/s OK
Compression level: 3
comp(write): 119.6 us, 33458.6 MB/s Final bytes: 1083544 Ratio: 3.87
decomp(read): 84.9 us, 47125.7 MB/s OK
Compression level: 4
comp(write): 252.0 us, 15875.8 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 104.6 us, 38251.6 MB/s OK
Compression level: 5
comp(write): 256.2 us, 15613.7 MB/s Final bytes: 165210 Ratio: 25.39
decomp(read): 102.6 us, 38987.8 MB/s OK
Compression level: 6
comp(write): 217.1 us, 18424.1 MB/s Final bytes: 141658 Ratio: 29.61
decomp(read): 107.2 us, 37310.1 MB/s OK
Compression level: 7
comp(write): 212.8 us, 18794.6 MB/s Final bytes: 138058 Ratio: 30.38
decomp(read): 108.5 us, 36864.3 MB/s OK
Compression level: 8
comp(write): 207.2 us, 19300.6 MB/s Final bytes: 137626 Ratio: 30.48
decomp(read): 170.7 us, 23428.3 MB/s OK
Compression level: 9
comp(write): 354.8 us, 11272.4 MB/s Final bytes: 73270 Ratio: 57.24
decomp(read): 188.1 us, 21269.9 MB/s OK
Round-trip compr/decompr on 120.0 GB
Elapsed time: 16.6 s, 16318.4 MB/s
Blosc-c-blosc2-d3bee7c/bench/results-corex/lz4-cl1-sum_openmp-corex.out 0000664 0000000 0000000 00000026023 15255167342 0026064 0 ustar 00root root 0000000 0000000 faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11813.2 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.294 s, 1296.8 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.0734 s, 5194.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11792.8 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.294 s, 1297.7 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.0735 s, 5188.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0176 s, 21640.8 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.294 s, 1296.4 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.0404 s, 9430.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0176 s, 21625.8 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1291.5 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.0403 s, 9465.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00947 s, 40275.9 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1294.1 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0209 s, 18257.6 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00952 s, 40051.5 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1294.5 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0209 s, 18262.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.00667 s, 57210.6 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1291.2 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0107 s, 35693.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248988
Sum time for uncompressed data: 0.0067 s, 56951.6 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.294 s, 1296.3 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0107 s, 35698.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00662 s, 57649.8 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1291.8 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00765 s, 49834.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285248
Sum time for uncompressed data: 0.0067 s, 56924.6 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.294 s, 1297.6 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00759 s, 50247.9 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290216
Sum time for uncompressed data: 0.00658 s, 57991.3 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1291.2 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00669 s, 57009.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290212
Sum time for uncompressed data: 0.0066 s, 57761.1 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.296 s, 1289.7 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00673 s, 56677.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295180
Sum time for uncompressed data: 0.0078 s, 48882.0 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1292.5 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00901 s, 42342.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.00778 s, 49019.5 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1292.9 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00902 s, 42268.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305116
Sum time for uncompressed data: 0.00689 s, 55397.8 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1295.1 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.00731 s, 52207.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00685 s, 55666.0 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.296 s, 1287.4 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.00734 s, 51948.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310864
Sum time for uncompressed data: 0.0069 s, 55283.1 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.296 s, 1290.3 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.00638 s, 59755.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310872
Sum time for uncompressed data: 0.00691 s, 55215.8 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.296 s, 1290.5 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.0065 s, 58668.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311964
Sum time for uncompressed data: 0.00675 s, 56497.3 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.297 s, 1285.7 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.00571 s, 66777.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311964
Sum time for uncompressed data: 0.00682 s, 55923.4 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1293.1 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.00566 s, 67353.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313064
Sum time for uncompressed data: 0.00817 s, 46704.5 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.295 s, 1293.4 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0116 s, 32778.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4 ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.0082 s, 46525.9 MB/s
Compression ratio: 381.5 MB -> 86.0 MB (4.4x)
Compression time: 0.296 s, 1289.1 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0104 s, 36658.2 MB/s
Blosc-c-blosc2-d3bee7c/bench/results-corex/lz4-suite-corex.out 0000664 0000000 0000000 00000071763 15255167342 0024371 0 ustar 00root root 0000000 0000000 Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
List of supported compressors in this build: blosclz,lz4,lz4hc,zlib,zstd
Supported compression libraries:
BloscLZ: 2.3.0
LZ4: 1.9.3
Zlib: 1.2.11.zlib-ng
Zstd: 1.4.9
Using compressor: lz4
Using shuffle type: shuffle
Running suite: suite
--> 1, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 1
********************** Running benchmarks *********************
memcpy(write): 852.6 us, 4691.4 MB/s
memcpy(read): 466.1 us, 8581.6 MB/s
Compression level: 0
comp(write): 315.2 us, 12689.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 447.2 us, 8943.9 MB/s OK
Compression level: 1
comp(write): 1935.4 us, 2066.7 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 755.6 us, 5293.7 MB/s OK
Compression level: 2
comp(write): 1945.6 us, 2055.9 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 762.4 us, 5246.6 MB/s OK
Compression level: 3
comp(write): 1958.3 us, 2042.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 760.7 us, 5258.3 MB/s OK
Compression level: 4
comp(write): 1213.4 us, 3296.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 523.9 us, 7635.0 MB/s OK
Compression level: 5
comp(write): 1208.6 us, 3309.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 522.6 us, 7654.2 MB/s OK
Compression level: 6
comp(write): 1222.1 us, 3272.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 522.8 us, 7650.9 MB/s OK
Compression level: 7
comp(write): 1245.0 us, 3212.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 522.4 us, 7656.3 MB/s OK
Compression level: 8
comp(write): 1229.3 us, 3253.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 522.7 us, 7652.5 MB/s OK
Compression level: 9
comp(write): 1265.7 us, 3160.2 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 622.1 us, 6429.5 MB/s OK
--> 2, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 2
********************** Running benchmarks *********************
memcpy(write): 405.3 us, 9868.3 MB/s
memcpy(read): 440.6 us, 9078.1 MB/s
Compression level: 0
comp(write): 167.2 us, 23923.3 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 237.4 us, 16847.3 MB/s OK
Compression level: 1
comp(write): 1036.9 us, 3857.6 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 400.9 us, 9977.2 MB/s OK
Compression level: 2
comp(write): 1030.8 us, 3880.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 406.1 us, 9850.6 MB/s OK
Compression level: 3
comp(write): 1028.9 us, 3887.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 403.0 us, 9926.7 MB/s OK
Compression level: 4
comp(write): 628.2 us, 6367.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 277.5 us, 14415.0 MB/s OK
Compression level: 5
comp(write): 627.7 us, 6372.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 277.9 us, 14392.8 MB/s OK
Compression level: 6
comp(write): 627.8 us, 6371.2 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 278.0 us, 14389.1 MB/s OK
Compression level: 7
comp(write): 627.5 us, 6374.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 277.5 us, 14412.4 MB/s OK
Compression level: 8
comp(write): 635.4 us, 6295.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 275.8 us, 14505.8 MB/s OK
Compression level: 9
comp(write): 667.2 us, 5994.8 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 329.5 us, 12140.1 MB/s OK
--> 3, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 3
********************** Running benchmarks *********************
memcpy(write): 405.4 us, 9867.1 MB/s
memcpy(read): 429.0 us, 9323.8 MB/s
Compression level: 0
comp(write): 137.7 us, 29058.1 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 163.1 us, 24529.9 MB/s OK
Compression level: 1
comp(write): 703.9 us, 5682.7 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 275.1 us, 14539.2 MB/s OK
Compression level: 2
comp(write): 723.5 us, 5529.0 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 298.0 us, 13422.1 MB/s OK
Compression level: 3
comp(write): 723.6 us, 5527.7 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 292.5 us, 13676.1 MB/s OK
Compression level: 4
comp(write): 501.6 us, 7974.7 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 224.1 us, 17846.1 MB/s OK
Compression level: 5
comp(write): 477.0 us, 8385.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 225.6 us, 17734.4 MB/s OK
Compression level: 6
comp(write): 474.4 us, 8432.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 215.3 us, 18580.1 MB/s OK
Compression level: 7
comp(write): 487.3 us, 8209.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 215.6 us, 18554.7 MB/s OK
Compression level: 8
comp(write): 479.0 us, 8350.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 214.5 us, 18649.0 MB/s OK
Compression level: 9
comp(write): 671.3 us, 5958.7 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 339.2 us, 11791.6 MB/s OK
--> 4, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 4
********************** Running benchmarks *********************
memcpy(write): 414.3 us, 9653.9 MB/s
memcpy(read): 447.6 us, 8937.2 MB/s
Compression level: 0
comp(write): 126.7 us, 31573.2 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 132.2 us, 30260.9 MB/s OK
Compression level: 1
comp(write): 574.9 us, 6958.3 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 229.0 us, 17468.0 MB/s OK
Compression level: 2
comp(write): 572.6 us, 6985.5 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 229.7 us, 17413.5 MB/s OK
Compression level: 3
comp(write): 583.3 us, 6857.8 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 229.6 us, 17421.8 MB/s OK
Compression level: 4
comp(write): 354.7 us, 11277.2 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 160.6 us, 24911.0 MB/s OK
Compression level: 5
comp(write): 358.6 us, 11153.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 160.6 us, 24901.9 MB/s OK
Compression level: 6
comp(write): 355.7 us, 11245.2 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 160.9 us, 24860.8 MB/s OK
Compression level: 7
comp(write): 353.2 us, 11326.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 160.9 us, 24858.3 MB/s OK
Compression level: 8
comp(write): 356.5 us, 11221.7 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 161.1 us, 24834.3 MB/s OK
Compression level: 9
comp(write): 376.7 us, 10617.7 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 184.0 us, 21736.9 MB/s OK
--> 5, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 5
********************** Running benchmarks *********************
memcpy(write): 416.2 us, 9610.5 MB/s
memcpy(read): 439.4 us, 9102.7 MB/s
Compression level: 0
comp(write): 118.4 us, 33778.4 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 111.1 us, 36017.4 MB/s OK
Compression level: 1
comp(write): 470.6 us, 8499.5 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 186.9 us, 21397.3 MB/s OK
Compression level: 2
comp(write): 467.4 us, 8558.0 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 190.1 us, 21036.4 MB/s OK
Compression level: 3
comp(write): 468.0 us, 8546.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 190.3 us, 21024.5 MB/s OK
Compression level: 4
comp(write): 353.3 us, 11322.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 164.1 us, 24378.6 MB/s OK
Compression level: 5
comp(write): 358.3 us, 11163.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 162.4 us, 24626.3 MB/s OK
Compression level: 6
comp(write): 352.8 us, 11338.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 161.8 us, 24726.0 MB/s OK
Compression level: 7
comp(write): 352.1 us, 11361.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 162.5 us, 24612.2 MB/s OK
Compression level: 8
comp(write): 353.3 us, 11322.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 162.8 us, 24576.1 MB/s OK
Compression level: 9
comp(write): 373.1 us, 10721.6 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 190.8 us, 20965.9 MB/s OK
--> 6, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 6
********************** Running benchmarks *********************
memcpy(write): 405.2 us, 9872.7 MB/s
memcpy(read): 426.4 us, 9381.2 MB/s
Compression level: 0
comp(write): 123.5 us, 32377.9 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 101.4 us, 39449.8 MB/s OK
Compression level: 1
comp(write): 398.3 us, 10043.1 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 160.2 us, 24962.8 MB/s OK
Compression level: 2
comp(write): 398.8 us, 10030.0 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 162.2 us, 24665.4 MB/s OK
Compression level: 3
comp(write): 398.9 us, 10026.7 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 162.8 us, 24571.8 MB/s OK
Compression level: 4
comp(write): 355.9 us, 11240.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 162.9 us, 24547.6 MB/s OK
Compression level: 5
comp(write): 354.1 us, 11297.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 162.7 us, 24584.4 MB/s OK
Compression level: 6
comp(write): 353.9 us, 11301.7 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 164.0 us, 24397.1 MB/s OK
Compression level: 7
comp(write): 353.4 us, 11317.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 165.1 us, 24225.9 MB/s OK
Compression level: 8
comp(write): 354.4 us, 11287.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 165.0 us, 24243.6 MB/s OK
Compression level: 9
comp(write): 377.0 us, 10610.2 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 191.8 us, 20859.0 MB/s OK
--> 7, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 7
********************** Running benchmarks *********************
memcpy(write): 416.5 us, 9603.7 MB/s
memcpy(read): 436.4 us, 9166.5 MB/s
Compression level: 0
comp(write): 113.6 us, 35224.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 94.2 us, 42474.0 MB/s OK
Compression level: 1
comp(write): 346.7 us, 11537.1 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 142.9 us, 27988.2 MB/s OK
Compression level: 2
comp(write): 360.2 us, 11103.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 149.4 us, 26770.6 MB/s OK
Compression level: 3
comp(write): 360.3 us, 11102.3 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 154.8 us, 25846.1 MB/s OK
Compression level: 4
comp(write): 353.5 us, 11314.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 167.1 us, 23932.2 MB/s OK
Compression level: 5
comp(write): 354.0 us, 11300.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 166.4 us, 24034.9 MB/s OK
Compression level: 6
comp(write): 353.5 us, 11314.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 167.8 us, 23832.8 MB/s OK
Compression level: 7
comp(write): 352.9 us, 11333.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 166.9 us, 23970.4 MB/s OK
Compression level: 8
comp(write): 353.7 us, 11307.7 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 167.7 us, 23855.4 MB/s OK
Compression level: 9
comp(write): 377.6 us, 10592.1 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 192.2 us, 20807.1 MB/s OK
--> 8, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 8
********************** Running benchmarks *********************
memcpy(write): 414.9 us, 9641.2 MB/s
memcpy(read): 446.3 us, 8962.2 MB/s
Compression level: 0
comp(write): 116.4 us, 34375.6 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 94.4 us, 42371.3 MB/s OK
Compression level: 1
comp(write): 303.7 us, 13172.7 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 127.6 us, 31337.5 MB/s OK
Compression level: 2
comp(write): 303.3 us, 13187.4 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 128.1 us, 31223.2 MB/s OK
Compression level: 3
comp(write): 301.3 us, 13277.9 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 126.1 us, 31728.6 MB/s OK
Compression level: 4
comp(write): 193.1 us, 20712.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 96.7 us, 41349.3 MB/s OK
Compression level: 5
comp(write): 194.4 us, 20577.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 95.3 us, 41961.1 MB/s OK
Compression level: 6
comp(write): 192.8 us, 20751.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 96.2 us, 41570.4 MB/s OK
Compression level: 7
comp(write): 193.5 us, 20667.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 95.4 us, 41945.2 MB/s OK
Compression level: 8
comp(write): 194.0 us, 20618.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 95.4 us, 41912.0 MB/s OK
Compression level: 9
comp(write): 383.2 us, 10437.2 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 196.8 us, 20322.1 MB/s OK
--> 9, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 9
********************** Running benchmarks *********************
memcpy(write): 428.4 us, 9336.2 MB/s
memcpy(read): 461.6 us, 8664.9 MB/s
Compression level: 0
comp(write): 115.9 us, 34510.4 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 95.6 us, 41852.2 MB/s OK
Compression level: 1
comp(write): 279.9 us, 14291.8 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 121.8 us, 32853.9 MB/s OK
Compression level: 2
comp(write): 293.7 us, 13618.5 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 130.9 us, 30558.0 MB/s OK
Compression level: 3
comp(write): 293.2 us, 13643.8 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 131.1 us, 30513.3 MB/s OK
Compression level: 4
comp(write): 194.7 us, 20539.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 98.2 us, 40740.5 MB/s OK
Compression level: 5
comp(write): 193.5 us, 20675.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 96.8 us, 41333.4 MB/s OK
Compression level: 6
comp(write): 195.3 us, 20483.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 97.2 us, 41172.1 MB/s OK
Compression level: 7
comp(write): 192.8 us, 20750.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 101.7 us, 39340.4 MB/s OK
Compression level: 8
comp(write): 194.8 us, 20537.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 97.0 us, 41219.5 MB/s OK
Compression level: 9
comp(write): 394.0 us, 10153.0 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 198.3 us, 20174.1 MB/s OK
--> 10, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 10
********************** Running benchmarks *********************
memcpy(write): 405.7 us, 9860.6 MB/s
memcpy(read): 429.4 us, 9315.7 MB/s
Compression level: 0
comp(write): 122.6 us, 32636.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 97.7 us, 40948.4 MB/s OK
Compression level: 1
comp(write): 252.3 us, 15854.9 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 112.4 us, 35595.4 MB/s OK
Compression level: 2
comp(write): 264.1 us, 15143.3 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 118.6 us, 33735.1 MB/s OK
Compression level: 3
comp(write): 266.5 us, 15006.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 120.3 us, 33258.5 MB/s OK
Compression level: 4
comp(write): 199.6 us, 20042.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 99.0 us, 40391.1 MB/s OK
Compression level: 5
comp(write): 196.8 us, 20330.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 99.6 us, 40150.9 MB/s OK
Compression level: 6
comp(write): 194.6 us, 20554.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 104.7 us, 38205.4 MB/s OK
Compression level: 7
comp(write): 195.3 us, 20483.2 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 98.7 us, 40536.8 MB/s OK
Compression level: 8
comp(write): 199.1 us, 20087.2 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 98.4 us, 40647.3 MB/s OK
Compression level: 9
comp(write): 405.3 us, 9868.3 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 195.9 us, 20415.6 MB/s OK
--> 11, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 11
********************** Running benchmarks *********************
memcpy(write): 428.5 us, 9335.9 MB/s
memcpy(read): 459.9 us, 8697.9 MB/s
Compression level: 0
comp(write): 137.1 us, 29175.5 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 97.3 us, 41129.3 MB/s OK
Compression level: 1
comp(write): 237.8 us, 16823.4 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 109.6 us, 36484.1 MB/s OK
Compression level: 2
comp(write): 240.6 us, 16623.2 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 106.2 us, 37658.0 MB/s OK
Compression level: 3
comp(write): 237.0 us, 16880.2 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 109.2 us, 36627.0 MB/s OK
Compression level: 4
comp(write): 199.6 us, 20040.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 100.3 us, 39889.5 MB/s OK
Compression level: 5
comp(write): 198.8 us, 20122.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 100.8 us, 39695.1 MB/s OK
Compression level: 6
comp(write): 198.6 us, 20141.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 100.0 us, 39990.1 MB/s OK
Compression level: 7
comp(write): 201.5 us, 19855.9 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 100.4 us, 39822.9 MB/s OK
Compression level: 8
comp(write): 198.7 us, 20129.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 100.3 us, 39880.3 MB/s OK
Compression level: 9
comp(write): 414.6 us, 9648.7 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 201.5 us, 19850.8 MB/s OK
--> 12, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 12
********************** Running benchmarks *********************
memcpy(write): 386.7 us, 10344.6 MB/s
memcpy(read): 447.9 us, 8930.5 MB/s
Compression level: 0
comp(write): 136.4 us, 29320.8 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 101.4 us, 39440.2 MB/s OK
Compression level: 1
comp(write): 230.2 us, 17378.1 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 106.8 us, 37463.8 MB/s OK
Compression level: 2
comp(write): 243.3 us, 16443.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 109.7 us, 36469.6 MB/s OK
Compression level: 3
comp(write): 244.3 us, 16374.5 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 112.4 us, 35581.9 MB/s OK
Compression level: 4
comp(write): 209.4 us, 19106.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 107.2 us, 37302.0 MB/s OK
Compression level: 5
comp(write): 209.0 us, 19142.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 107.9 us, 37084.2 MB/s OK
Compression level: 6
comp(write): 208.8 us, 19153.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 107.4 us, 37229.6 MB/s OK
Compression level: 7
comp(write): 210.7 us, 18988.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 107.1 us, 37342.2 MB/s OK
Compression level: 8
comp(write): 208.5 us, 19183.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 107.3 us, 37287.1 MB/s OK
Compression level: 9
comp(write): 431.1 us, 9279.1 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 205.1 us, 19505.1 MB/s OK
--> 13, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 13
********************** Running benchmarks *********************
memcpy(write): 416.5 us, 9602.8 MB/s
memcpy(read): 441.6 us, 9058.5 MB/s
Compression level: 0
comp(write): 127.2 us, 31453.3 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 100.3 us, 39869.9 MB/s OK
Compression level: 1
comp(write): 223.6 us, 17891.5 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 102.9 us, 38884.2 MB/s OK
Compression level: 2
comp(write): 224.9 us, 17789.0 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 119.4 us, 33503.4 MB/s OK
Compression level: 3
comp(write): 214.0 us, 18692.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 100.8 us, 39663.7 MB/s OK
Compression level: 4
comp(write): 213.8 us, 18709.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 109.6 us, 36510.1 MB/s OK
Compression level: 5
comp(write): 211.8 us, 18882.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 110.2 us, 36289.9 MB/s OK
Compression level: 6
comp(write): 211.9 us, 18879.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 110.5 us, 36184.7 MB/s OK
Compression level: 7
comp(write): 212.8 us, 18797.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 109.9 us, 36386.6 MB/s OK
Compression level: 8
comp(write): 213.8 us, 18709.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 109.4 us, 36563.6 MB/s OK
Compression level: 9
comp(write): 436.8 us, 9157.3 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 205.8 us, 19435.3 MB/s OK
--> 14, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 14
********************** Running benchmarks *********************
memcpy(write): 394.2 us, 10145.9 MB/s
memcpy(read): 430.6 us, 9290.1 MB/s
Compression level: 0
comp(write): 141.7 us, 28224.5 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 101.3 us, 39497.1 MB/s OK
Compression level: 1
comp(write): 213.3 us, 18754.4 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 108.6 us, 36840.4 MB/s OK
Compression level: 2
comp(write): 213.4 us, 18747.0 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 103.9 us, 38513.2 MB/s OK
Compression level: 3
comp(write): 215.9 us, 18524.1 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 104.1 us, 38428.4 MB/s OK
Compression level: 4
comp(write): 223.3 us, 17916.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 112.3 us, 35612.6 MB/s OK
Compression level: 5
comp(write): 217.1 us, 18428.5 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 111.7 us, 35799.9 MB/s OK
Compression level: 6
comp(write): 211.7 us, 18897.0 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 112.2 us, 35647.3 MB/s OK
Compression level: 7
comp(write): 216.8 us, 18450.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 111.2 us, 35971.7 MB/s OK
Compression level: 8
comp(write): 213.2 us, 18765.6 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 112.4 us, 35578.3 MB/s OK
Compression level: 9
comp(write): 436.2 us, 9169.2 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 205.6 us, 19457.9 MB/s OK
--> 15, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 15
********************** Running benchmarks *********************
memcpy(write): 412.6 us, 9695.7 MB/s
memcpy(read): 460.7 us, 8682.0 MB/s
Compression level: 0
comp(write): 139.7 us, 28629.5 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 114.5 us, 34922.7 MB/s OK
Compression level: 1
comp(write): 212.9 us, 18788.3 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 145.3 us, 27527.1 MB/s OK
Compression level: 2
comp(write): 213.9 us, 18703.5 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 123.6 us, 32371.0 MB/s OK
Compression level: 3
comp(write): 214.9 us, 18611.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 114.2 us, 35041.1 MB/s OK
Compression level: 4
comp(write): 231.4 us, 17282.7 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 116.2 us, 34425.6 MB/s OK
Compression level: 5
comp(write): 231.2 us, 17301.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 133.4 us, 29995.7 MB/s OK
Compression level: 6
comp(write): 228.2 us, 17531.3 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 121.3 us, 32978.0 MB/s OK
Compression level: 7
comp(write): 231.6 us, 17274.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 128.2 us, 31211.2 MB/s OK
Compression level: 8
comp(write): 239.0 us, 16734.2 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 127.7 us, 31320.3 MB/s OK
Compression level: 9
comp(write): 451.1 us, 8866.9 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 307.5 us, 13009.8 MB/s OK
--> 16, 4194304, 4, 19, lz4, shuffle
********************** Run info ******************************
Blosc version: 2.0.0.beta.6.dev ($Date:: 2020-04-21 #$)
Using synthetic data with 19 significant bits (out of 32)
Dataset size: 4194304 bytes Type size: 4 bytes
Working set: 256.0 MB Number of threads: 16
********************** Running benchmarks *********************
memcpy(write): 407.0 us, 9828.8 MB/s
memcpy(read): 429.6 us, 9311.2 MB/s
Compression level: 0
comp(write): 136.2 us, 29374.6 MB/s Final bytes: 4194336 Ratio: 1.00
decomp(read): 112.9 us, 35428.1 MB/s OK
Compression level: 1
comp(write): 202.3 us, 19772.6 MB/s Final bytes: 882352 Ratio: 4.75
decomp(read): 140.4 us, 28495.0 MB/s OK
Compression level: 2
comp(write): 210.7 us, 18980.6 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 143.2 us, 27928.5 MB/s OK
Compression level: 3
comp(write): 215.0 us, 18602.8 MB/s Final bytes: 798592 Ratio: 5.25
decomp(read): 143.8 us, 27809.7 MB/s OK
Compression level: 4
comp(write): 241.5 us, 16563.8 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 116.4 us, 34369.9 MB/s OK
Compression level: 5
comp(write): 241.5 us, 16565.7 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 116.1 us, 34447.2 MB/s OK
Compression level: 6
comp(write): 234.3 us, 17069.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 117.1 us, 34151.1 MB/s OK
Compression level: 7
comp(write): 246.7 us, 16216.4 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 119.5 us, 33468.5 MB/s OK
Compression level: 8
comp(write): 249.9 us, 16005.1 MB/s Final bytes: 334136 Ratio: 12.55
decomp(read): 116.8 us, 34238.3 MB/s OK
Compression level: 9
comp(write): 445.7 us, 8974.0 MB/s Final bytes: 316064 Ratio: 13.27
decomp(read): 218.9 us, 18275.0 MB/s OK
Round-trip compr/decompr on 120.0 GB
Elapsed time: 20.4 s, 13264.9 MB/s
Blosc-c-blosc2-d3bee7c/bench/results-corex/lz4hc-cl1-sum_openmp-corex.out 0000664 0000000 0000000 00000026760 15255167342 0026407 0 ustar 00root root 0000000 0000000 faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0322 s, 11844.8 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.65 s, 231.8 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.0625 s, 6107.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11800.8 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.65 s, 231.8 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.0624 s, 6109.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0177 s, 21602.1 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.0343 s, 11114.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0177 s, 21600.5 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.0344 s, 11081.9 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00957 s, 39848.6 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.3 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0179 s, 21368.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00956 s, 39921.2 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.2 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0179 s, 21352.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248988
Sum time for uncompressed data: 0.00676 s, 56414.8 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0091 s, 41932.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.0069 s, 55266.6 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.2 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.00916 s, 41632.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00666 s, 57268.5 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00642 s, 59449.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00676 s, 56467.4 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00656 s, 58192.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290212
Sum time for uncompressed data: 0.00662 s, 57588.5 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 231.9 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00571 s, 66780.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290216
Sum time for uncompressed data: 0.00657 s, 58060.3 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00573 s, 66628.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295172
Sum time for uncompressed data: 0.00776 s, 49178.0 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.3 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00742 s, 51385.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.00779 s, 48998.3 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.2 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.00757 s, 50380.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00688 s, 55486.0 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.3 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.00637 s, 59930.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00691 s, 55199.4 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.65 s, 231.8 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.00623 s, 61220.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310868
Sum time for uncompressed data: 0.00691 s, 55174.0 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.65 s, 231.8 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.00551 s, 69262.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310864
Sum time for uncompressed data: 0.00685 s, 55664.6 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.65 s, 231.8 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.00556 s, 68665.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311968
Sum time for uncompressed data: 0.00671 s, 56871.7 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.65 s, 231.6 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.00484 s, 78759.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311960
Sum time for uncompressed data: 0.00679 s, 56205.3 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.1 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.00463 s, 82380.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311968
Sum time for uncompressed data: 0.00671 s, 56841.4 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.2 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.00564 s, 67624.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00815 s, 46788.0 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.2 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0102 s, 37305.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=lz4hc ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00817 s, 46708.0 MB/s
Compression ratio: 381.5 MB -> 75.8 MB (5.0x)
Compression time: 1.64 s, 232.0 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0126 s, 30329.6 MB/s
Blosc-c-blosc2-d3bee7c/bench/results-corex/zlib-cl1-sum_openmp-corex.out 0000664 0000000 0000000 00000025711 15255167342 0026316 0 ustar 00root root 0000000 0000000 faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11817.7 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 366.5 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.456 s, 836.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0322 s, 11829.1 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 366.3 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.458 s, 833.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0176 s, 21670.8 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.2 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.248 s, 1540.9 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0177 s, 21593.3 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.7 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.248 s, 1539.9 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00964 s, 39585.1 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 366.0 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.127 s, 3002.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00959 s, 39782.9 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.1 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.127 s, 3003.6 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.00676 s, 56406.0 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.2 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0638 s, 5978.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.00664 s, 57491.2 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.1 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0641 s, 5954.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00661 s, 57684.3 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.3 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0437 s, 8726.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00665 s, 57387.5 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 363.9 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0437 s, 8733.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290212
Sum time for uncompressed data: 0.0067 s, 56901.7 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 363.5 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.039 s, 9789.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290212
Sum time for uncompressed data: 0.00668 s, 57103.1 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.8 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.039 s, 9772.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.00775 s, 49219.3 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.3 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0496 s, 7690.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.00771 s, 49485.3 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.5 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0487 s, 7838.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00688 s, 55470.9 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.5 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.0395 s, 9649.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305116
Sum time for uncompressed data: 0.00684 s, 55748.9 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.6 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.0403 s, 9466.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310872
Sum time for uncompressed data: 0.00687 s, 55562.1 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.2 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.0336 s, 11357.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310868
Sum time for uncompressed data: 0.00686 s, 55588.8 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.06 s, 360.9 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.0337 s, 11325.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311968
Sum time for uncompressed data: 0.00677 s, 56360.0 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.3 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.0291 s, 13109.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311964
Sum time for uncompressed data: 0.00677 s, 56349.4 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.6 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.0291 s, 13091.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00812 s, 46977.8 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.04 s, 365.7 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0645 s, 5913.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=zlib ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00813 s, 46931.5 MB/s
Compression ratio: 381.5 MB -> 80.1 MB (4.8x)
Compression time: 1.05 s, 364.7 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0538 s, 7092.2 MB/s
Blosc-c-blosc2-d3bee7c/bench/results-corex/zstd-cl1-sum_openmp-corex.out 0000664 0000000 0000000 00000025760 15255167342 0026346 0 ustar 00root root 0000000 0000000 faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11818.2 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.671 s, 568.2 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.241 s, 1582.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=1 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45319996
Sum time for uncompressed data: 0.0323 s, 11814.4 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.671 s, 568.5 MB/s
Sum for *compressed* data: 46323164
Sum time for *compressed* data: 0.239 s, 1593.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0176 s, 21651.6 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.672 s, 567.6 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.13 s, 2926.1 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=2 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 45826496
Sum time for uncompressed data: 0.0177 s, 21586.1 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 566.7 MB/s
Sum for *compressed* data: 46321328
Sum time for *compressed* data: 0.129 s, 2955.5 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00974 s, 39182.0 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.672 s, 567.3 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0673 s, 5669.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=4 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46118560
Sum time for uncompressed data: 0.00958 s, 39816.2 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 566.7 MB/s
Sum for *compressed* data: 46320460
Sum time for *compressed* data: 0.0673 s, 5664.8 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248988
Sum time for uncompressed data: 0.00665 s, 57392.3 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.674 s, 565.8 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0341 s, 11201.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=8 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46248984
Sum time for uncompressed data: 0.00669 s, 57026.4 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.674 s, 566.0 MB/s
Sum for *compressed* data: 46320768
Sum time for *compressed* data: 0.0339 s, 11244.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00665 s, 57329.8 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 567.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0233 s, 16363.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=12 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46285244
Sum time for uncompressed data: 0.00659 s, 57899.4 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.671 s, 568.5 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0234 s, 16315.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290216
Sum time for uncompressed data: 0.00667 s, 57162.5 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.672 s, 567.8 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0208 s, 18306.0 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=14 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46290212
Sum time for uncompressed data: 0.00668 s, 57129.5 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.675 s, 564.7 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0208 s, 18377.6 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295180
Sum time for uncompressed data: 0.00767 s, 49711.6 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.674 s, 566.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0298 s, 12814.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=16 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46295176
Sum time for uncompressed data: 0.00768 s, 49680.6 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.674 s, 566.0 MB/s
Sum for *compressed* data: 46320348
Sum time for *compressed* data: 0.0297 s, 12865.2 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00691 s, 55204.8 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.674 s, 566.1 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.0241 s, 15817.6 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=20 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46305112
Sum time for uncompressed data: 0.00687 s, 55565.0 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 566.7 MB/s
Sum for *compressed* data: 46320364
Sum time for *compressed* data: 0.0241 s, 15830.7 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310864
Sum time for uncompressed data: 0.0069 s, 55261.4 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.675 s, 565.4 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.0205 s, 18641.4 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=24 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46310868
Sum time for uncompressed data: 0.00691 s, 55239.8 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 566.5 MB/s
Sum for *compressed* data: 46320384
Sum time for *compressed* data: 0.0204 s, 18736.9 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311968
Sum time for uncompressed data: 0.00679 s, 56218.0 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 566.5 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.0178 s, 21474.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=28 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46311964
Sum time for uncompressed data: 0.00674 s, 56574.7 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.672 s, 567.9 MB/s
Sum for *compressed* data: 46320396
Sum time for *compressed* data: 0.0181 s, 21132.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00812 s, 46950.8 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.673 s, 567.0 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0364 s, 10466.3 MB/s
faltet@corex14 ~/b/c/bench (main)> env LD_LIBRARY_PATH=../build/blosc OMP_PROC_BIND=spread OMP_NUM_THREADS=32 SUM_CLEVEL=1 SUM_COMPRESSOR=zstd ./sum_openmp (base)
Blosc version info: 2.0.0.rc.2.dev ($Date:: 2021-05-06 #$)
Sum for uncompressed data: 46313056
Sum time for uncompressed data: 0.00809 s, 47138.4 MB/s
Compression ratio: 381.5 MB -> 64.3 MB (5.9x)
Compression time: 0.674 s, 565.6 MB/s
Sum for *compressed* data: 46320392
Sum time for *compressed* data: 0.0366 s, 10420.4 MB/s
Blosc-c-blosc2-d3bee7c/bench/sframe_bench.c 0000664 0000000 0000000 00000031656 15255167342 0020620 0 ustar 00root root 0000000 0000000 /*********************************************************************
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark for testing sframe vs frame.
For usage instructions of this benchmark, please see:
https://www.blosc.org/pages/synthetic-benchmarks/
We are collecting speeds for different machines, so the output of your
benchmarks and your processor specifications are welcome!
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
#include "blosc2.h"
#if defined(_WIN32)
#include
#endif
#if defined(_WIN32) && !defined(__MINGW32__)
#include
#else
#include
#endif
#include
#include
#include
#include
#include
#include
#define KB 1024
#define MB (1024*KB)
#define GB (1024*MB)
#define NCHUNKS 1000 /* number of chunks */
#define CHUNKSIZE (2000 * 1000)
int nchunks = NCHUNKS;
int iterations = 5;
int io_type = BLOSC2_IO_FILESYSTEM;
void test_update(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) {
blosc_timestamp_t last, current;
double cframe_update_time, sframe_update_time;
size_t isize = CHUNKSIZE * sizeof(int32_t);
int32_t* data = malloc(isize);
// Random update list
int64_t* update_chunks = malloc(sizeof(int64_t) * iterations);
srand(time(NULL));
for (int i = 0; i < iterations; i++) {
update_chunks[i] = rand() % schunk_sframe->nchunks;
}
printf("*******************************************************\n");
printf("******************* Updating %d chunks ******************\n", iterations);
printf("*******************************************************\n");
// Update the sframe chunks
sframe_update_time = 0.0;
cframe_update_time = 0.0;
int32_t datasize = sizeof(int32_t) * CHUNKSIZE;
int32_t chunksize = sizeof(int32_t) * CHUNKSIZE + BLOSC2_MAX_OVERHEAD;
uint8_t* chunk;
int csize;
int64_t _nchunks;
for (int i = 0; i < iterations; i++) {
// Generate data
for (int j = 0; j < CHUNKSIZE; j++) {
data[j] = i * CHUNKSIZE;
}
chunk = malloc(chunksize);
csize = blosc2_compress_ctx(schunk_sframe->cctx, data, datasize, chunk, chunksize);
if (csize < 0) {
printf("ERROR: chunk cannot be compressed\n");
}
// Sframe
blosc_set_timestamp(¤t);
_nchunks = blosc2_schunk_update_chunk(schunk_sframe, update_chunks[i], chunk, true);
blosc_set_timestamp(&last);
if (_nchunks < 0){
printf("ERROR: chunk cannot be updated correctly\n");
}
sframe_update_time += blosc_elapsed_secs(current, last);
// Frame
blosc_set_timestamp(¤t);
_nchunks = blosc2_schunk_update_chunk(schunk_cframe, update_chunks[i], chunk, true);
blosc_set_timestamp(&last);
free(chunk);
if (_nchunks <= 0){
printf("ERROR: chunk cannot be updated correctly\n");
}
cframe_update_time += blosc_elapsed_secs(current, last);
}
printf("[Sframe Update] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n",
sframe_update_time, schunk_sframe->cbytes);
printf("[Cframe Update] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n",
cframe_update_time, schunk_cframe->cbytes);
/* Free resources */
free(update_chunks);
free(data);
}
void test_insert(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) {
blosc_timestamp_t last, current;
double cframe_insert_time, sframe_insert_time;
size_t isize = CHUNKSIZE * sizeof(int32_t);
int32_t* data = malloc(isize);
// Random insert list
int64_t* insert_chunks = malloc(sizeof(int64_t) * iterations);
srand(time(NULL));
for (int i = 0; i < iterations; i++) {
insert_chunks[i] = rand() % schunk_sframe->nchunks;
}
printf("*******************************************************\n");
printf("****************** Inserting %d chunks *****************\n", iterations);
printf("*******************************************************\n");
// Update the sframe chunks
sframe_insert_time = 0.0;
cframe_insert_time = 0.0;
int32_t datasize = sizeof(int32_t) * CHUNKSIZE;
int32_t chunksize = sizeof(int32_t) * CHUNKSIZE + BLOSC2_MAX_OVERHEAD;
uint8_t* chunk;
int csize;
int64_t _nchunks;
for (int i = 0; i < iterations; i++) {
// Generate data
for (int j = 0; j < CHUNKSIZE; j++) {
data[j] = j + i * CHUNKSIZE;
}
chunk = malloc(chunksize);
csize = blosc2_compress_ctx(schunk_sframe->cctx, data, datasize, chunk, chunksize);
if (csize < 0) {
printf("ERROR: chunk cannot be compressed\n");
}
// Sframe
blosc_set_timestamp(¤t);
_nchunks = blosc2_schunk_insert_chunk(schunk_sframe, insert_chunks[i], chunk, true);
blosc_set_timestamp(&last);
if (_nchunks < 0){
printf("ERROR: chunk cannot be updated correctly\n");
}
sframe_insert_time += blosc_elapsed_secs(current, last);
// Frame
blosc_set_timestamp(¤t);
_nchunks = blosc2_schunk_update_chunk(schunk_cframe, insert_chunks[i], chunk, true);
blosc_set_timestamp(&last);
free(chunk);
if (_nchunks <= 0){
printf("ERROR: chunk cannot be updated correctly\n");
}
cframe_insert_time += blosc_elapsed_secs(current, last);
}
printf("[Sframe Insert] Elapsed time:\t %6.3f s. Total sframe size: %.3" PRId64 " bytes\n",
sframe_insert_time, schunk_sframe->cbytes);
printf("[Cframe Insert] Elapsed time:\t %6.3f s. Total cframe size: %.3" PRId64 " bytes\n",
cframe_insert_time, schunk_cframe->cbytes);
/* Free resources */
free(insert_chunks);
free(data);
}
void test_reorder(blosc2_schunk* schunk_sframe, blosc2_schunk* schunk_cframe) {
blosc_timestamp_t last, current;
double cframe_reorder_time, sframe_reorder_time;
// Reorder list
int64_t *offsets_order = malloc(sizeof(int64_t) * schunk_sframe->nchunks);
for (int i = 0; i < schunk_sframe->nchunks; ++i) {
offsets_order[i] = (i + 3) % schunk_sframe->nchunks;
}
printf("*******************************************************\n");
printf("****************** Reordering chunks ******************\n");
printf("*******************************************************\n");
// Reorder sframe
blosc_set_timestamp(¤t);
int err = blosc2_schunk_reorder_offsets(schunk_sframe, offsets_order);
blosc_set_timestamp(&last);
if (err < 0) {
printf("ERROR: cannot reorder chunks\n");
}
sframe_reorder_time = blosc_elapsed_secs(current, last);
// Reorder frame
blosc_set_timestamp(¤t);
err = blosc2_schunk_reorder_offsets(schunk_cframe, offsets_order);
blosc_set_timestamp(&last);
if (err < 0) {
printf("ERROR: cannot reorder chunks\n");
}
cframe_reorder_time = blosc_elapsed_secs(current, last);
printf("[Sframe Update] Elapsed time:\t %f s. Total sframe size: %.3" PRId64 " bytes\n",
sframe_reorder_time, schunk_sframe->cbytes);
printf("[Cframe Update] Elapsed time:\t %f s. Total cframe size: %.3" PRId64 " bytes\n",
cframe_reorder_time, schunk_cframe->cbytes);
/* Free resources */
free(offsets_order);
}
void test_create_sframe_frame(char* operation) {
blosc_timestamp_t last, current;
double cframe_append_time, sframe_append_time, cframe_decompress_time, sframe_decompress_time;
int64_t nbytes, cbytes;
int32_t isize = CHUNKSIZE * sizeof(int32_t);
int dsize;
float totalsize = (float)isize * nchunks;
int32_t* data = malloc(isize);
int32_t* data_dest = malloc(isize);
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
blosc2_schunk* schunk_sframe;
blosc2_schunk* schunk_cframe;
/* Initialize the Blosc compressor */
blosc2_init();
printf("*******************************************************\n");
printf("***** Creating the frame and sframe with %d chunks ****\n", nchunks);
printf("*******************************************************\n");
/* Create a frame container */
cparams.typesize = sizeof(int32_t);
cparams.nthreads = 2;
dparams.nthreads = 2;
blosc2_storage storage = {.contiguous=false, .urlpath="dir.b2frame", .cparams=&cparams, .dparams=&dparams};
blosc2_remove_urlpath(storage.urlpath);
schunk_sframe = blosc2_schunk_new(&storage);
blosc2_stdio_mmap mmap_file = BLOSC2_STDIO_MMAP_DEFAULTS;
mmap_file.mode = "w+";
blosc2_io io_mmap = {.id = BLOSC2_IO_FILESYSTEM_MMAP, .name = "filesystem_mmap", .params = &mmap_file};
blosc2_storage storage2 = {.contiguous=true, .urlpath="test_cframe.b2frame", .cparams=&cparams, .dparams=&dparams};
if (io_type == BLOSC2_IO_FILESYSTEM) {
storage2.io = (blosc2_io*)&BLOSC2_IO_DEFAULTS;
}
else if (io_type == BLOSC2_IO_FILESYSTEM_MMAP) {
storage2.io = &io_mmap;
}
blosc2_remove_urlpath(storage2.urlpath);
schunk_cframe = blosc2_schunk_new(&storage2);
printf("Test comparison frame vs sframe with %d chunks.\n", nchunks);
// Feed it with data
sframe_append_time=0.0;
cframe_append_time=0.0;
blosc_set_timestamp(¤t);
for (int nchunk = 0; nchunk < nchunks; nchunk++) {
for (int i = 0; i < CHUNKSIZE; i++) {
data[i] = i + nchunk * CHUNKSIZE;
}
blosc_set_timestamp(¤t);
blosc2_schunk_append_buffer(schunk_sframe, data, isize);
blosc_set_timestamp(&last);
sframe_append_time += blosc_elapsed_secs(current, last);
blosc_set_timestamp(¤t);
blosc2_schunk_append_buffer(schunk_cframe, data, isize);
blosc_set_timestamp(&last);
cframe_append_time += blosc_elapsed_secs(current, last);
}
printf("[Sframe Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
sframe_append_time, totalsize / GB, totalsize / (GB * sframe_append_time));
printf("[Cframe Compr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
cframe_append_time, totalsize / GB, totalsize / (GB * cframe_append_time));
/* Gather some info */
nbytes = schunk_sframe->nbytes;
cbytes = schunk_sframe->cbytes;
printf("Compression super-chunk-sframe: %ld -> %ld (%.1fx)\n",
(long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes);
nbytes = schunk_cframe->nbytes;
cbytes = schunk_cframe->cbytes;
printf("Compression super-chunk-cframe: %ld -> %ld (%.1fx)\n",
(long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes);
// Decompress the data
sframe_decompress_time = 0;
cframe_decompress_time = 0;
for (int nchunk = 0; nchunk < nchunks; nchunk++) {
// Sframe
blosc_set_timestamp(¤t);
dsize = blosc2_schunk_decompress_chunk(schunk_sframe, nchunk, (void *) data_dest, isize);
blosc_set_timestamp(&last);
if (dsize < 0) {
printf("Decompression error sframe. Error code: %d\n", dsize);
}
assert (dsize == (int)isize);
sframe_decompress_time += blosc_elapsed_secs(current, last);
// Frame
blosc_set_timestamp(¤t);
dsize = blosc2_schunk_decompress_chunk(schunk_cframe, nchunk, (void *) data_dest, isize);
blosc_set_timestamp(&last);
if (dsize < 0) {
printf("Decompression error cframe. Error code: %d\n", dsize);
}
assert (dsize == (int)isize);
cframe_decompress_time += blosc_elapsed_secs(current, last);
}
printf("[Sframe Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
sframe_decompress_time, totalsize / GB, totalsize / (GB * sframe_decompress_time));
printf("[Cframe Decompr] Elapsed time:\t %6.3f s. Processed data: %.3f GB (%.3f GB/s)\n",
cframe_decompress_time, totalsize / GB, totalsize / (GB * cframe_decompress_time));
printf("Decompression successful!\n");
printf("Successful roundtrip!\n");
/* Free resources */
free(data_dest);
free(data);
if (operation != NULL) {
if (strcmp(operation, "insert") == 0) {
test_insert(schunk_sframe, schunk_cframe);
}
else if (strcmp(operation, "update") == 0) {
test_update(schunk_sframe, schunk_cframe);
}
else if (strcmp(operation, "reorder") == 0) {
test_reorder(schunk_sframe, schunk_cframe);
}
}
blosc2_remove_urlpath(schunk_sframe->storage->urlpath);
blosc2_remove_urlpath(schunk_cframe->storage->urlpath);
blosc2_schunk_free(schunk_sframe);
blosc2_schunk_free(schunk_cframe);
/* Destroy the Blosc environment */
blosc2_destroy();
}
int main(int argc, char* argv[]) {
char* operation = NULL;
if (argc >= 6) {
printf("Usage: ./sframe_bench [nchunks] [insert | update | reorder] [num operations] [io_file | io_mmap]\n");
exit(1);
}
else if (argc >= 2) {
nchunks = (int)strtol(argv[1], NULL, 10);
}
if (argc >= 3) {
operation = argv[2];
}
if (argc >= 4) {
iterations = (int)strtol(argv[3], NULL, 10);
}
if (argc == 5) {
if (strcmp(argv[4], "io_file") == 0) {
io_type = BLOSC2_IO_FILESYSTEM;
} else if (strcmp(argv[4], "io_mmap") == 0) {
io_type = BLOSC2_IO_FILESYSTEM_MMAP;
} else {
printf("Invalid io type. Use io_file or io_mmap\n");
exit(1);
}
}
test_create_sframe_frame(operation);
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/sum_openmp.c 0000664 0000000 0000000 00000016311 15255167342 0020355 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Example program showing how to operate with compressed buffers.
To compile this program for synthetic data (default):
$ gcc -fopenmp -O3 sum_openmp.c -o sum_openmp -lblosc2
To run:
$ OMP_PROC_BIND=spread OMP_NUM_THREADS=8 ./sum_openmp
Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$)
Sum for uncompressed data: 199950000000
Sum time for uncompressed data: 0.0288 s, 26459.3 MB/s
Compression ratio: 762.9 MB -> 14.0 MB (54.6x)
Compression time: 0.288 s, 2653.5 MB/s
Sum for *compressed* data: 199950000000
Sum time for *compressed* data: 0.0188 s, 40653.7 MB/s
To use real (rainfall) data:
$ gcc -DRAINFALL -fopenmp -Ofast sum_openmp.c -o sum_openmp
And running it:
$ OMP_PROC_BIND=spread OMP_NUM_THREADS=8 ./sum_openmp
Blosc version info: 2.0.0a6.dev ($Date:: 2018-05-18 #$)
Sum for uncompressed data: 29741012
Sum time for uncompressed data: 0.0149 s, 25627.4 MB/s
Compression ratio: 381.5 MB -> 71.3 MB (5.3x)
Compression time: 1.53 s, 249.1 MB/s
Sum for *compressed* data: 29741012
Sum time for *compressed* data: 0.0247 s, 15467.5 MB/s
*/
#include
#include
#include
#include
#include
#include
#include "blosc2.h"
#define KB 1024.
#define MB (1024*KB)
#define GB (1024*MB)
#define N (100 * 1000 * 1000)
#define CHUNKSIZE (16 * 1000)
#define NCHUNKS (N / CHUNKSIZE)
#define NTHREADS 8
#define NITER 5
#ifdef RAINFALL
#define SYNTHETIC false
#else
#define SYNTHETIC true
#endif
#if SYNTHETIC == true
#define DTYPE int64_t
#define CLEVEL 3
#define CODEC BLOSC_BLOSCLZ
#else
#define DTYPE float
#define CLEVEL 1
#define CODEC BLOSC_LZ4
#endif
int main(void) {
blosc2_init();
static DTYPE udata[N];
DTYPE chunk_buf[CHUNKSIZE];
int32_t isize = CHUNKSIZE * sizeof(DTYPE);
DTYPE sum, compressed_sum;
int64_t nbytes, cbytes;
blosc2_schunk* schunk;
int i, j, nchunk;
blosc_timestamp_t last, current;
double ttotal, itotal;
char* envvar = NULL;
printf("Blosc version info: %s (%s)\n",
BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE);
// Fill the buffer for a chunk
if (SYNTHETIC) {
for (j = 0; j < CHUNKSIZE; j++) {
chunk_buf[j] = j;
}
}
else {
struct stat info;
const char *filegrid = "rainfall-grid-150x150.bin";
if (stat(filegrid, &info) != 0) {
printf("Grid file %s not found!", filegrid);
exit(1);
}
char *cdata = malloc(info.st_size);
FILE *f = fopen(filegrid, "rb");
size_t blocks_read = fread(cdata, info.st_size, 1, f);
if (blocks_read != 1) {
printf("Error reading file!\n");
exit(1);
}
fclose(f);
int dsize = blosc1_getitem(cdata, 0, CHUNKSIZE, chunk_buf);
if (dsize < 0) {
printf("blosc1_getitem() error. Error code: %d\n. Probably reading too much data?", dsize);
exit(1);
}
free(cdata);
}
// Fill the uncompressed dataset with data chunks
for (i = 0; i < N / CHUNKSIZE; i++) {
for (j = 0; j < CHUNKSIZE; j++) {
udata[i * CHUNKSIZE + j] = chunk_buf[j];
}
}
// Reduce uncompressed dataset
ttotal = 1e10;
sum = 0;
for (int n = 0; n < NITER; n++) {
sum = 0;
blosc_set_timestamp(&last);
#pragma omp parallel for reduction (+:sum)
for (i = 0; i < N; i++) {
sum += udata[i];
}
blosc_set_timestamp(¤t);
itotal = blosc_elapsed_secs(last, current);
if (itotal < ttotal) ttotal = itotal;
}
printf("Sum for uncompressed data: %10.0f\n", (double)sum);
printf("Sum time for uncompressed data: %.3g s, %.1f MB/s\n",
ttotal, (double)(isize * NCHUNKS) / (double)(ttotal * MB));
// Create a super-chunk container for the compressed container
long codec = CODEC;
envvar = getenv("SUM_COMPRESSOR");
if (envvar != NULL) {
codec = blosc2_compname_to_compcode(envvar);
if (codec < 0) {
printf("Unknown compressor: %s\n", envvar);
return 1;
}
}
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.compcode = (uint8_t)codec;
long clevel = CLEVEL;
envvar = getenv("SUM_CLEVEL");
if (envvar != NULL) {
clevel = strtol(envvar, NULL, 10);
}
cparams.clevel = (uint8_t)clevel;
cparams.typesize = sizeof(DTYPE);
cparams.nthreads = 1;
blosc2_dparams dparams = BLOSC2_DPARAMS_DEFAULTS;
dparams.nthreads = 1;
blosc_set_timestamp(&last);
blosc2_storage storage = {.cparams=&cparams, .dparams=&dparams};
schunk = blosc2_schunk_new(&storage);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
for (i = 0; i < CHUNKSIZE; i++) {
chunk_buf[i] = udata[i + nchunk * CHUNKSIZE];
}
blosc2_schunk_append_buffer(schunk, chunk_buf, isize);
}
blosc_set_timestamp(¤t);
ttotal = blosc_elapsed_secs(last, current);
nbytes = schunk->nbytes;
cbytes = schunk->cbytes;
printf("Compression ratio: %.1f MB -> %.1f MB (%.1fx)\n",
(double)nbytes / MB, (double)cbytes / MB, (1. * (double)nbytes) / (double)cbytes);
printf("Compression time: %.3g s, %.1f MB/s\n",
ttotal, (double)nbytes / (ttotal * MB));
int nthreads = NTHREADS;
envvar = getenv("OMP_NUM_THREADS");
if (envvar != NULL) {
long value;
value = strtol(envvar, NULL, 10);
if ((value != EINVAL) && (value >= 0)) {
nthreads = (int)value;
}
}
// Build buffers and contexts for computations
int nchunks_thread = NCHUNKS / nthreads;
int remaining_chunks = NCHUNKS - nchunks_thread * nthreads;
blosc2_context **dctx = malloc(nthreads * sizeof(void*));
DTYPE** chunk = malloc(nthreads * sizeof(void*));
for (j = 0; j < nthreads; j++) {
chunk[j] = malloc(CHUNKSIZE * sizeof(DTYPE));
}
// Reduce uncompressed dataset
blosc_set_timestamp(&last);
ttotal = 1e10;
compressed_sum = 0;
for (int n = 0; n < NITER; n++) {
compressed_sum = 0;
#pragma omp parallel for private(nchunk) reduction (+:compressed_sum)
for (j = 0; j < nthreads; j++) {
dctx[j] = blosc2_create_dctx(dparams);
for (nchunk = 0; nchunk < nchunks_thread; nchunk++) {
blosc2_decompress_ctx(dctx[j], schunk->data[j * nchunks_thread + nchunk], INT32_MAX,
(void*)(chunk[j]), isize);
for (i = 0; i < CHUNKSIZE; i++) {
compressed_sum += chunk[j][i];
//compressed_sum += i + (j * nchunks_thread + nchunk) * CHUNKSIZE;
}
}
}
for (nchunk = NCHUNKS - remaining_chunks; nchunk < NCHUNKS; nchunk++) {
blosc2_decompress_ctx(dctx[0], schunk->data[nchunk], INT32_MAX, (void*)(chunk[0]), isize);
for (i = 0; i < CHUNKSIZE; i++) {
compressed_sum += chunk[0][i];
//compressed_sum += i + nchunk * CHUNKSIZE;
}
}
blosc_set_timestamp(¤t);
itotal = blosc_elapsed_secs(last, current);
if (itotal < ttotal) ttotal = itotal;
}
printf("Sum for *compressed* data: %10.0f\n", (double)compressed_sum);
printf("Sum time for *compressed* data: %.3g s, %.1f MB/s\n",
ttotal, (double)nbytes / (ttotal * MB));
//printf("sum, csum: %f, %f\n", sum, compressed_sum);
if (SYNTHETIC) {
// difficult to fulfill for single precision
assert(sum == compressed_sum);
}
/* Free resources */
blosc2_schunk_free(schunk);
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/trunc_prec_schunk.c 0000664 0000000 0000000 00000010406 15255167342 0021711 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark showing Blosc TRUNC_PREC filter from C code.
To compile this program:
$ gcc -O3 trunc_prec_schunk.c -o trunc_prec_schunk -lblosc2
*/
#include
#include
#include
#include
#include "blosc2.h"
#define KB 1024
#define MB (1024*KB)
#define GB (1024*MB)
#define NCHUNKS 200
#define CHUNKSIZE (500 * 1000)
#define NTHREADS 4
void fill_buffer(double *buffer, int nchunk) {
double incx = 10. / (NCHUNKS * CHUNKSIZE);
for (int i = 0; i < CHUNKSIZE; i++) {
double x = incx * (nchunk * CHUNKSIZE + i);
buffer[i] = (x - .25) * (x - 4.45) * (x - 8.95);
//buffer[i] = x;
}
}
int main(void) {
blosc2_schunk *schunk;
int32_t isize = CHUNKSIZE * sizeof(double);
int dsize;
int64_t nbytes, cbytes;
int nchunk;
int64_t nchunks = 0;
blosc_timestamp_t last, current;
double totaltime;
float totalsize = (float)(isize * NCHUNKS);
double *data_buffer = malloc(CHUNKSIZE * sizeof(double));
double *rec_buffer = malloc(CHUNKSIZE * sizeof(double));
printf("Blosc version info: %s (%s)\n",
BLOSC2_VERSION_STRING, BLOSC2_VERSION_DATE);
/* Initialize the Blosc compressor */
blosc2_init();
/* Create a super-chunk container */
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.filters[0] = BLOSC_TRUNC_PREC;
cparams.filters_meta[0] = 23; // treat doubles as floats
cparams.typesize = sizeof(double);
// DELTA makes compression ratio quite worse in this case
//cparams.filters[1] = BLOSC_DELTA;
// BLOSC_BITSHUFFLE is not compressing better and it quite slower here
//cparams.filters[BLOSC2_MAX_FILTERS - 1] = BLOSC_BITSHUFFLE;
// Good codec params for this dataset
cparams.compcode = BLOSC_BLOSCLZ;
cparams.clevel = 9;
cparams.nthreads = NTHREADS;
blosc2_storage storage = {.cparams=&cparams, .contiguous=true};
schunk = blosc2_schunk_new(&storage);
/* Append the chunks */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
fill_buffer(data_buffer, nchunk);
nchunks = blosc2_schunk_append_buffer(schunk, data_buffer, isize);
}
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
printf("[Compr] Elapsed time:\t %6.3f s."
" Processed data: %.3f GB (%.3f GB/s)\n",
totaltime, totalsize / GB, totalsize / (GB * totaltime));
/* Gather some info */
nbytes = schunk->nbytes;
cbytes = schunk->cbytes;
printf("Compression super-chunk: %ld -> %ld (%.1fx)\n",
(long)nbytes, (long)cbytes, (1. * (double)nbytes) / (double)cbytes);
/* Retrieve and decompress the chunks */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, rec_buffer, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
assert (dsize == (int)isize);
}
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
totalsize = (float)(isize * nchunks);
printf("[Decompr] Elapsed time:\t %6.3f s."
" Processed data: %.3f GB (%.3f GB/s)\n",
totaltime, totalsize / GB, totalsize / (GB * totaltime));
/* Check that all the values are in the precision range */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, (void *) rec_buffer, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
return dsize;
}
assert (dsize == (int)isize);
fill_buffer(data_buffer, nchunk);
for (int i = 0; i < CHUNKSIZE; i++) {
if (fabs(data_buffer[i] - rec_buffer[i]) > 1e-5) {
printf("Value not in tolerance margin: ");
printf("%g - %g: %g, (nchunk: %d, nelem: %d)\n",
data_buffer[i], rec_buffer[i],
(data_buffer[i] - rec_buffer[i]), nchunk, i);
return -1;
}
}
}
printf("All data did a good roundtrip!\n");
/* Free resources */
free(data_buffer);
free(rec_buffer);
/* Destroy the super-chunk */
blosc2_schunk_free(schunk);
/* Destroy the Blosc environment */
blosc2_destroy();
return 0;
}
Blosc-c-blosc2-d3bee7c/bench/zero_runlen.c 0000664 0000000 0000000 00000017222 15255167342 0020537 0 ustar 00root root 0000000 0000000 /*
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
Benchmark showing Blosc zero detection capabilities via run-length.
*/
#include
#include
#include
#include
#include
#include "blosc2.h"
#define KB 1024
#define MB (1024*KB)
#define GB (1024*MB)
#define NCHUNKS (2000)
#define CHUNKSIZE (500 * 1000) // > NCHUNKS for the bench purposes
#define NTHREADS 8
enum {
ZERO_DETECTION = 0,
CHECK_ZEROS = 1,
CHECK_NANS = 2,
CHECK_VALUES = 3,
CHECK_UNINIT = 4,
};
#define REPEATED_VALUE 1
int check_special_values(int svalue) {
blosc2_schunk *schunk;
int32_t isize = CHUNKSIZE * sizeof(int32_t);
int32_t osize = CHUNKSIZE * sizeof(int32_t) + BLOSC2_MAX_OVERHEAD;
int dsize, csize;
int64_t nbytes, frame_len;
int nchunk;
int64_t nchunks = 0;
int rc;
int32_t value = REPEATED_VALUE;
float fvalue;
blosc_timestamp_t last, current;
double totaltime;
double totalsize = (double)isize * NCHUNKS;
int32_t *data_buffer = malloc(CHUNKSIZE * sizeof(int32_t));
int32_t *rec_buffer = malloc(CHUNKSIZE * sizeof(int32_t));
/* Initialize the Blosc compressor */
blosc2_init();
/* Create a super-chunk container */
blosc2_cparams cparams = BLOSC2_CPARAMS_DEFAULTS;
cparams.typesize = sizeof(int32_t);
cparams.compcode = BLOSC_BLOSCLZ;
cparams.clevel = 9;
cparams.nthreads = NTHREADS;
blosc2_storage storage = {.cparams=&cparams, .contiguous=false};
schunk = blosc2_schunk_new(&storage);
void* chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH + isize);
// Cache the special chunks
switch (svalue) {
case ZERO_DETECTION:
memset(data_buffer, 0, isize);
csize = blosc2_compress(5, 1, sizeof(int32_t), data_buffer, isize, chunk, osize);
break;
case CHECK_ZEROS:
csize = blosc2_chunk_zeros(cparams, isize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
break;
case CHECK_UNINIT:
csize = blosc2_chunk_uninit(cparams, isize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
break;
case CHECK_NANS:
csize = blosc2_chunk_nans(cparams, isize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
break;
case CHECK_VALUES:
csize = blosc2_chunk_repeatval(cparams, isize, chunk,
BLOSC_EXTENDED_HEADER_LENGTH + sizeof(int32_t), &value);
break;
default:
printf("Unknown case\n");
exit(1);
}
if (csize < 0) {
printf("Error creating chunk: %d\n", csize);
exit(1);
}
/* Append the chunks */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
nchunks = blosc2_schunk_append_chunk(schunk, chunk, true);
if (nchunks < 0) {
printf("Error appending chunk: %" PRId64 "\n", nchunks);
exit(1);
}
}
blosc_set_timestamp(¤t);
free(chunk);
totaltime = blosc_elapsed_secs(last, current);
printf("\n[Compr] Elapsed time:\t %6.3f s."
" Processed data: %.3f GB (%.3f GB/s)\n",
totaltime, totalsize / GB, totalsize / (GB * totaltime));
/* Gather some info */
nbytes = schunk->nbytes;
frame_len = blosc2_schunk_frame_len(schunk);
printf("Compression super-chunk: %ld -> %ld (%.1fx)\n",
(long)nbytes, (long)frame_len, (1. * (double)nbytes) / (double)frame_len);
/* Retrieve and decompress the chunks */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, rec_buffer, isize);
if (dsize < 0) {
printf("Decompression error. Error code: %d\n", dsize);
exit(dsize);
}
assert (dsize == (int)isize);
}
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
totalsize = (double)(isize) * (double)nchunks;
printf("[Decompr] Elapsed time:\t %6.3f s."
" Processed data: %.3f GB (%.3f GB/s)\n",
totaltime, totalsize / GB, totalsize / (GB * totaltime));
/* Exercise the getitem */
blosc_set_timestamp(&last);
for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
bool needs_free;
uint8_t* chunk_;
csize = blosc2_schunk_get_chunk(schunk, nchunk, &chunk_, &needs_free);
if (csize < 0) {
printf("blosc2_schunk_get_chunk error. Error code: %d\n", dsize);
return csize;
}
switch (svalue) {
case CHECK_VALUES:
rc = blosc1_getitem(chunk_, nchunk, 1, &value);
if (rc < 0) {
printf("Error in getitem of a special value\n");
return rc;
}
if (value != REPEATED_VALUE) {
printf("Wrong value!");
exit(1);
}
break;
case CHECK_NANS:
rc = blosc1_getitem(chunk_, nchunk, 1, &fvalue);
if (rc < 0) {
printf("Error in getitem of a special value\n");
return rc;
}
if (!isnan(fvalue)) {
printf("Wrong value!");
exit(1);
}
break;
case CHECK_ZEROS:
rc = blosc1_getitem(chunk_, nchunk, 1, &value);
if (rc < 0) {
printf("Error in getitem of zeros value\n");
return rc;
}
if (value != 0) {
printf("Wrong value!");
exit(1);
}
break;
default:
// It can only be non-initialized
rc = blosc1_getitem(chunk_, nchunk, 1, &value);
if (rc < 0) {
printf("Error in getitem of an non-initialized value\n");
return rc;
}
}
if (needs_free) {
free(chunk_);
}
}
blosc_set_timestamp(¤t);
totaltime = blosc_elapsed_secs(last, current);
printf("[getitem] Elapsed time:\t %6.3f s.\n", totaltime);
// /* Check that all the values have a good roundtrip */
// blosc_set_timestamp(&last);
// for (nchunk = 0; nchunk < NCHUNKS; nchunk++) {
// dsize = blosc2_schunk_decompress_chunk(schunk, nchunk, (void *) rec_buffer, isize);
// if (dsize < 0) {
// printf("Decompression error. Error code: %d\n", dsize);
// return dsize;
// }
// assert (dsize == (int)isize);
// if (CHECK_VALUE) {
// int32_t* buffer = (int32_t*)rec_buffer;
// for (int i = 0; i < CHUNKSIZE; i++) {
// if (buffer[i] != REPEATED_VALUE) {
// printf("Value is not correct in chunk %d, position: %d\n", nchunk, i);
// return -1;
// }
// }
// }
// else if (CHECK_NAN) {
// float* buffer = (float*)rec_buffer;
// for (int i = 0; i < CHUNKSIZE; i++) {
// if (!isnan(buffer[i])) {
// printf("Value is not correct in chunk %d, position: %d\n", nchunk, i);
// return -1;
// }
// }
// }
// else {
// int32_t* buffer = (int32_t*)rec_buffer;
// for (int i = 0; i < CHUNKSIZE; i++) {
// if (buffer[i] != 0) {
// printf("Value is not correct in chunk %d, position: %d\n", nchunk, i);
// return -1;
// }
// }
// }
// }
// printf("All data did a good roundtrip!\n");
/* Free resources */
free(data_buffer);
free(rec_buffer);
/* Destroy the super-chunk */
blosc2_schunk_free(schunk);
/* Destroy the Blosc environment */
blosc2_destroy();
return 0;
}
int main(void) {
int rc;
printf("*** Testing special zeros...");
rc = check_special_values(CHECK_ZEROS);
if (rc < 0) {
return rc;
}
printf("*** Testing NaNs...");
rc = check_special_values(CHECK_NANS);
if (rc < 0) {
return rc;
}
printf("*** Testing repeated values...");
rc = check_special_values(CHECK_VALUES);
if (rc < 0) {
return rc;
}
printf("*** Testing non-initialized values...");
rc = check_special_values(CHECK_UNINIT);
if (rc < 0) {
return rc;
}
printf("Testing zero detection...");
rc = check_special_values(ZERO_DETECTION);
if (rc < 0) {
return rc;
}
}
Blosc-c-blosc2-d3bee7c/blosc/ 0000775 0000000 0000000 00000000000 15255167342 0016050 5 ustar 00root root 0000000 0000000 Blosc-c-blosc2-d3bee7c/blosc/CMakeLists.txt 0000664 0000000 0000000 00000037450 15255167342 0020621 0 ustar 00root root 0000000 0000000 # Blosc - Blocked Shuffling and Compression Library
#
# Copyright (c) 2021 Blosc Development Team
# https://blosc.org
# License: BSD 3-Clause (see LICENSE.txt)
#
# See LICENSE.txt for details about copyright and rights to use.
# A simple way to detect that we are using CMAKE
add_definitions(-DUSING_CMAKE)
set(version_string ${BLOSC2_VERSION_MAJOR}.${BLOSC2_VERSION_MINOR}.${BLOSC2_VERSION_PATCH})
# targets
if(BUILD_SHARED)
add_library(blosc2_shared SHARED)
# ALIAS for superbuilds that use Blosc2 as sub-project
# must be the same as the NAMESPACE in Blosc2Targets
add_library(Blosc2::blosc2_shared ALIAS blosc2_shared)
set_target_properties(blosc2_shared PROPERTIES
OUTPUT_NAME blosc2
# Hide symbols by default unless they're specifically exported.
# This makes it easier to keep the set of exported symbols the
# same across all compilers/platforms.
C_VISIBILITY_PRESET hidden
)
if(MSVC OR MINGW)
set_target_properties(blosc2_shared PROPERTIES PREFIX lib)
endif()
set_target_properties(blosc2_shared PROPERTIES
VERSION ${version_string}
SOVERSION 9 # Change this when an ABI change happens
)
target_compile_definitions(blosc2_shared PRIVATE BLOSC_SHARED_LIBRARY)
target_include_directories(blosc2_shared PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(blosc2_shared PUBLIC
$
$)
endif()
if(BUILD_STATIC)
add_compile_definitions(BUILD_STATIC)
add_library(blosc2_static STATIC)
# ALIAS for superbuilds that use Blosc2 as sub-project
# must be the same as the NAMESPACE in Blosc2Targets
add_library(Blosc2::blosc2_static ALIAS blosc2_static)
set_target_properties(blosc2_static PROPERTIES
OUTPUT_NAME blosc2
POSITION_INDEPENDENT_CODE ON
# Hide symbols by default unless they're specifically exported.
# This makes it easier to keep the set of exported symbols the
# same across all compilers/platforms.
C_VISIBILITY_PRESET hidden
)
if(MSVC OR MINGW)
set_target_properties(blosc2_static PROPERTIES PREFIX lib)
endif()
target_include_directories(blosc2_static PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(blosc2_static PUBLIC
$
$)
endif()
# Add unified target
add_library(Blosc2::blosc2 ALIAS ${UNIFIED_TARGET_ALIAS})
# When the option has been selected to compile the test suite,
# compile an additional version of blosc2_static which exports
# some normally-hidden symbols (to facilitate unit testing).
if(BUILD_TESTS)
add_library(blosc_testing STATIC)
set_target_properties(blosc_testing PROPERTIES OUTPUT_NAME blosc_testing)
if(MSVC OR MINGW)
set_target_properties(blosc_testing PROPERTIES PREFIX lib)
endif()
target_compile_definitions(blosc_testing PUBLIC
BLOSC_TESTING
BLOSC_SHARED_LIBRARY # for EXPORT macro
)
target_include_directories(blosc_testing PUBLIC
$
$)
endif()
set(INTERNAL_LIBS ${PROJECT_SOURCE_DIR}/internal-complibs)
# Threads
if(HAVE_THREADS)
if(CMAKE_VERSION VERSION_LESS 3.1)
set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT})
else()
set(LIBS ${LIBS} Threads::Threads)
endif()
else()
message(FATAL_ERROR "Threads required but not found.")
endif()
if(WIN32)
list(APPEND SOURCES blosc/win32/threading.c)
endif()
# dlopen/dlclose
if(NOT WIN32)
set(LIBS ${LIBS} ${CMAKE_DL_LIBS})
endif()
# link dependencies
# "link" dependent targets via target_link_libraries (preferred) and
# manually add includes / libs for others
if(LZ4_FOUND)
if(TARGET LZ4::lz4)
list(APPEND LIBS LZ4::lz4)
elseif(TARGET lz4_static)
list(APPEND LIBS lz4_static)
elseif(TARGET lz4)
list(APPEND LIBS lz4)
else()
list(APPEND LIBS ${LZ4_LIBRARY})
list(APPEND BLOSC_INCLUDE_DIRS ${LZ4_INCLUDE_DIRS})
endif()
else()
message(FATAL_ERROR "LZ4 support enabled, but neither external lz4 nor FetchContent lz4 is available.")
endif()
if(NOT DEACTIVATE_ZLIB)
if(ZLIB_NG_FOUND)
if(TARGET zlib-ng::zlibstatic)
list(APPEND LIBS zlib-ng::zlibstatic)
elseif(TARGET zlib-ng::zlib)
list(APPEND LIBS zlib-ng::zlib)
else()
list(APPEND LIBS ${ZLIB_NG_LIBRARIES})
list(APPEND BLOSC_INCLUDE_DIRS ${ZLIB_NG_INCLUDE_DIR})
endif()
elseif(ZLIB_FOUND)
list(APPEND LIBS ZLIB::ZLIB)
else()
message(FATAL_ERROR "ZLIB support enabled, but neither external zlib nor FetchContent zlib-ng is available.")
endif()
endif()
if(NOT DEACTIVATE_ZSTD)
if(ZSTD_FOUND)
if(TARGET zstd::libzstd_shared)
list(APPEND LIBS zstd::libzstd_shared)
elseif(TARGET zstd::libzstd_static)
list(APPEND LIBS zstd::libzstd_static)
elseif(TARGET zstd::libzstd)
list(APPEND LIBS zstd::libzstd)
elseif(TARGET libzstd_static)
list(APPEND LIBS libzstd_static)
elseif(TARGET libzstd_shared)
list(APPEND LIBS libzstd_shared)
else()
list(APPEND LIBS ${ZSTD_LIBRARY})
list(APPEND BLOSC_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
endif()
else()
message(FATAL_ERROR "ZSTD support enabled, but neither external zstd nor FetchContent zstd is available.")
endif()
endif()
if(BUILD_PLUGINS AND TARGET zfp)
list(APPEND LIBS zfp)
endif()
if(UNIX AND NOT APPLE)
set(LIBS ${LIBS} "rt")
set(LIBS ${LIBS} "m")
# set(LIBS ${LIBS} "profiler")
endif()
# Blosc2 library source files
list(APPEND SOURCES
blosc/blosc2.c
blosc/blosclz.c
blosc/fastcopy.c
blosc/fastcopy.h
blosc/schunk.c
blosc/frame.c
blosc/stune.c
blosc/stune.h
blosc/context.h
blosc/delta.c
blosc/delta.h
blosc/shuffle-generic.c
blosc/bitshuffle-generic.c
blosc/trunc-prec.c
blosc/trunc-prec.h
blosc/timestamp.c
blosc/sframe.c
blosc/directories.c
blosc/blosc2-stdio.c
blosc/b2nd.c
blosc/b2nd_utils.c
)
if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL arm64)
if(COMPILER_SUPPORT_SSE2)
message(STATUS "Adding run-time support for SSE2")
list(APPEND SOURCES blosc/shuffle-sse2.c blosc/bitshuffle-sse2.c)
endif()
if(COMPILER_SUPPORT_AVX2)
message(STATUS "Adding run-time support for AVX2")
list(APPEND SOURCES blosc/shuffle-avx2.c blosc/bitshuffle-avx2.c)
endif()
if(COMPILER_SUPPORT_AVX512)
message(STATUS "Adding run-time support for AVX512")
list(APPEND SOURCES blosc/bitshuffle-avx512.c)
endif()
endif()
if(COMPILER_SUPPORT_NEON)
message(STATUS "Adding run-time support for NEON")
# bitshuffle-neon.c does not offer better speed than generic on arm64 (Mac M1).
# Besides, it does not compile on raspberry pi (armv7l), so disable it.
list(APPEND SOURCES blosc/shuffle-neon.c) # blosc/bitshuffle-neon.c)
endif()
if(COMPILER_SUPPORT_ALTIVEC)
message(STATUS "Adding run-time support for ALTIVEC")
list(APPEND SOURCES blosc/shuffle-altivec.c blosc/bitshuffle-altivec.c)
endif()
list(APPEND SOURCES blosc/shuffle.c)
# Based on the target architecture and hardware features supported
# by the C compiler, set hardware architecture optimization flags
# for specific shuffle implementations.
if(COMPILER_SUPPORT_SSE2)
include(CheckCCompilerFlag)
# Probe for SSSE3 flags for various compilers
check_c_compiler_flag("-mssse3" COMPILER_HAS_mssse3)
check_c_compiler_flag("/arch:SSSE3" COMPILER_HAS_arch_SSSE3)
if(COMPILER_HAS_mssse3)
set(SSSE3_FLAG "-mssse3")
elseif(COMPILER_HAS_arch_SSSE3)
set(SSSE3_FLAG "/arch:SSSE3")
else()
set(SSSE3_FLAG "")
endif()
if(MSVC)
# MSVC targets SSE2 by default on 64-bit configurations, but not 32-bit configurations.
if(${CMAKE_SIZEOF_VOID_P} EQUAL 4)
set_source_files_properties(
shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c
PROPERTIES COMPILE_OPTIONS "/arch:SSE2")
endif()
else()
set_source_files_properties(
shuffle-sse2.c bitshuffle-sse2.c blosclz.c fastcopy.c
PROPERTIES COMPILE_OPTIONS -msse2)
endif()
if(SSSE3_FLAG)
set_source_files_properties( ${PROJECT_SOURCE_DIR}/plugins/filters/bytedelta/bytedelta.c PROPERTIES COMPILE_OPTIONS "${SSSE3_FLAG}" COMPILE_DEFINITIONS "HAVE_SSSE3=1" )
endif()
# Define a symbol for the shuffle-dispatch implementation
# so it knows SSE2 is supported even though that file is
# compiled without SSE2 support (for portability).
set_property(
SOURCE shuffle.c
APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_SSE2_ENABLED)
endif()
if(COMPILER_SUPPORT_AVX2)
if(MSVC)
set_source_files_properties(
shuffle-avx2.c bitshuffle-avx2.c
PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
else()
set_source_files_properties(
shuffle-avx2.c bitshuffle-avx2.c
PROPERTIES COMPILE_OPTIONS -mavx2)
endif()
# Define a symbol for the shuffle-dispatch implementation
# so it knows AVX2 is supported even though that file is
# compiled without AVX2 support (for portability).
set_property(
SOURCE shuffle.c
APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_AVX2_ENABLED)
endif()
if(COMPILER_SUPPORT_AVX512)
if(MSVC)
set_source_files_properties(
bitshuffle-avx512.c
PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
else()
set_source_files_properties(
bitshuffle-avx512.c
PROPERTIES COMPILE_OPTIONS "-mavx512f;-mavx512bw")
endif()
# Define a symbol for the shuffle-dispatch implementation
# so it knows AVX512 is supported even though that file is
# compiled without AVX512 support (for portability).
set_property(
SOURCE shuffle.c
APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_AVX512_ENABLED)
endif()
if(COMPILER_SUPPORT_NEON)
set_source_files_properties(
shuffle-neon.c bitshuffle-neon.c
PROPERTIES COMPILE_OPTIONS "-flax-vector-conversions")
if(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)
# Only armv7l needs special -mfpu=neon flag; aarch64 doesn't.
set_source_files_properties(
shuffle-neon.c bitshuffle-neon.c
PROPERTIES COMPILE_OPTIONS "-mfpu=neon;-flax-vector-conversions")
endif()
# Define a symbol for the shuffle-dispatch implementation
# so it knows NEON is supported even though that file is
# compiled without NEON support (for portability).
set_property(
SOURCE shuffle.c
APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_NEON_ENABLED)
endif()
if(COMPILER_SUPPORT_ALTIVEC)
set_source_files_properties(
shuffle-altivec.c bitshuffle-altivec.c
PROPERTIES COMPILE_OPTIONS -DNO_WARN_X86_INTRINSICS)
set_property(
SOURCE shuffle.c
APPEND PROPERTY COMPILE_OPTIONS -DNO_WARN_X86_INTRINSICS)
# Define a symbol for the shuffle-dispatch implementation
# so it knows ALTIVEC is supported even though that file is
# compiled without ALTIVEC support (for portability).
set_property(
SOURCE shuffle.c
APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_ALTIVEC_ENABLED)
endif()
# add libraries for dependencies that are not CMake targets
if(BUILD_SHARED)
target_link_libraries(blosc2_shared PRIVATE ${LIBS})
target_include_directories(blosc2_shared PRIVATE ${BLOSC_INCLUDE_DIRS})
endif()
if(BUILD_STATIC)
target_link_libraries(blosc2_static PRIVATE ${LIBS})
target_include_directories(blosc2_static PRIVATE ${BLOSC_INCLUDE_DIRS})
endif()
if(BUILD_TESTS)
target_link_libraries(blosc_testing PRIVATE ${LIBS})
target_include_directories(blosc_testing PRIVATE ${BLOSC_INCLUDE_DIRS})
endif()
# When using fetched dependencies, make the installed static library self-contained
# by folding the private bundled dependency archives into libblosc2.a. This keeps
# bundled dependencies out of the public install layout and lets consumers link the
# wheel-provided static library without chasing private dependency archives.
set(BLOSC_BUNDLED_STATIC_TARGETS)
if(BUILD_STATIC)
if(LZ4_FETCHED AND TARGET lz4_static)
list(APPEND BLOSC_BUNDLED_STATIC_TARGETS lz4_static)
endif()
if(ZLIB_NG_FETCHED AND ZLIB_NG_INSTALL_TARGET AND TARGET ${ZLIB_NG_INSTALL_TARGET})
list(APPEND BLOSC_BUNDLED_STATIC_TARGETS ${ZLIB_NG_INSTALL_TARGET})
endif()
if(ZSTD_FETCHED AND TARGET libzstd_static)
list(APPEND BLOSC_BUNDLED_STATIC_TARGETS libzstd_static)
endif()
if(ZFP_FETCHED AND TARGET zfp)
list(APPEND BLOSC_BUNDLED_STATIC_TARGETS zfp)
endif()
endif()
if(BLOSC_BUNDLED_STATIC_TARGETS)
set(BLOSC_STATIC_MERGE_INPUTS "$")
foreach(BLOSC_BUNDLED_STATIC_TARGET IN LISTS BLOSC_BUNDLED_STATIC_TARGETS)
list(APPEND BLOSC_STATIC_MERGE_INPUTS "$")
endforeach()
string(JOIN "|" BLOSC_STATIC_MERGE_INPUTS_ARG ${BLOSC_STATIC_MERGE_INPUTS})
set(BLOSC_STATIC_MERGE_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/merge_blosc2_static_$.cmake")
file(GENERATE OUTPUT "${BLOSC_STATIC_MERGE_SCRIPT}" CONTENT
"set(OUTPUT [=[$]=])
set(INPUTS [=[${BLOSC_STATIC_MERGE_INPUTS_ARG}]=])
set(AR [=[${CMAKE_AR}]=])
set(RANLIB [=[${CMAKE_RANLIB}]=])
set(SYSTEM_NAME [=[${CMAKE_SYSTEM_NAME}]=])
set(MSVC [=[$]=])
include([=[${PROJECT_SOURCE_DIR}/cmake/merge_static_libs.cmake]=])
")
set(BLOSC_CMAKE_COMMAND "${CMAKE_COMMAND}")
if(WIN32)
# Avoid nested cmd.exe quoting problems with full paths such as
# "C:\Program Files\CMake\bin\cmake.exe" in Ninja post-build rules.
set(BLOSC_CMAKE_COMMAND cmake)
endif()
add_custom_command(TARGET blosc2_static POST_BUILD
COMMAND "${BLOSC_CMAKE_COMMAND}" -P "${BLOSC_STATIC_MERGE_SCRIPT}"
COMMENT "Merging bundled dependency archives into libblosc2.a"
VERBATIM)
# The bundled dependency objects are now inside libblosc2.a, so do not make
# installed CMake consumers depend on the private dependency targets.
get_target_property(BLOSC_STATIC_INTERFACE_LINK_LIBRARIES blosc2_static INTERFACE_LINK_LIBRARIES)
if(BLOSC_STATIC_INTERFACE_LINK_LIBRARIES)
set(BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS ${BLOSC_BUNDLED_STATIC_TARGETS})
if(LZ4_FETCHED)
list(APPEND BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS lz4_static LZ4::lz4)
endif()
if(ZLIB_NG_FETCHED)
list(APPEND BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS
zlib-ng-static zlib-ng zlib zlibstatic zlib-ng::zlibstatic zlib-ng::zlib)
endif()
if(ZSTD_FETCHED)
list(APPEND BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS
libzstd_static libzstd_shared zstd::libzstd_static zstd::libzstd_shared zstd::libzstd)
endif()
if(ZFP_FETCHED)
list(APPEND BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS zfp ZFP::zfp zfp::zfp ZFP::ZFP)
endif()
list(REMOVE_DUPLICATES BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS)
foreach(BLOSC_BUNDLED_STATIC_TARGET IN LISTS BLOSC_BUNDLED_STATIC_INTERFACE_TARGETS)
list(REMOVE_ITEM BLOSC_STATIC_INTERFACE_LINK_LIBRARIES "$")
endforeach()
set_target_properties(blosc2_static PROPERTIES
INTERFACE_LINK_LIBRARIES "${BLOSC_STATIC_INTERFACE_LINK_LIBRARIES}")
endif()
endif()
# we use this variable in the CMake file one directory above ours
set(SOURCES ${SOURCES} PARENT_SCOPE)
Blosc-c-blosc2-d3bee7c/blosc/b2nd-private.h 0000664 0000000 0000000 00000002436 15255167342 0020523 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
#ifndef BLOSC_B2ND_PRIVATE_H
#define BLOSC_B2ND_PRIVATE_H
#include "b2nd.h"
#include
#include
#include
/*********************************************************************
Functions meant to be used internally.
*********************************************************************/
/**
* @brief Get the chunk indexes needed to get the slice.
*
* @param array The b2nd array.
* @param start The coordinates where the slice will begin.
* @param stop The coordinates where the slice will end.
* @param chunks_idx The pointer to the buffer where the indexes of the chunks will be written.
*
* @return The number of chunks needed to get the slice. If some problem is
* detected, a negative code is returned instead.
*/
int64_t b2nd_get_slice_nchunks(b2nd_array_t *array, const int64_t *start, const int64_t *stop, int64_t **chunks_idx);
#endif /* BLOSC_B2ND_PRIVATE_H */
Blosc-c-blosc2-d3bee7c/blosc/b2nd.c 0000664 0000000 0000000 00000322532 15255167342 0017050 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
#include "b2nd.h"
#include "context.h"
#include "frame.h"
#include "blosc2/blosc2-common.h"
#include "blosc2.h"
#include
#include
#include
#include
#include
static bool b2nd_mul_overflow_size_t(size_t a, size_t b, size_t *out) {
if (a != 0 && b > SIZE_MAX / a) {
return true;
}
if (out != NULL) {
*out = a * b;
}
return false;
}
/* Signed counterparts of the above. Shape products are accumulated with these
* so that crafted metadata cannot wrap them: signed overflow is UB, and a
* wrapped product reaches divisors (see the nblocks computations) and
* allocation sizes. Negative operands are reported as overflow too, so callers
* get one error path rather than two. */
static bool b2nd_mul_overflow_int32(int32_t a, int32_t b, int32_t *out) {
if (a < 0 || b < 0 || (a != 0 && b > INT32_MAX / a)) {
return true;
}
if (out != NULL) {
*out = a * b;
}
return false;
}
static bool b2nd_mul_overflow_int64(int64_t a, int64_t b, int64_t *out) {
if (a < 0 || b < 0 || (a != 0 && b > INT64_MAX / a)) {
return true;
}
if (out != NULL) {
*out = a * b;
}
return false;
}
int b2nd_serialize_meta(int8_t ndim, const int64_t *shape, const int32_t *chunkshape,
const int32_t *blockshape, const char *dtype, int8_t dtype_format,
uint8_t **smeta) {
if (dtype == NULL) {
dtype = B2ND_DEFAULT_DTYPE;
}
// dtype checks
if (dtype_format < 0) {
BLOSC_TRACE_ERROR("dtype_format cannot be negative");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
size_t dtype_len0 = strlen(dtype);
if (dtype_len0 > INT32_MAX) {
BLOSC_TRACE_ERROR("dtype is too large (len > %d)", INT32_MAX);
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
const int32_t dtype_len = (int32_t) dtype_len0;
// Allocate space for b2nd metalayer
int32_t max_smeta_len = (int32_t) (1 + 1 + 1 + (1 + ndim * (1 + sizeof(int64_t))) +
(1 + ndim * (1 + sizeof(int32_t))) + (1 + ndim * (1 + sizeof(int32_t))) +
1 + 1 + sizeof(int32_t) + dtype_len);
*smeta = malloc((size_t) max_smeta_len);
BLOSC_ERROR_NULL(*smeta, BLOSC2_ERROR_MEMORY_ALLOC);
uint8_t *pmeta = *smeta;
// Build an array with 7 entries (version, ndim, shape, chunkshape, blockshape, dtype_format, dtype)
*pmeta++ = 0x90 + 7;
// version entry
*pmeta++ = B2ND_METALAYER_VERSION; // positive fixnum (7-bit positive integer)
// ndim entry
*pmeta++ = (uint8_t) ndim; // positive fixnum (7-bit positive integer)
// shape entry
*pmeta++ = (uint8_t) (0x90) + ndim; // fix array with ndim elements
for (uint8_t i = 0; i < ndim; i++) {
*pmeta++ = 0xd3; // int64
swap_store(pmeta, shape + i, sizeof(int64_t));
pmeta += sizeof(int64_t);
}
// chunkshape entry
*pmeta++ = (uint8_t) (0x90) + ndim; // fix array with ndim elements
for (uint8_t i = 0; i < ndim; i++) {
*pmeta++ = 0xd2; // int32
swap_store(pmeta, chunkshape + i, sizeof(int32_t));
pmeta += sizeof(int32_t);
}
// blockshape entry
*pmeta++ = (uint8_t) (0x90) + ndim; // fix array with ndim elements
for (uint8_t i = 0; i < ndim; i++) {
*pmeta++ = 0xd2; // int32
swap_store(pmeta, blockshape + i, sizeof(int32_t));
pmeta += sizeof(int32_t);
}
// dtype entry
*pmeta++ = dtype_format; // positive fixint (7-bit positive integer)
*pmeta++ = (uint8_t) (0xdb); // str with up to 2^31 elements
swap_store(pmeta, &dtype_len, sizeof(int32_t));
pmeta += sizeof(int32_t);
memcpy(pmeta, dtype, dtype_len);
pmeta += dtype_len;
int32_t slen = (int32_t) (pmeta - *smeta);
if (max_smeta_len != slen) {
BLOSC_TRACE_ERROR("meta length is inconsistent!");
return BLOSC2_ERROR_FAILURE;
}
return (int)slen;
}
int b2nd_deserialize_meta(const uint8_t *smeta, int32_t smeta_len, int8_t *ndim, int64_t *shape,
int32_t *chunkshape, int32_t *blockshape, char **dtype, int8_t *dtype_format) {
return b2nd_deserialize_meta_inline(smeta, smeta_len, ndim, shape, chunkshape, blockshape, dtype, dtype_format);
}
static int validate_shape_chunkshape_blockshape(int8_t ndim, const int64_t *shape,
const int32_t *chunkshape, const int32_t *blockshape) {
for (int i = 0; i < ndim; ++i) {
if (shape[i] < 0) {
BLOSC_TRACE_ERROR("shape[%d] cannot be negative", i);
return BLOSC2_ERROR_INVALID_PARAM;
}
if (chunkshape[i] < 0 || blockshape[i] < 0) {
BLOSC_TRACE_ERROR("chunkshape[%d] and blockshape[%d] cannot be negative", i, i);
return BLOSC2_ERROR_INVALID_PARAM;
}
bool chunkshape_is_zero = (chunkshape[i] == 0);
bool blockshape_is_zero = (blockshape[i] == 0);
// Keep compatibility with contexts that use chunkshape=0 (e.g. empty slices).
// Reject only invalid tuples that can produce divide-by-zero when chunkshape is non-zero.
if (blockshape_is_zero && !chunkshape_is_zero) {
BLOSC_TRACE_ERROR("blockshape[%d] cannot be zero when chunkshape[%d] is non-zero", i, i);
return BLOSC2_ERROR_INVALID_PARAM;
}
}
return BLOSC2_ERROR_SUCCESS;
}
/* Recompute the cached geometry (shape/extshape/strides/...) without touching
* the b2nd metalayer. Used both by update_shape() and by the refresh path
* that re-syncs the struct from a metalayer another handle rewrote. */
static int update_shape_struct(b2nd_array_t *array, int8_t ndim, const int64_t *shape,
const int32_t *chunkshape, const int32_t *blockshape) {
BLOSC_ERROR(validate_shape_chunkshape_blockshape(ndim, shape, chunkshape, blockshape));
array->ndim = ndim;
array->nitems = 1;
array->extnitems = 1;
array->extchunknitems = 1;
array->chunknitems = 1;
array->blocknitems = 1;
for (int i = 0; i < B2ND_MAX_DIM; ++i) {
if (i < ndim) {
array->shape[i] = shape[i];
array->chunkshape[i] = chunkshape[i];
array->blockshape[i] = blockshape[i];
if (array->chunkshape[i] != 0) {
if (shape[i] % array->chunkshape[i] == 0) {
array->extshape[i] = shape[i];
} else {
// Rounding shape up to the next chunk must not wrap int64
if (shape[i] > INT64_MAX - chunkshape[i]) {
BLOSC_TRACE_ERROR("shape[%d] is too large to round up to chunkshape[%d]", i, i);
return BLOSC2_ERROR_INVALID_PARAM;
}
array->extshape[i] = shape[i] + chunkshape[i] - shape[i] % chunkshape[i];
}
if (chunkshape[i] % blockshape[i] == 0) {
array->extchunkshape[i] = chunkshape[i];
} else {
// Done in 64 bits: the sum of two int32 values does not fit in int32,
// and extchunkshape is int64 anyway
array->extchunkshape[i] =
(int64_t) chunkshape[i] + blockshape[i] - chunkshape[i] % blockshape[i];
}
} else {
array->extchunkshape[i] = chunkshape[i];
array->extshape[i] = 0;
}
} else {
array->blockshape[i] = 1;
array->chunkshape[i] = 1;
array->extshape[i] = 1;
array->extchunkshape[i] = 1;
array->shape[i] = 1;
}
// Crafted metadata can make any of these products wrap, which is UB and
// lands in divisors and allocation sizes downstream (see #795)
if (b2nd_mul_overflow_int64(array->nitems, array->shape[i], &array->nitems) ||
b2nd_mul_overflow_int64(array->extnitems, array->extshape[i], &array->extnitems) ||
b2nd_mul_overflow_int64(array->extchunknitems, array->extchunkshape[i],
&array->extchunknitems) ||
b2nd_mul_overflow_int32(array->chunknitems, array->chunkshape[i], &array->chunknitems) ||
b2nd_mul_overflow_int32(array->blocknitems, array->blockshape[i], &array->blocknitems)) {
BLOSC_TRACE_ERROR("shape, chunkshape or blockshape product overflows at dimension %d", i);
return BLOSC2_ERROR_INVALID_PARAM;
}
}
// Compute strides
if (ndim > 0) {
array->item_array_strides[ndim - 1] = 1;
array->item_extchunk_strides[ndim - 1] = 1;
array->item_chunk_strides[ndim - 1] = 1;
array->item_block_strides[ndim - 1] = 1;
array->block_chunk_strides[ndim - 1] = 1;
array->chunk_array_strides[ndim - 1] = 1;
}
for (int i = ndim - 2; i >= 0; --i) {
// Treat (chunkshape[i+1] == 0) the same as (shape[i+1] == 0): the
// dimension carries no actual chunking and any stride into it is moot.
// validate_shape_chunkshape_blockshape allows the chunkshape==0
// combination (paired with blockshape==0) as a placeholder for empty
// contexts (see example_empty_shape.c). Without this guard a crafted
// b2nd metalayer with shape[i+1]>0 but chunkshape[i+1]==blockshape[i+1]==0
// -- accepted by the validator -- reaches the (extchunkshape / blockshape)
// division below as 0/0, which is undefined behaviour and aborts the
// process with SIGFPE on x86. Crafted cframes loaded via b2nd_from_schunk
// / b2nd_open / b2nd_from_cframe would otherwise crash any consumer.
if (shape[i + 1] != 0 && array->chunkshape[i + 1] != 0 &&
array->blockshape[i + 1] != 0) {
array->item_array_strides[i] = array->item_array_strides[i + 1] * array->shape[i + 1];
array->item_extchunk_strides[i] =
array->item_extchunk_strides[i + 1] * array->extchunkshape[i + 1];
array->item_chunk_strides[i] =
array->item_chunk_strides[i + 1] * array->chunkshape[i + 1];
array->item_block_strides[i] =
array->item_block_strides[i + 1] * array->blockshape[i + 1];
array->block_chunk_strides[i] = array->block_chunk_strides[i + 1] *
(array->extchunkshape[i + 1] /
array->blockshape[i + 1]);
array->chunk_array_strides[i] = array->chunk_array_strides[i + 1] *
(array->extshape[i + 1] * array->chunkshape[i + 1]);
} else {
array->item_array_strides[i] = 0;
array->item_extchunk_strides[i] = 0;
array->item_chunk_strides[i] = 0;
array->item_block_strides[i] = 0;
array->block_chunk_strides[i] = 0;
array->chunk_array_strides[i] = 0;
}
}
return BLOSC2_ERROR_SUCCESS;
}
/* Publish the current cached geometry into the "b2nd" metalayer. */
static int publish_shape_meta(b2nd_array_t *array) {
if (!array->sc) {
return BLOSC2_ERROR_SUCCESS;
}
uint8_t *smeta = NULL;
// Serialize the dimension info ...
int32_t smeta_len =
b2nd_serialize_meta(array->ndim, array->shape, array->chunkshape, array->blockshape,
array->dtype, array->dtype_format, &smeta);
if (smeta_len < 0) {
BLOSC_TRACE_ERROR("Error during serializing dims info for Blosc2 NDim");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
// ... and update it in its metalayer
if (blosc2_meta_exists(array->sc, "b2nd") < 0) {
if (blosc2_meta_add(array->sc, "b2nd", smeta, smeta_len) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
} else {
if (blosc2_meta_update(array->sc, "b2nd", smeta, smeta_len) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
free(smeta);
// The meta mutation bumped the schunk change tick; this struct is in sync
array->last_tick = array->sc->change_tick;
return BLOSC2_ERROR_SUCCESS;
}
int update_shape(b2nd_array_t *array, int8_t ndim, const int64_t *shape,
const int32_t *chunkshape, const int32_t *blockshape) {
BLOSC_ERROR(update_shape_struct(array, ndim, shape, chunkshape, blockshape));
return publish_shape_meta(array);
}
/* Re-sync the cached geometry when another handle changed the b2nd metalayer
* behind our back (the growth-SWMR case: a writer resizes, readers follow).
* A no-op for arrays not backed by an on-disk frame.
* Returns 1 if the geometry was re-synced, 0 if it was already current, or
* a negative error code. */
static int refresh_if_stale(b2nd_array_t *array) {
blosc2_schunk *sc = array->sc;
if (sc == NULL || sc->frame == NULL) {
return BLOSC2_ERROR_SUCCESS;
}
BLOSC_ERROR(frame_check_stale((blosc2_frame_s *) sc->frame));
if (array->last_tick == sc->change_tick) {
return BLOSC2_ERROR_SUCCESS;
}
uint8_t *smeta;
int32_t smeta_len;
if (blosc2_meta_get(sc, "b2nd", &smeta, &smeta_len) < 0) {
if (blosc2_meta_get(sc, "caterva", &smeta, &smeta_len) < 0) {
// blosc2_meta_get() itself is a pure in-memory lookup on
// schunk->metalayers[], but that array was just repopulated by
// frame_check_stale() -> frame_refresh_if_stale() -> frame_get_metalayers(),
// which does its *own*, fully independent disk read (its own fresh
// get_header_info() + header re-read, frame.c) -- separate from both
// frame_check_stale()'s header read and the trailer read a few lines
// up in this same call chain. Without locking, nothing holds these
// reads together atomically, so this one can catch the header mid
// rewrite just as easily as the other two; a b2nd array always has
// one of these two metalayers from creation, so a miss here reflects
// that specific read landing in a torn window, not a genuine
// absence. Same opportunistic treatment as those reads: keep the
// cached view and let the next poll (its own fresh, independent read)
// resolve it, instead of hard-erroring on what is normally transient.
return BLOSC2_ERROR_SUCCESS;
}
}
int8_t ndim;
int64_t shape[B2ND_MAX_DIM];
int32_t chunkshape[B2ND_MAX_DIM];
int32_t blockshape[B2ND_MAX_DIM];
char *dtype;
int8_t dtype_format;
int rc = b2nd_deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape,
&dtype, &dtype_format);
free(smeta);
BLOSC_ERROR(rc);
free(dtype);
// Shape-only SWMR scope (matching HDF5): chunk/block geometry or ndim
// changing under a live handle is a misuse of the single-writer contract,
// not something to follow silently.
if (ndim != array->ndim ||
memcmp(chunkshape, array->chunkshape, ndim * sizeof(int32_t)) != 0 ||
memcmp(blockshape, array->blockshape, ndim * sizeof(int32_t)) != 0) {
BLOSC_TRACE_ERROR("ndim/chunkshape/blockshape changed behind this handle; "
"only shape changes are followed");
BLOSC_ERROR(BLOSC2_ERROR_DATA);
}
BLOSC_ERROR(update_shape_struct(array, ndim, shape, chunkshape, blockshape));
array->last_tick = sc->change_tick;
return 1;
}
int b2nd_refresh(b2nd_array_t *array) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
return refresh_if_stale(array);
}
int array_without_schunk(b2nd_context_t *ctx, b2nd_array_t **array) {
/* Create a b2nd_array_t buffer */
(*array) = (b2nd_array_t *) malloc(sizeof(b2nd_array_t));
BLOSC_ERROR_NULL(*array, BLOSC2_ERROR_MEMORY_ALLOC);
(*array)->sc = NULL;
(*array)->last_tick = 0;
(*array)->ndim = ctx->ndim;
int64_t *shape = ctx->shape;
int32_t *chunkshape = ctx->chunkshape;
int32_t *blockshape = ctx->blockshape;
// update_shape() rejects shapes whose products overflow, so this can fail on
// crafted metadata. Hand back NULL rather than the half-built array, which
// would otherwise leak and leave callers with a non-NULL pointer on error.
int rc = update_shape(*array, ctx->ndim, shape, chunkshape, blockshape);
if (rc < 0) {
free(*array);
*array = NULL;
BLOSC_ERROR(rc);
}
if (ctx->dtype != NULL) {
(*array)->dtype = malloc(strlen(ctx->dtype) + 1);
strcpy((*array)->dtype, ctx->dtype);
} else {
(*array)->dtype = NULL;
}
(*array)->dtype_format = ctx->dtype_format;
// The partition cache (empty initially)
(*array)->chunk_cache.data = NULL;
(*array)->chunk_cache.nchunk = -1; // means no valid cache yet
return BLOSC2_ERROR_SUCCESS;
}
int array_new(b2nd_context_t *ctx, int special_value, b2nd_array_t **array) {
BLOSC_ERROR(array_without_schunk(ctx, array));
blosc2_schunk *sc = blosc2_schunk_new(ctx->b2_storage);
if (sc == NULL) {
BLOSC_TRACE_ERROR("Pointer is NULL");
return BLOSC2_ERROR_FAILURE;
}
// Set the chunksize for the schunk, as it cannot be derived from storage
if (sc->typesize <= 0) {
BLOSC_TRACE_ERROR("Invalid chunk parameters");
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_INVALID_PARAM;
}
if ((*array)->extchunknitems < 0) {
BLOSC_TRACE_ERROR("Invalid chunk parameters");
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_INVALID_PARAM;
}
if ((uint64_t)(*array)->extchunknitems > (uint64_t)SIZE_MAX) {
BLOSC_TRACE_ERROR("extchunknitems too large");
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_INVALID_PARAM;
}
size_t chunkbytes_size = 0;
if (b2nd_mul_overflow_size_t((size_t)(*array)->extchunknitems, (size_t)sc->typesize, &chunkbytes_size)) {
BLOSC_TRACE_ERROR("Chunksize overflows size limits");
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_INVALID_PARAM;
}
if (chunkbytes_size > BLOSC2_MAX_BUFFERSIZE || chunkbytes_size > INT32_MAX) {
BLOSC_TRACE_ERROR("Chunksize exceeds maximum of %d", BLOSC2_MAX_BUFFERSIZE);
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED;
}
int32_t chunksize = (int32_t)chunkbytes_size;
sc->chunksize = chunksize;
// Serialize the dimension info
if (sc->nmetalayers >= BLOSC2_MAX_METALAYERS) {
BLOSC_TRACE_ERROR("the number of metalayers for this schunk has been exceeded");
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_FAILURE;
}
uint8_t *smeta = NULL;
int32_t smeta_len = b2nd_serialize_meta(ctx->ndim,
(*array)->shape,
(*array)->chunkshape,
(*array)->blockshape,
(*array)->dtype,
(*array)->dtype_format,
&smeta);
if (smeta_len < 0) {
BLOSC_TRACE_ERROR("error during serializing dims info for Blosc2 NDim");
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_FAILURE;
}
// And store it in b2nd metalayer
if (blosc2_meta_add(sc, "b2nd", smeta, smeta_len) < 0) {
free(smeta);
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_FAILURE;
}
free(smeta);
for (int i = 0; i < ctx->nmetalayers; ++i) {
char *name = ctx->metalayers[i].name;
uint8_t *data = ctx->metalayers[i].content;
int32_t size = ctx->metalayers[i].content_len;
if (blosc2_meta_add(sc, name, data, size) < 0) {
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
return BLOSC2_ERROR_FAILURE;
}
}
// Fill schunk with uninit values
if ((*array)->nitems != 0) {
int64_t nchunks = (*array)->extnitems / (*array)->chunknitems;
int64_t nitems = nchunks * (*array)->extchunknitems;
if (blosc2_schunk_fill_special(sc, nitems, special_value, chunksize) < 0) {
blosc2_schunk_free(sc);
free((*array)->dtype);
free(*array);
*array = NULL;
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
(*array)->sc = sc;
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_uninit(b2nd_context_t *ctx, b2nd_array_t **array) {
BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_UNINIT, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_empty(b2nd_context_t *ctx, b2nd_array_t **array) {
BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
// Fill with zeros to avoid variable cratios
BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_ZERO, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_zeros(b2nd_context_t *ctx, b2nd_array_t **array) {
BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_ZERO, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_nans(b2nd_context_t *ctx, b2nd_array_t **array) {
BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(array_new(ctx, BLOSC2_SPECIAL_NAN, array));
const int32_t typesize = (*array)->sc->typesize;
if (typesize != 4 && typesize != 8)
{
BLOSC_TRACE_ERROR("Unsupported typesize for NaN");
return BLOSC2_ERROR_DATA;
}
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_full(b2nd_context_t *ctx, b2nd_array_t **array, const void *fill_value) {
BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(b2nd_empty(ctx, array));
if ((*array)->sc->typesize <= 0) {
BLOSC_TRACE_ERROR("Invalid chunk parameters");
return BLOSC2_ERROR_INVALID_PARAM;
}
if ((*array)->extchunknitems < 0) {
BLOSC_TRACE_ERROR("Invalid chunk parameters");
return BLOSC2_ERROR_INVALID_PARAM;
}
if ((uint64_t)(*array)->extchunknitems > (uint64_t)SIZE_MAX) {
BLOSC_TRACE_ERROR("extchunknitems too large");
return BLOSC2_ERROR_INVALID_PARAM;
}
size_t chunkbytes_size = 0;
if (b2nd_mul_overflow_size_t((size_t)(*array)->extchunknitems, (size_t)(*array)->sc->typesize, &chunkbytes_size)) {
BLOSC_TRACE_ERROR("Chunk bytes overflows size limits");
return BLOSC2_ERROR_INVALID_PARAM;
}
if (chunkbytes_size > BLOSC2_MAX_BUFFERSIZE || chunkbytes_size > INT32_MAX) {
BLOSC_TRACE_ERROR("Chunk bytes exceeds maximum of %d", BLOSC2_MAX_BUFFERSIZE);
return BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED;
}
int32_t chunkbytes = (int32_t)chunkbytes_size;
blosc2_cparams *cparams;
if (blosc2_schunk_get_cparams((*array)->sc, &cparams) != 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
int32_t chunksize = BLOSC_EXTENDED_HEADER_LENGTH + (*array)->sc->typesize;
uint8_t *chunk = malloc(chunksize);
BLOSC_ERROR_NULL(chunk, BLOSC2_ERROR_MEMORY_ALLOC);
if (blosc2_chunk_repeatval(*cparams, chunkbytes, chunk, chunksize, fill_value) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
free(cparams);
for (int64_t i = 0; i < (*array)->sc->nchunks; ++i) {
if (blosc2_schunk_update_chunk((*array)->sc, i, chunk, true) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
free(chunk);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_from_schunk(blosc2_schunk *schunk, b2nd_array_t **array) {
BLOSC_ERROR_NULL(schunk, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
if (schunk == NULL) {
BLOSC_TRACE_ERROR("Schunk is null");
return BLOSC2_ERROR_NULL_POINTER;
}
blosc2_cparams *cparams;
if (blosc2_schunk_get_cparams(schunk, &cparams) < 0) {
BLOSC_TRACE_ERROR("Blosc error");
return BLOSC2_ERROR_NULL_POINTER;
}
free(cparams);
b2nd_context_t params = {0};
params.b2_storage = schunk->storage;
// Deserialize the b2nd metalayer
uint8_t *smeta;
int32_t smeta_len;
if (blosc2_meta_get(schunk, "b2nd", &smeta, &smeta_len) < 0) {
// Try with a caterva metalayer; we are meant to be backward compatible with it
if (blosc2_meta_get(schunk, "caterva", &smeta, &smeta_len) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_METALAYER_NOT_FOUND);
}
}
BLOSC_ERROR(b2nd_deserialize_meta(smeta, smeta_len, ¶ms.ndim, params.shape,
params.chunkshape, params.blockshape, ¶ms.dtype,
¶ms.dtype_format));
free(smeta);
// Reject crafted metalayers that pair shape[i]>0 with chunkshape[i]==0 or
// blockshape[i]==0. validate_shape_chunkshape_blockshape is intentionally
// lenient about chunkshape==0/blockshape==0 to support the placeholder
// pattern in b2nd_create_ctx (see example_empty_shape.c), where the
// companion shape entry is also 0 so no divide-by-zero ever materializes.
// Crafted on-wire metadata can violate that pairing, and many code paths
// downstream (b2nd_get_slice, b2nd_set_slice, set/get_orthogonal_selection,
// squeeze, resize) divide or mod by chunkshape[i] / blockshape[i] without
// re-checking. Enforce the pairing here so the deserialization path is
// strict even when the create-ctx path remains permissive.
for (int i = 0; i < params.ndim; ++i) {
if (params.shape[i] != 0 &&
(params.chunkshape[i] == 0 || params.blockshape[i] == 0)) {
BLOSC_TRACE_ERROR("b2nd metalayer: shape[%d] is non-zero but "
"chunkshape[%d] or blockshape[%d] is zero", i, i, i);
free(params.dtype);
return BLOSC2_ERROR_INVALID_PARAM;
}
}
// Free the deserialized dtype whether or not the array could be built; the
// rejection path is now reachable for crafted shapes (see #795)
int rc = array_without_schunk(¶ms, array);
free(params.dtype);
BLOSC_ERROR(rc);
(*array)->sc = schunk;
(*array)->last_tick = schunk->change_tick;
if ((*array) == NULL) {
BLOSC_TRACE_ERROR("Error creating a b2nd container from a frame");
return BLOSC2_ERROR_NULL_POINTER;
}
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_to_cframe(const b2nd_array_t *array, uint8_t **cframe, int64_t *cframe_len,
bool *needs_free) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(cframe, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(cframe_len, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(needs_free, BLOSC2_ERROR_NULL_POINTER);
*cframe_len = blosc2_schunk_to_buffer(array->sc, cframe, needs_free);
if (*cframe_len <= 0) {
BLOSC_TRACE_ERROR("Error serializing the b2nd array");
return BLOSC2_ERROR_FAILURE;
}
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_from_cframe(uint8_t *cframe, int64_t cframe_len, bool copy, b2nd_array_t **array) {
BLOSC_ERROR_NULL(cframe, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
blosc2_schunk *sc = blosc2_schunk_from_buffer(cframe, cframe_len, copy);
if (sc == NULL) {
BLOSC_TRACE_ERROR("Blosc error");
return BLOSC2_ERROR_FAILURE;
}
// ...and create a b2nd array out of it
BLOSC_ERROR(b2nd_from_schunk(sc, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_open(const char *urlpath, b2nd_array_t **array) {
BLOSC_ERROR_NULL(urlpath, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
blosc2_schunk *sc = blosc2_schunk_open(urlpath);
// ...and create a b2nd array out of it
BLOSC_ERROR(b2nd_from_schunk(sc, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_open_offset(const char *urlpath, b2nd_array_t **array, int64_t offset) {
BLOSC_ERROR_NULL(urlpath, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
blosc2_schunk *sc = blosc2_schunk_open_offset(urlpath, offset);
// ...and create a b2nd array out of it
BLOSC_ERROR(b2nd_from_schunk(sc, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_free(b2nd_array_t *array) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
if (array) {
if (array->sc != NULL) {
blosc2_schunk_free(array->sc);
}
free(array->dtype);
free(array);
}
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_from_cbuffer(b2nd_context_t *ctx, b2nd_array_t **array, const void *buffer, int64_t buffersize) {
BLOSC_ERROR_NULL(ctx, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(b2nd_empty(ctx, array));
if (buffersize < (int64_t) (*array)->nitems * (*array)->sc->typesize) {
BLOSC_TRACE_ERROR("The buffersize (%lld) is smaller than the array size (%lld)",
(long long) buffersize, (long long) (*array)->nitems * (*array)->sc->typesize);
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if ((*array)->nitems == 0) {
return BLOSC2_ERROR_SUCCESS;
}
int64_t start[B2ND_MAX_DIM] = {0};
int64_t *stop = (*array)->shape;
int64_t *shape = (*array)->shape;
BLOSC_ERROR(b2nd_set_slice_cbuffer(buffer, shape, buffersize, start, stop, *array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_to_cbuffer(const b2nd_array_t *array, void *buffer,
int64_t buffersize) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) array));
if (buffersize < (int64_t) array->nitems * array->sc->typesize) {
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if (array->nitems == 0) {
return BLOSC2_ERROR_SUCCESS;
}
int64_t start[B2ND_MAX_DIM] = {0};
const int64_t *stop = array->shape;
BLOSC_ERROR(b2nd_get_slice_cbuffer(array, start, stop, buffer, array->shape, buffersize));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_get_sparse_cbuffer(const b2nd_array_t *array, int64_t ncoords,
const int64_t *coords, void *buffer, int64_t buffersize) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) array));
if (ncoords < 0) {
BLOSC_TRACE_ERROR("ncoords must be non-negative");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if (ncoords == 0) {
return BLOSC2_ERROR_SUCCESS;
}
BLOSC_ERROR_NULL(coords, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
if (buffersize < ncoords * array->sc->typesize) {
BLOSC_TRACE_ERROR("Buffer is smaller than expected");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if (array->ndim == 0) {
BLOSC_TRACE_ERROR("Sparse indexing is not supported for 0-dimensional arrays");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
int64_t chunks_in_array[B2ND_MAX_DIM] = {0};
int64_t chunks_in_array_strides[B2ND_MAX_DIM] = {0};
chunks_in_array_strides[array->ndim - 1] = 1;
for (int j = 0; j < array->ndim; ++j) {
chunks_in_array[j] = array->chunkshape[j] == 0 ? 0 : array->extshape[j] / array->chunkshape[j];
}
for (int j = array->ndim - 2; j >= 0; --j) {
chunks_in_array_strides[j] = chunks_in_array_strides[j + 1] * chunks_in_array[j + 1];
}
int64_t *storage_coords = malloc((size_t)ncoords * sizeof(int64_t));
BLOSC_ERROR_NULL(storage_coords, BLOSC2_ERROR_MEMORY_ALLOC);
int rc = BLOSC2_ERROR_SUCCESS;
for (int64_t i = 0; i < ncoords; ++i) {
int64_t coord = coords[i];
if (coord < 0 || coord >= array->nitems) {
BLOSC_TRACE_ERROR("Coordinate out of bounds");
rc = BLOSC2_ERROR_INVALID_PARAM;
goto cleanup;
}
int64_t logical_index[B2ND_MAX_DIM] = {0};
int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
int64_t item_in_chunk[B2ND_MAX_DIM] = {0};
int64_t nblock_ndim[B2ND_MAX_DIM] = {0};
int64_t item_in_block[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim(array->ndim, (int64_t *)array->shape, coord, logical_index);
for (int j = 0; j < array->ndim; ++j) {
nchunk_ndim[j] = logical_index[j] / array->chunkshape[j];
item_in_chunk[j] = logical_index[j] % array->chunkshape[j];
nblock_ndim[j] = item_in_chunk[j] / array->blockshape[j];
item_in_block[j] = item_in_chunk[j] % array->blockshape[j];
}
int64_t nchunk;
blosc2_multidim_to_unidim(nchunk_ndim, array->ndim, chunks_in_array_strides, &nchunk);
int64_t nblock;
blosc2_multidim_to_unidim(nblock_ndim, array->ndim, array->block_chunk_strides, &nblock);
int64_t storage_item_in_block;
blosc2_multidim_to_unidim(item_in_block, array->ndim, array->item_block_strides, &storage_item_in_block);
storage_coords[i] = nchunk * array->extchunknitems + nblock * array->blocknitems + storage_item_in_block;
}
rc = blosc2_schunk_get_sparse_buffer(array->sc, ncoords, storage_coords, buffer);
cleanup:
free(storage_coords);
BLOSC_ERROR(rc);
return BLOSC2_ERROR_SUCCESS;
}
int64_t b2nd_get_slice_nchunks(const b2nd_array_t *array, const int64_t *start, const int64_t *stop, int64_t **chunks_idx) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(start, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(stop, BLOSC2_ERROR_NULL_POINTER);
int8_t ndim = array->ndim;
if (array->nitems == 0) {
*chunks_idx = NULL;
return 0;
}
// 0-dim case
if (ndim == 0) {
*chunks_idx = malloc(1 * sizeof(int64_t));
*chunks_idx[0] = 0;
return 1;
}
int64_t chunks_in_array[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunks_in_array[i] = array->extshape[i] / array->chunkshape[i];
}
int64_t chunks_in_array_strides[B2ND_MAX_DIM];
chunks_in_array_strides[ndim - 1] = 1;
for (int i = ndim - 2; i >= 0; --i) {
chunks_in_array_strides[i] = chunks_in_array_strides[i + 1] * chunks_in_array[i + 1];
}
// Compute the number of chunks to update
int64_t update_start[B2ND_MAX_DIM];
int64_t update_shape[B2ND_MAX_DIM];
int64_t update_nchunks = 1;
for (int i = 0; i < ndim; ++i) {
int64_t pos = 0;
while (pos <= start[i]) {
pos += array->chunkshape[i];
}
update_start[i] = pos / array->chunkshape[i] - 1;
while (pos < stop[i]) {
pos += array->chunkshape[i];
}
update_shape[i] = pos / array->chunkshape[i] - update_start[i];
update_nchunks *= update_shape[i];
}
int64_t nchunks = 0;
// Initially we do not know the number of chunks that will be affected
*chunks_idx = malloc(array->sc->nchunks * sizeof(int64_t));
int64_t *ptr = *chunks_idx;
for (int64_t update_nchunk = 0; update_nchunk < update_nchunks; ++update_nchunk) {
int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim(ndim, update_shape, update_nchunk, nchunk_ndim);
for (int i = 0; i < ndim; ++i) {
nchunk_ndim[i] += update_start[i];
}
int64_t nchunk;
blosc2_multidim_to_unidim(nchunk_ndim, ndim, chunks_in_array_strides, &nchunk);
// Check if the chunk is inside the slice domain
int64_t chunk_start[B2ND_MAX_DIM] = {0};
int64_t chunk_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunk_start[i] = nchunk_ndim[i] * array->chunkshape[i];
chunk_stop[i] = chunk_start[i] + array->chunkshape[i];
if (chunk_stop[i] > array->shape[i]) {
chunk_stop[i] = array->shape[i];
}
}
bool chunk_empty = false;
for (int i = 0; i < ndim; ++i) {
chunk_empty |= (chunk_stop[i] <= start[i] || chunk_start[i] >= stop[i]);
}
if (chunk_empty) {
continue;
}
ptr[nchunks] = nchunk;
nchunks++;
}
if (nchunks < array->sc->nchunks) {
*chunks_idx = realloc(ptr, nchunks * sizeof(int64_t));
}
return nchunks;
}
// Check whether the slice defined by start and stop is a single chunk and contiguous
// in the C order. This is a fast path for the get_slice and set_slice functions.
int64_t nchunk_fastpath(const b2nd_array_t *array, const int64_t *start,
const int64_t *stop, const int64_t slice_size) {
if (slice_size != array->chunknitems) {
return -1;
}
int ndim = (int) array->ndim;
int k = 0;
for (int i = 0; i < ndim; ++i) {
// The slice needs to correspond to a whole chunk (without padding)
if (start[i] % array->chunkshape[i] != 0) {
return -1;
}
if (stop[i] - start[i] != array->chunkshape[i]) {
return -1;
}
// There needs to exist 0 <= k <= ndim such that:
// - for i < k, blockshape[i] == 1
// - for i == k, blockshape[i] divides chunkshape[i]
// - for i > k, blockshape[i] == chunkshape[i]
if (array->chunkshape[i] % array->blockshape[i] != 0) {
return -1;
}
if (i > k && array->chunkshape[i] != array->blockshape[i]) {
return -1;
}
if (i == k && array->blockshape[i] == 1) {
k++;
}
}
// Compute the chunk number
int64_t *chunks_idx;
int64_t nchunks = b2nd_get_slice_nchunks(array, start, stop, &chunks_idx);
if (nchunks != 1) {
free(chunks_idx);
BLOSC_TRACE_ERROR("The number of chunks to read is not 1; go fix the code");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
int64_t nchunk = chunks_idx[0];
free(chunks_idx);
return nchunk;
}
// Setting and getting slices
int get_set_slice(void *buffer, int64_t buffersize, const int64_t *start, const int64_t *stop,
const int64_t *shape, b2nd_array_t *array, bool set_slice) {
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(start, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(stop, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
if (buffersize < 0) {
BLOSC_TRACE_ERROR("buffersize is < 0");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
uint8_t *buffer_b = buffer;
int8_t ndim = array->ndim;
if (!set_slice) {
// get_slice paths may touch only a subset of the destination buffer.
// Pre-initialize so unread regions are defined and deterministic.
memset(buffer_b, 0, (size_t)buffersize);
}
// 0-dim case
if (ndim == 0) {
if (set_slice) {
int32_t chunk_size = array->sc->typesize + BLOSC2_MAX_OVERHEAD;
uint8_t *chunk = malloc(chunk_size);
BLOSC_ERROR_NULL(chunk, BLOSC2_ERROR_MEMORY_ALLOC);
if (blosc2_compress_ctx(array->sc->cctx, buffer_b, array->sc->typesize, chunk, chunk_size) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
if (blosc2_schunk_update_chunk(array->sc, 0, chunk, false) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
} else {
if (blosc2_schunk_decompress_chunk(array->sc, 0, buffer_b, array->sc->typesize) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
return BLOSC2_ERROR_SUCCESS;
}
if (array->nitems == 0) {
return BLOSC2_ERROR_SUCCESS;
}
int64_t nelems_slice = 1;
for (int i = 0; i < array->ndim; ++i) {
if (stop[i] - start[i] > shape[i]) {
BLOSC_TRACE_ERROR("The buffer shape can not be smaller than the slice shape");
return BLOSC2_ERROR_INVALID_PARAM;
}
nelems_slice *= stop[i] - start[i];
}
int64_t slice_nbytes = nelems_slice * array->sc->typesize;
// Compute in 64 bits and range-check: the cast used to bind to extchunknitems
// alone, truncating it before the multiply and sizing buffers from the
// wrapped value while offsets kept using the full extent (see #795)
int64_t data_nbytes_64 = 0;
if (b2nd_mul_overflow_int64(array->extchunknitems, array->sc->typesize, &data_nbytes_64) ||
data_nbytes_64 > INT32_MAX) {
BLOSC_TRACE_ERROR("Chunk buffer size overflows the maximum of %d bytes", INT32_MAX);
return BLOSC2_ERROR_INVALID_PARAM;
}
int32_t data_nbytes = (int32_t) data_nbytes_64;
if (buffersize < slice_nbytes) {
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// Check for fast path for aligned slices with chunks and blocks (only 1 chunk is supported)
int64_t nchunk = nchunk_fastpath(array, start, stop, nelems_slice);
if (nchunk >= 0) {
if (set_slice) {
// Fast path for set. Let's set the chunk buffer straight into the array.
// Compress the chunk
int32_t chunk_nbytes = data_nbytes + BLOSC2_MAX_OVERHEAD;
uint8_t *chunk = malloc(chunk_nbytes);
BLOSC_ERROR_NULL(chunk, BLOSC2_ERROR_MEMORY_ALLOC);
int brc;
// Update current_chunk in case a prefilter is applied
array->sc->current_nchunk = nchunk;
brc = blosc2_compress_ctx(array->sc->cctx, buffer, data_nbytes, chunk, chunk_nbytes);
if (brc < 0) {
BLOSC_TRACE_ERROR("Blosc can not compress the data");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
int64_t brc_ = blosc2_schunk_update_chunk(array->sc, nchunk, chunk, false);
if (brc_ < 0) {
BLOSC_TRACE_ERROR("Blosc can not update the chunk");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
// We are done
return BLOSC2_ERROR_SUCCESS;
}
else {
// Fast path for get. Let's read the chunk straight into the buffer.
if (blosc2_schunk_decompress_chunk(array->sc, nchunk, buffer, (int32_t) slice_nbytes) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
return BLOSC2_ERROR_SUCCESS;
}
}
// Slow path for set and get
// Chunk-sized scratch, allocated lazily: the compact get path below never
// needs it, and for large chunks just the malloc/free pair costs
// O(chunksize) in page-table work (it is way above the malloc mmap
// threshold), which would dominate small reads.
uint8_t *data = NULL;
// Block-sized scratch for the compact get path (lazily allocated too).
uint8_t *block_data = NULL;
int32_t block_nbytes = (int32_t) (array->blocknitems * array->sc->typesize);
int64_t chunks_in_array[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunks_in_array[i] = array->extshape[i] / array->chunkshape[i];
}
int64_t chunks_in_array_strides[B2ND_MAX_DIM];
chunks_in_array_strides[ndim - 1] = 1;
for (int i = ndim - 2; i >= 0; --i) {
chunks_in_array_strides[i] = chunks_in_array_strides[i + 1] * chunks_in_array[i + 1];
}
int64_t blocks_in_chunk[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
blocks_in_chunk[i] = array->extchunkshape[i] / array->blockshape[i];
}
// Compute the number of chunks to update
int64_t update_start[B2ND_MAX_DIM];
int64_t update_shape[B2ND_MAX_DIM];
int64_t update_nchunks = 1;
for (int i = 0; i < ndim; ++i) {
int64_t pos = 0;
while (pos <= start[i]) {
pos += array->chunkshape[i];
}
update_start[i] = pos / array->chunkshape[i] - 1;
while (pos < stop[i]) {
pos += array->chunkshape[i];
}
update_shape[i] = pos / array->chunkshape[i] - update_start[i];
update_nchunks *= update_shape[i];
}
for (int64_t update_nchunk = 0; update_nchunk < update_nchunks; ++update_nchunk) {
int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim(ndim, update_shape, update_nchunk, nchunk_ndim);
for (int i = 0; i < ndim; ++i) {
nchunk_ndim[i] += update_start[i];
}
int64_t nchunk;
blosc2_multidim_to_unidim(nchunk_ndim, ndim, chunks_in_array_strides, &nchunk);
// Check if the chunk needs to be updated
int64_t chunk_start[B2ND_MAX_DIM] = {0};
int64_t chunk_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunk_start[i] = nchunk_ndim[i] * array->chunkshape[i];
chunk_stop[i] = chunk_start[i] + array->chunkshape[i];
if (chunk_stop[i] > array->shape[i]) {
chunk_stop[i] = array->shape[i];
}
}
bool chunk_empty = false;
for (int i = 0; i < ndim; ++i) {
chunk_empty |= (chunk_stop[i] <= start[i] || chunk_start[i] >= stop[i]);
}
if (chunk_empty) {
continue;
}
int32_t nblocks = (int32_t) array->extchunknitems / array->blocknitems;
// Compact get path state for this chunk (see below).
bool use_compact = false;
uint8_t *lazychunk = NULL;
bool lazychunk_needs_free = false;
int32_t lazychunk_cbytes = 0;
if (set_slice) {
if (data == NULL) {
data = malloc(data_nbytes);
BLOSC_ERROR_NULL(data, BLOSC2_ERROR_MEMORY_ALLOC);
}
// Check if all the chunk is going to be updated and avoid the decompression
bool decompress_chunk = false;
for (int i = 0; i < ndim; ++i) {
decompress_chunk |= (chunk_start[i] < start[i] || chunk_stop[i] > stop[i]);
}
if (decompress_chunk) {
int err = blosc2_schunk_decompress_chunk(array->sc, nchunk, data, data_nbytes);
if (err < 0) {
BLOSC_TRACE_ERROR("Error decompressing chunk");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
} else {
// Avoid writing non zero padding from previous chunk
memset(data, 0, data_nbytes);
}
} else {
bool *block_maskout = malloc(nblocks);
BLOSC_ERROR_NULL(block_maskout, BLOSC2_ERROR_MEMORY_ALLOC);
int32_t nblocks_needed = 0;
for (int nblock = 0; nblock < nblocks; ++nblock) {
int64_t nblock_ndim[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim(ndim, blocks_in_chunk, nblock, nblock_ndim);
// Check if the block needs to be updated
int64_t block_start[B2ND_MAX_DIM] = {0};
int64_t block_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
block_start[i] = nblock_ndim[i] * array->blockshape[i];
block_stop[i] = block_start[i] + array->blockshape[i];
block_start[i] += chunk_start[i];
block_stop[i] += chunk_start[i];
if (block_start[i] > chunk_stop[i]) {
block_start[i] = chunk_stop[i];
}
if (block_stop[i] > chunk_stop[i]) {
block_stop[i] = chunk_stop[i];
}
}
bool block_empty = false;
for (int i = 0; i < ndim; ++i) {
block_empty |= (block_stop[i] <= start[i] || block_start[i] >= stop[i]);
}
block_maskout[nblock] = block_empty ? true : false;
if (!block_empty) {
nblocks_needed++;
}
}
// Compact get path: when only a fraction of the chunk's blocks is
// needed, decompress just those blocks -- one at a time, straight
// from the (lazy)chunk -- instead of decompressing into a chunk-sized
// scratch. This keeps the working buffer at one block, so small reads
// from arrays with large chunks stay O(request), not O(chunksize).
// Larger requests keep the maskout path, which decompresses the needed
// blocks in parallel.
//
// The threshold depends on the storage backend. On disk, the maskout
// path reads the *whole* compressed chunk per call while the compact
// path only reads the touched blocks, so compact wins by 10-100x well
// past needed == nblocks/4 for typical data. In memory there is no
// I/O to save: for poorly-compressible data the maskout path's
// parallel decompression overtakes the compact path's serial one at
// around 2-4 needed blocks, so stay conservative there (1/16 keeps
// the possible loss window narrow while still capturing the common
// 1-2 block reads, where compact always wins or ties).
bool on_disk = array->sc->storage != NULL && array->sc->storage->urlpath != NULL;
int64_t frac = on_disk ? 4 : 16;
use_compact = ((int64_t) nblocks_needed * block_nbytes) * frac <= (int64_t) data_nbytes;
if (use_compact) {
lazychunk_cbytes = blosc2_schunk_get_lazychunk(array->sc, nchunk, &lazychunk,
&lazychunk_needs_free);
if (lazychunk_cbytes < 0) {
BLOSC_TRACE_ERROR("Error getting the lazy chunk");
free(block_maskout);
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
if (block_data == NULL) {
block_data = malloc(block_nbytes);
if (block_data == NULL) {
if (lazychunk_needs_free) {
free(lazychunk);
}
free(block_maskout);
BLOSC_ERROR(BLOSC2_ERROR_MEMORY_ALLOC);
}
}
} else {
if (data == NULL) {
data = malloc(data_nbytes);
if (data == NULL) {
free(block_maskout);
BLOSC_ERROR(BLOSC2_ERROR_MEMORY_ALLOC);
}
}
if (blosc2_set_maskout(array->sc->dctx, block_maskout, nblocks) != BLOSC2_ERROR_SUCCESS) {
BLOSC_TRACE_ERROR("Error setting the maskout");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
int err = blosc2_schunk_decompress_chunk(array->sc, nchunk, data, data_nbytes);
if (err < 0) {
BLOSC_TRACE_ERROR("Error decompressing chunk");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
free(block_maskout);
}
// Iterate over blocks
for (int nblock = 0; nblock < nblocks; ++nblock) {
int64_t nblock_ndim[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim(ndim, blocks_in_chunk, nblock, nblock_ndim);
// Check if the block needs to be updated
int64_t block_start[B2ND_MAX_DIM] = {0};
int64_t block_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
block_start[i] = nblock_ndim[i] * array->blockshape[i];
block_stop[i] = block_start[i] + array->blockshape[i];
block_start[i] += chunk_start[i];
block_stop[i] += chunk_start[i];
if (block_start[i] > chunk_stop[i]) {
block_start[i] = chunk_stop[i];
}
if (block_stop[i] > chunk_stop[i]) {
block_stop[i] = chunk_stop[i];
}
}
int64_t block_shape[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
block_shape[i] = block_stop[i] - block_start[i];
}
bool block_empty = false;
for (int i = 0; i < ndim; ++i) {
block_empty |= (block_stop[i] <= start[i] || block_start[i] >= stop[i]);
}
if (block_empty) {
continue;
}
// compute the start of the slice inside the block
int64_t slice_start[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
if (block_start[i] < start[i]) {
slice_start[i] = start[i] - block_start[i];
} else {
slice_start[i] = 0;
}
slice_start[i] += block_start[i];
}
int64_t slice_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
if (block_stop[i] > stop[i]) {
slice_stop[i] = block_shape[i] - (block_stop[i] - stop[i]);
} else {
slice_stop[i] = block_stop[i] - block_start[i];
}
slice_stop[i] += block_start[i];
}
int64_t slice_shape[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
slice_shape[i] = slice_stop[i] - slice_start[i];
}
uint8_t *src = &buffer_b[0];
int64_t src_start[B2ND_MAX_DIM] = {0};
int64_t src_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
src_start[i] = slice_start[i] - start[i];
src_stop[i] = slice_stop[i] - start[i];
}
uint8_t *dst;
if (use_compact) {
// Decompress just this block from the (lazy)chunk.
int grc = blosc2_getitem_bytes_ctx(array->sc->dctx, lazychunk, lazychunk_cbytes,
(int32_t) ((int64_t) nblock * block_nbytes), block_nbytes,
block_data, block_nbytes);
if (grc < 0) {
BLOSC_TRACE_ERROR("Error decompressing block");
if (lazychunk_needs_free) {
free(lazychunk);
}
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
dst = block_data;
} else {
dst = &data[nblock * array->blocknitems * array->sc->typesize];
}
int64_t dst_pad_shape[B2ND_MAX_DIM];
for (int i = 0; i < ndim; ++i) {
dst_pad_shape[i] = array->blockshape[i];
}
int64_t dst_start[B2ND_MAX_DIM] = {0};
int64_t dst_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
dst_start[i] = slice_start[i] - block_start[i];
dst_stop[i] = dst_start[i] + slice_shape[i];
}
if (set_slice) {
b2nd_copy_buffer2(ndim, array->sc->typesize,
src, shape, src_start, src_stop,
dst, dst_pad_shape, dst_start);
} else {
b2nd_copy_buffer2(ndim, array->sc->typesize,
dst, dst_pad_shape, dst_start, dst_stop,
src, shape, src_start);
}
}
if (use_compact && lazychunk_needs_free) {
free(lazychunk);
}
if (set_slice) {
// Recompress the data
int32_t chunk_nbytes = data_nbytes + BLOSC2_MAX_OVERHEAD;
uint8_t *chunk = malloc(chunk_nbytes);
BLOSC_ERROR_NULL(chunk, BLOSC2_ERROR_MEMORY_ALLOC);
int brc;
// Update current_chunk in case a prefilter is applied
array->sc->current_nchunk = nchunk;
brc = blosc2_compress_ctx(array->sc->cctx, data, data_nbytes, chunk, chunk_nbytes);
if (brc < 0) {
BLOSC_TRACE_ERROR("Blosc can not compress the data");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
int64_t brc_ = blosc2_schunk_update_chunk(array->sc, nchunk, chunk, false);
if (brc_ < 0) {
BLOSC_TRACE_ERROR("Blosc can not update the chunk");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
}
free(data);
free(block_data);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_get_slice_cbuffer(const b2nd_array_t *array, const int64_t *start, const int64_t *stop,
void *buffer, const int64_t *buffershape, int64_t buffersize) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(start, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(stop, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffershape, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) array));
BLOSC_ERROR(get_set_slice(buffer, buffersize, start, stop, buffershape, (b2nd_array_t *)array, false));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_set_slice_cbuffer(const void *buffer, const int64_t *buffershape, int64_t buffersize,
const int64_t *start, const int64_t *stop,
b2nd_array_t *array) {
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(start, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(stop, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale(array));
// A slice write can span several chunks, each updated with its own
// blosc2_schunk_update_chunk() call; hold the exclusive frame lock across
// all of them (as b2nd_resize() already does for its metalayer+chunks
// sequence) so locked handles -- including plain readers -- see the whole
// slice atomically instead of a chunk-wise interleave with other writers
// (plans/todo-locking-swmr.md item 2 / python-blosc2's
// todo/locking-mwmr.md item 2). A no-op for unlocked handles.
blosc2_frame_s *frame = (blosc2_frame_s *) array->sc->frame;
BLOSC_ERROR(frame_lock(frame, true));
int rc = get_set_slice((void*)buffer, buffersize, start, stop, (int64_t *)buffershape, array, true);
frame_unlock(frame);
BLOSC_ERROR(rc);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_get_slice(b2nd_context_t *ctx, b2nd_array_t **array, const b2nd_array_t *src, const int64_t *start,
const int64_t *stop) {
BLOSC_ERROR_NULL(src, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(start, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(stop, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) src));
ctx->ndim = src->ndim;
for (int i = 0; i < src->ndim; ++i) {
ctx->shape[i] = stop[i] - start[i];
}
// Add data
BLOSC_ERROR(b2nd_empty(ctx, array));
if ((*array)->nitems == 0) {
return BLOSC2_ERROR_SUCCESS;
}
int8_t ndim = (*array)->ndim;
int64_t chunks_in_array[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunks_in_array[i] = (*array)->extshape[i] / (*array)->chunkshape[i];
}
int64_t nchunks = (*array)->sc->nchunks;
for (int64_t nchunk = 0; nchunk < nchunks; ++nchunk) {
int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim(ndim, chunks_in_array, nchunk, nchunk_ndim);
// Check if the chunk needs to be updated
int64_t chunk_start[B2ND_MAX_DIM] = {0};
int64_t chunk_stop[B2ND_MAX_DIM] = {0};
int64_t chunk_shape[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunk_start[i] = nchunk_ndim[i] * (*array)->chunkshape[i];
chunk_stop[i] = chunk_start[i] + (*array)->chunkshape[i];
if (chunk_stop[i] > (*array)->shape[i]) {
chunk_stop[i] = (*array)->shape[i];
}
chunk_shape[i] = chunk_stop[i] - chunk_start[i];
}
int64_t src_start[B2ND_MAX_DIM] = {0};
int64_t src_stop[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
src_start[i] = chunk_start[i] + start[i];
src_stop[i] = chunk_stop[i] + start[i];
}
if (ctx->b2_storage->cparams->typesize <= 0) {
BLOSC_TRACE_ERROR("Invalid typesize");
return BLOSC2_ERROR_INVALID_PARAM;
}
size_t buffersize = (size_t)ctx->b2_storage->cparams->typesize;
for (int i = 0; i < ndim; ++i) {
if (chunk_shape[i] < 0) {
BLOSC_TRACE_ERROR("Invalid chunk shape");
return BLOSC2_ERROR_INVALID_PARAM;
}
if (b2nd_mul_overflow_size_t(buffersize, (size_t)chunk_shape[i], &buffersize)) {
BLOSC_TRACE_ERROR("Buffer size overflows size limits");
return BLOSC2_ERROR_INVALID_PARAM;
}
}
if (buffersize == 0 || buffersize > (size_t)INT64_MAX) {
BLOSC_TRACE_ERROR("Buffer size is out of range");
return BLOSC2_ERROR_INVALID_PARAM;
}
uint8_t *buffer = malloc(buffersize);
if (buffer == NULL) {
BLOSC_ERROR(BLOSC2_ERROR_MEMORY_ALLOC);
}
BLOSC_ERROR(b2nd_get_slice_cbuffer(src, src_start, src_stop, buffer, chunk_shape,
(int64_t)buffersize));
BLOSC_ERROR(b2nd_set_slice_cbuffer(buffer, chunk_shape, (int64_t)buffersize, chunk_start,
chunk_stop, *array));
free(buffer);
}
return BLOSC2_ERROR_SUCCESS;
}
/**
* @brief Return a view of a b2nd array.
*
* @param array The memory pointer of the array which will be viewed.
* @param view The memory pointer where the view will be created.
* @param ctx1 The b2nd context for the new array, containing new shape and other metadata.
*
* @return An error code.
*
* @note This doesn't support slices of arrays and is only useful for adding (or removing) dimensions.
*
*/
int view_new(const b2nd_array_t *array, b2nd_array_t **view, b2nd_context_t *ctx1) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(view, BLOSC2_ERROR_NULL_POINTER);
// The view is not contiguous (points to the original contiguous cframe which has different shape)
// so we set contiguous to false, which forces a copy when calling to_cframe
ctx1->b2_storage->contiguous = false;
/* Fill view with zeros */
BLOSC_ERROR(b2nd_zeros(ctx1, view));
// Free the chunks in base array
for (int i = 0; i < (*view)->sc->nchunks; i++) {
free((*view)->sc->data[i]);
}
free((*view)->sc->data);
(*view)->sc->view = true;
(*view)->sc->data = array->sc->data; // point view to the same data
(*view)->sc->frame = array->sc->frame; // if original array is contiguous, point to frame
(*view)->sc->nvlmetalayers = array->sc->nvlmetalayers; //
for (int i = 0; i< array->sc->nvlmetalayers; i++) {
(*view)->sc->vlmetalayers[i] = array->sc->vlmetalayers[i]; // add ptrs to vlmetalayers
}
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_expand_dims(const b2nd_array_t *array, b2nd_array_t **view, const bool *axis, const uint8_t final_dims) {
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) array));
for (int i = 0; i < array->sc->nmetalayers; ++i) {
if (strcmp(array->sc->metalayers[i]->name, "b2nd") != 0) {
BLOSC_TRACE_ERROR("Cannot expand dimensions of an array with non-b2nd metalayers");
return BLOSC2_ERROR_INVALID_PARAM;
}
}
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(view, BLOSC2_ERROR_NULL_POINTER);
uint8_t old_idx = 0;
int64_t newshape[B2ND_MAX_DIM];
int32_t newchunkshape[B2ND_MAX_DIM];
int32_t newblockshape[B2ND_MAX_DIM];
for (int i = 0; i < final_dims; ++i) {
if (axis[i] == true) {
newshape[i] = 1;
newchunkshape[i] = 1;
newblockshape[i] = 1;
}
else {
if (old_idx == array->ndim) {
BLOSC_TRACE_ERROR("Error in axis list: original array has fewer dimensions than the axis list implies!");
return BLOSC2_ERROR_INVALID_PARAM;
}
newshape[i] = array->shape[old_idx];
newchunkshape[i] = array->chunkshape[old_idx];
newblockshape[i] = array->blockshape[old_idx];
old_idx++;
}
}
//views only deal with cparams/dparams; storage is always in-memory (ephemeral).
blosc2_cparams cparams = *(array->sc->storage->cparams);
blosc2_dparams dparams = *(array->sc->storage->dparams);
blosc2_storage b2_storage1 = {.cparams=&cparams, .dparams=&dparams};
b2nd_context_t *ctx1 = b2nd_create_ctx(&b2_storage1, final_dims, newshape,
newchunkshape, newblockshape, array->dtype,
array->dtype_format, NULL, 0);
view_new(array, view, ctx1);
b2nd_free_ctx(ctx1);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_squeeze(b2nd_array_t *array, b2nd_array_t **view) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(view, BLOSC2_ERROR_NULL_POINTER);
bool index[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
if (array->shape[i] != 1) {
index[i] = false;
} else {
index[i] = true;
}
}
BLOSC_ERROR(b2nd_squeeze_index(array, view, index));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_squeeze_index(b2nd_array_t *array, b2nd_array_t **view, const bool *index) {
BLOSC_ERROR(refresh_if_stale(array));
for (int i = 0; i < array->sc->nmetalayers; ++i) {
if (strcmp(array->sc->metalayers[i]->name, "b2nd") != 0) {
BLOSC_TRACE_ERROR("Cannot squeeze dimensions of an array with non-b2nd metalayers");
return BLOSC2_ERROR_INVALID_PARAM;
}
}
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(view, BLOSC2_ERROR_NULL_POINTER);
uint8_t nones = 0;
int64_t newshape[B2ND_MAX_DIM];
int32_t newchunkshape[B2ND_MAX_DIM];
int32_t newblockshape[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
if (index[i] == true) {
if (array->shape[i] != 1) {
BLOSC_ERROR(BLOSC2_ERROR_INVALID_INDEX);
}
} else {
newshape[nones] = array->shape[i];
newchunkshape[nones] = array->chunkshape[i];
newblockshape[nones] = array->blockshape[i];
nones += 1;
}
}
//views only deal with cparams/dparams; storage is always in-memory (ephemeral).
blosc2_cparams cparams = *(array->sc->storage->cparams);
blosc2_dparams dparams = *(array->sc->storage->dparams);
blosc2_storage b2_storage1 = {.cparams=&cparams, .dparams=&dparams};
b2nd_context_t *ctx1 = b2nd_create_ctx(&b2_storage1, nones, newshape,
newchunkshape, newblockshape, array->dtype,
array->dtype_format, NULL, 0);
view_new(array, view, ctx1);
b2nd_free_ctx(ctx1);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_copy(b2nd_context_t *ctx, const b2nd_array_t *src, b2nd_array_t **array) {
BLOSC_ERROR_NULL(src, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) src));
ctx->ndim = src->ndim;
for (int i = 0; i < src->ndim; ++i) {
ctx->shape[i] = src->shape[i];
}
bool equals = true;
for (int i = 0; i < src->ndim; ++i) {
if (src->chunkshape[i] != ctx->chunkshape[i]) {
equals = false;
break;
}
if (src->blockshape[i] != ctx->blockshape[i]) {
equals = false;
break;
}
}
if (equals) {
BLOSC_ERROR(array_without_schunk(ctx, array));
blosc2_schunk *new_sc = blosc2_schunk_copy(src->sc, ctx->b2_storage);
if (new_sc == NULL) {
return BLOSC2_ERROR_FAILURE;
}
(*array)->sc = new_sc;
} else {
int64_t start[B2ND_MAX_DIM] = {0};
int64_t stop[B2ND_MAX_DIM];
for (int i = 0; i < src->ndim; ++i) {
stop[i] = src->shape[i];
}
// Copy metalayers
b2nd_context_t params_meta;
memcpy(¶ms_meta, ctx, sizeof(params_meta));
int j = 0;
for (int i = 0; i < src->sc->nmetalayers; ++i) {
if (strcmp(src->sc->metalayers[i]->name, "b2nd") == 0) {
continue;
}
blosc2_metalayer *meta = ¶ms_meta.metalayers[j];
meta->name = src->sc->metalayers[i]->name;
meta->content = src->sc->metalayers[i]->content;
meta->content_len = src->sc->metalayers[i]->content_len;
j++;
}
params_meta.nmetalayers = j;
// Copy data
BLOSC_ERROR(b2nd_get_slice(¶ms_meta, array, src, start, stop));
// Copy vlmetayers
for (int i = 0; i < src->sc->nvlmetalayers; ++i) {
uint8_t *content;
int32_t content_len;
if (blosc2_vlmeta_get(src->sc, src->sc->vlmetalayers[i]->name, &content,
&content_len) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
BLOSC_ERROR(blosc2_vlmeta_add((*array)->sc, src->sc->vlmetalayers[i]->name, content, content_len,
(*array)->sc->storage->cparams));
free(content);
}
}
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_concatenate(b2nd_context_t *ctx, const b2nd_array_t *src1, const b2nd_array_t *src2,
int8_t axis, bool copy, b2nd_array_t **array) {
BLOSC_ERROR_NULL(src1, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(src2, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) src1));
BLOSC_ERROR(refresh_if_stale((b2nd_array_t *) src2));
// Validate the axis parameter
if (axis < 0 || axis >= src1->ndim) {
BLOSC_TRACE_ERROR("axis parameter is out of bounds: axis=%d, expected range=[0, %d)", axis, src1->ndim - 1);
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// typesize must be the same for both arrays
if (src1->sc->typesize != src2->sc->typesize) {
BLOSC_TRACE_ERROR("The two arrays must have the same typesize");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// Keep the src1 shape for later use
int64_t src1_shape[B2ND_MAX_DIM];
for (int i = 0; i < src1->ndim; ++i) {
src1_shape[i] = src1->shape[i];
}
// Support for 0-dim arrays is not implemented
if (src1->ndim == 0 || src2->ndim == 0) {
BLOSC_TRACE_ERROR("Concatenation of 0-dim arrays is not supported");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// Check that the shapes are compatible for concatenation
if (src1->ndim != src2->ndim) {
BLOSC_TRACE_ERROR("The two arrays must have the same number of dimensions");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// Compute the new shape
int64_t newshape[B2ND_MAX_DIM];
for (int8_t i = 0; i < src1->ndim; ++i) {
if (i == axis) {
newshape[i] = src1->shape[i] + src2->shape[i];
} else {
if (src1->shape[i] != src2->shape[i]) {
BLOSC_TRACE_ERROR("The two arrays must have the same shape in all dimensions except the concatenation axis");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
newshape[i] = src1->shape[i];
}
}
if (copy) {
BLOSC_ERROR(b2nd_copy(ctx, src1, array));
}
else {
*array = (b2nd_array_t *)src1;
}
// Extend the array, we don't need to specify the start in resize, as we are extending the shape from the end
BLOSC_ERROR(b2nd_resize(*array, newshape, NULL));
// Copy the data from the second array
int64_t start[B2ND_MAX_DIM];
int64_t stop[B2ND_MAX_DIM];
// Check if the chunk is aligned with dest chunks, and has the same blockshape
bool aligned = true;
for (int8_t i = 0; i < src2->ndim; ++i) {
if (src1->chunkshape[i] != src2->chunkshape[i] ||
src2->blockshape[i] != (*array)->blockshape[i] ||
(i == axis && (src1_shape[i]) % (*array)->chunkshape[i] != 0)
) {
aligned = false;
break;
}
}
// ...and get the chunk index in the dest array if aligned
int64_t chunks_in_array_strides[B2ND_MAX_DIM];
// Calculate strides for destination array
chunks_in_array_strides[(*array)->ndim - 1] = 1;
for (int i = (*array)->ndim - 2; i >= 0; --i) {
chunks_in_array_strides[i] = chunks_in_array_strides[i + 1] *
((*array)->extshape[i + 1] / (*array)->chunkshape[i + 1]);
}
// Copy chunk by chunk
void *buffer = malloc(src2->sc->typesize * src2->extchunknitems);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_MEMORY_ALLOC);
for (int64_t nchunk = 0; nchunk < src2->sc->nchunks; ++nchunk) {
// Get multidimensional chunk position
int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
int64_t chunkshape[B2ND_MAX_DIM] = {0};
for (int8_t i = 0; i < src2->ndim; ++i) {
chunkshape[i] = src2->chunkshape[i];
}
int64_t chunks_in_dim[B2ND_MAX_DIM] = {0};
for (int8_t i = 0; i < src2->ndim; ++i) {
chunks_in_dim[i] = src2->extshape[i] / src2->chunkshape[i];
}
blosc2_unidim_to_multidim(src2->ndim, chunks_in_dim, nchunk, nchunk_ndim);
if (aligned) {
// Get the uncompressed chunk buffer from the source array
bool needs_free = false;
uint8_t *chunk;
int32_t cbytes = blosc2_schunk_get_chunk(src2->sc, nchunk, &chunk, &needs_free);
if (cbytes < 0) {
BLOSC_TRACE_ERROR("Error getting chunk from source array");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
// Update the chunk in the destination array
// We need to free only if needs_free is true or copy is false
// bool needs_copy = !needs_free || copy;
// BLOSC_ERROR(blosc2_schunk_update_chunk((*array)->sc, nchunk_dest, chunk, needs_copy));
// if (needs_free && !copy) {
// free(chunk);
// }
// TODO: the above makes some tests to crash, so always force a copy; try to optimize this later
int64_t nchunk_dest = 0;
nchunk_ndim[axis] += src1_shape[axis] / (*array)->chunkshape[axis];
for ( int i =0; i< src2->ndim; i++) {
nchunk_dest += nchunk_ndim[i] * chunks_in_array_strides[i];
}
BLOSC_ERROR(blosc2_schunk_update_chunk((*array)->sc, nchunk_dest, chunk, true));
if (needs_free) {
free(chunk);
}
}
else {
// Set positions for each dimension
for (int8_t i = 0; i < src2->ndim; ++i) {
start[i] = nchunk_ndim[i] * src2->chunkshape[i];
stop[i] = start[i] + src2->chunkshape[i];
if (stop[i] > src2->shape[i]) {
stop[i] = src2->shape[i]; // Handle boundary chunks
}
}
// Load chunk into buffer
BLOSC_ERROR(b2nd_get_slice_cbuffer(src2, start, stop, buffer, chunkshape, src2->sc->chunksize));
// Apply chunk offset only for concatenation axis
start[axis] += src1_shape[axis];
stop[axis] += src1_shape[axis];
// Copy the chunk to the correct position
BLOSC_ERROR(b2nd_set_slice_cbuffer(buffer, chunkshape,
src2->sc->typesize * src2->extchunknitems,
start, stop, *array));
}
}
free(buffer);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_save(const b2nd_array_t *array, char *urlpath) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(urlpath, BLOSC2_ERROR_NULL_POINTER);
b2nd_array_t *tmp;
blosc2_storage b2_storage = BLOSC2_STORAGE_DEFAULTS;
b2nd_context_t params = {.b2_storage=&b2_storage};
b2_storage.urlpath = urlpath;
b2_storage.contiguous = array->sc->storage->contiguous;
for (int i = 0; i < array->ndim; ++i) {
params.chunkshape[i] = array->chunkshape[i];
params.blockshape[i] = array->blockshape[i];
}
BLOSC_ERROR(b2nd_copy(¶ms, array, &tmp));
BLOSC_ERROR(b2nd_free(tmp));
return BLOSC2_ERROR_SUCCESS;
}
int64_t b2nd_save_append(const b2nd_array_t *array, const char *urlpath) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
return blosc2_schunk_append_file(array->sc, urlpath);
}
int b2nd_print_meta(const b2nd_array_t *array) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
int8_t ndim;
int64_t shape[B2ND_MAX_DIM];
int32_t chunkshape[B2ND_MAX_DIM];
int32_t blockshape[B2ND_MAX_DIM];
char *dtype;
int8_t dtype_format;
uint8_t *smeta;
int32_t smeta_len;
if (blosc2_meta_get(array->sc, "b2nd", &smeta, &smeta_len) < 0) {
// Try with a caterva metalayer; we are meant to be backward compatible with it
if (blosc2_meta_get(array->sc, "caterva", &smeta, &smeta_len) < 0) {
BLOSC_ERROR(BLOSC2_ERROR_METALAYER_NOT_FOUND);
}
}
BLOSC_ERROR(b2nd_deserialize_meta(smeta, smeta_len, &ndim, shape, chunkshape, blockshape,
&dtype, &dtype_format));
free(smeta);
printf("b2nd metalayer parameters:\n Ndim: %d", ndim);
printf("\n shape: %" PRId64 "", shape[0]);
for (int i = 1; i < ndim; ++i) {
printf(", %" PRId64 "", shape[i]);
}
printf("\n chunkshape: %d", chunkshape[0]);
for (int i = 1; i < ndim; ++i) {
printf(", %d", chunkshape[i]);
}
if (dtype != NULL) {
printf("\n dtype: %s", dtype);
free(dtype);
}
printf("\n blockshape: %d", blockshape[0]);
for (int i = 1; i < ndim; ++i) {
printf(", %d", blockshape[i]);
}
printf("\n");
return BLOSC2_ERROR_SUCCESS;
}
int extend_shape(b2nd_array_t *array, const int64_t *new_shape, const int64_t *start) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(new_shape, BLOSC2_ERROR_NULL_POINTER);
int8_t ndim = array->ndim;
int64_t diffs_shape[B2ND_MAX_DIM];
int64_t diffs_sum = 0;
for (int i = 0; i < ndim; i++) {
diffs_shape[i] = new_shape[i] - array->shape[i];
diffs_sum += diffs_shape[i];
if (diffs_shape[i] < 0) {
BLOSC_TRACE_ERROR("The new shape must be greater than the old one");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if (array->shape[i] == INT64_MAX) {
BLOSC_TRACE_ERROR("Cannot extend array with shape[%d] = %" PRId64, i, (int64_t)INT64_MAX);
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
}
if (diffs_sum == 0) {
// Shapes are equal. Do nothing.
return BLOSC2_ERROR_SUCCESS;
}
int64_t old_nchunks = array->sc->nchunks;
// aux array to keep old shapes
b2nd_array_t *aux = malloc(sizeof(b2nd_array_t));
BLOSC_ERROR_NULL(aux, BLOSC2_ERROR_MEMORY_ALLOC);
aux->sc = NULL;
BLOSC_ERROR(update_shape(aux, ndim, array->shape, array->chunkshape, array->blockshape));
// SWMR ordering for growth: make the data (special chunks) exist first and
// publish the new shape in the metalayer last, so a concurrent reader never
// sees a shape that points at chunks that are not there yet.
BLOSC_ERROR(update_shape_struct(array, ndim, new_shape, array->chunkshape, array->blockshape));
int64_t nchunks = array->extnitems / array->chunknitems;
int64_t nchunks_;
int64_t nchunk_ndim[B2ND_MAX_DIM];
blosc2_cparams *cparams;
BLOSC_ERROR(blosc2_schunk_get_cparams(array->sc, &cparams));
void *chunk;
int64_t csize;
if (nchunks != old_nchunks) {
if (start == NULL) {
start = aux->shape;
}
int64_t chunks_in_array[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunks_in_array[i] = array->extshape[i] / array->chunkshape[i];
}
for (int64_t i = 0; i < nchunks; ++i) {
blosc2_unidim_to_multidim(ndim, chunks_in_array, i, nchunk_ndim);
for (int j = 0; j < ndim; ++j) {
if (start[j] <= (array->chunkshape[j] * nchunk_ndim[j])
&& (array->chunkshape[j] * nchunk_ndim[j]) < (start[j] + new_shape[j] - aux->shape[j])) {
chunk = malloc(BLOSC_EXTENDED_HEADER_LENGTH);
BLOSC_ERROR_NULL(chunk, BLOSC2_ERROR_MEMORY_ALLOC);
csize = blosc2_chunk_zeros(*cparams, array->sc->chunksize, chunk, BLOSC_EXTENDED_HEADER_LENGTH);
if (csize < 0) {
free(aux);
free(cparams);
BLOSC_TRACE_ERROR("Blosc error when creating a chunk");
return BLOSC2_ERROR_FAILURE;
}
nchunks_ = blosc2_schunk_insert_chunk(array->sc, i, chunk, false);
if (nchunks_ < 0) {
free(aux);
free(cparams);
BLOSC_TRACE_ERROR("Blosc error when inserting a chunk");
return BLOSC2_ERROR_FAILURE;
}
break;
}
}
}
}
free(aux);
free(cparams);
// Data is in place; now publish the grown shape (shape last, see above)
BLOSC_ERROR(publish_shape_meta(array));
return BLOSC2_ERROR_SUCCESS;
}
int shrink_shape(b2nd_array_t *array, const int64_t *new_shape, const int64_t *start) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(new_shape, BLOSC2_ERROR_NULL_POINTER);
int8_t ndim = array->ndim;
int64_t diffs_shape[B2ND_MAX_DIM];
int64_t diffs_sum = 0;
for (int i = 0; i < ndim; i++) {
diffs_shape[i] = new_shape[i] - array->shape[i];
diffs_sum += diffs_shape[i];
if (diffs_shape[i] > 0) {
BLOSC_TRACE_ERROR("The new shape must be smaller than the old one");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if (array->shape[i] == 0) {
continue;
}
}
if (diffs_sum == 0) {
// Shapes are equal. Do nothing.
return BLOSC2_ERROR_SUCCESS;
}
int64_t old_nchunks = array->sc->nchunks;
// aux array to keep old shapes
b2nd_array_t *aux = malloc(sizeof(b2nd_array_t));
BLOSC_ERROR_NULL(aux, BLOSC2_ERROR_MEMORY_ALLOC);
aux->sc = NULL;
BLOSC_ERROR(update_shape(aux, ndim, array->shape, array->chunkshape, array->blockshape));
BLOSC_ERROR(update_shape(array, ndim, new_shape, array->chunkshape, array->blockshape));
// Delete chunks if needed
int64_t chunks_in_array_old[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
chunks_in_array_old[i] = aux->extshape[i] / aux->chunkshape[i];
}
if (start == NULL) {
start = new_shape;
}
int64_t nchunk_ndim[B2ND_MAX_DIM] = {0};
int64_t nchunks_;
for (int i = (int) old_nchunks - 1; i >= 0; --i) {
blosc2_unidim_to_multidim(ndim, chunks_in_array_old, i, nchunk_ndim);
for (int j = 0; j < ndim; ++j) {
if (start[j] <= (array->chunkshape[j] * nchunk_ndim[j])
&& (array->chunkshape[j] * nchunk_ndim[j]) < (start[j] + aux->shape[j] - new_shape[j])) {
nchunks_ = blosc2_schunk_delete_chunk(array->sc, i);
if (nchunks_ < 0) {
free(aux);
BLOSC_TRACE_ERROR("Blosc error when deleting a chunk");
return BLOSC2_ERROR_FAILURE;
}
break;
}
}
}
free(aux);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_resize(b2nd_array_t *array, const int64_t *new_shape,
const int64_t *start) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(new_shape, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale(array));
if (start != NULL) {
for (int i = 0; i < array->ndim; ++i) {
if (start[i] > array->shape[i]) {
BLOSC_TRACE_ERROR("`start` must be lower or equal than old array shape in all dims");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if ((new_shape[i] > array->shape[i] && start[i] != array->shape[i])
|| (new_shape[i] < array->shape[i]
&& (start[i] + array->shape[i] - new_shape[i]) != array->shape[i])) {
// Chunks cannot be cut unless they are in the last position
if (start[i] % array->chunkshape[i] != 0) {
BLOSC_TRACE_ERROR("If array end is not being modified "
"`start` must be a multiple of chunkshape in all dims");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
if ((new_shape[i] - array->shape[i]) % array->chunkshape[i] != 0) {
BLOSC_TRACE_ERROR("If array end is not being modified "
"`(new_shape - shape)` must be multiple of chunkshape in all dims");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
}
}
}
// Get shrunk shape
int64_t shrunk_shape[B2ND_MAX_DIM] = {0};
for (int i = 0; i < array->ndim; ++i) {
if (new_shape[i] <= array->shape[i]) {
shrunk_shape[i] = new_shape[i];
} else {
shrunk_shape[i] = array->shape[i];
}
}
// A resize is a sequence (metalayer update + chunk inserts/deletes); hold
// the exclusive frame lock across it so locked handles see it atomically.
// For unlocked handles this is a no-op and the data-first/shape-last
// ordering in extend_shape/shrink_shape keeps readers safe.
blosc2_frame_s *frame = (blosc2_frame_s *) array->sc->frame;
BLOSC_ERROR(frame_lock(frame, true));
int rc = shrink_shape(array, shrunk_shape, start);
if (rc >= 0) {
rc = extend_shape(array, new_shape, start);
}
frame_unlock(frame);
BLOSC_ERROR(rc);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_insert(b2nd_array_t *array, const void *buffer, int64_t buffersize,
int8_t axis, int64_t insert_start) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale(array));
if (axis >= array->ndim) {
BLOSC_TRACE_ERROR("`axis` cannot be greater than the number of dimensions");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
int64_t axis_size = array->sc->typesize;
int64_t buffershape[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
if (i != axis) {
axis_size *= array->shape[i];
buffershape[i] = array->shape[i];
}
}
if (buffersize % axis_size != 0) {
BLOSC_TRACE_ERROR("`buffersize` must be multiple of the array");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
int64_t newshape[B2ND_MAX_DIM];
memcpy(newshape, array->shape, array->ndim * sizeof(int64_t));
newshape[axis] += buffersize / axis_size;
buffershape[axis] = newshape[axis] - array->shape[axis];
int64_t start[B2ND_MAX_DIM] = {0};
start[axis] = insert_start;
if (insert_start == array->shape[axis]) {
BLOSC_ERROR(b2nd_resize(array, newshape, NULL));
} else {
BLOSC_ERROR(b2nd_resize(array, newshape, start));
}
int64_t stop[B2ND_MAX_DIM];
memcpy(stop, array->shape, sizeof(int64_t) * array->ndim);
stop[axis] = start[axis] + buffershape[axis];
BLOSC_ERROR(b2nd_set_slice_cbuffer(buffer, buffershape, buffersize, start, stop, array));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_append(b2nd_array_t *array, const void *buffer, int64_t buffersize,
int8_t axis) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(buffer, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale(array));
int32_t chunksize = array->sc->chunksize;
int64_t nchunks_append = buffersize / chunksize;
// Check whether chunkshape and blockshape are compatible with accelerated path.
// Essentially, we are checking whether the buffer is a multiple of the chunksize
// and that the chunkshape and blockshape are the same, except for the first axis.
// Also, axis needs to be the first one.
bool compat_chunks_blocks = true;
for (int i = 1; i < array->ndim; ++i) {
if (array->chunkshape[i] != array->blockshape[i]) {
compat_chunks_blocks = false;
break;
}
}
if (axis > 0) {
compat_chunks_blocks = false;
}
// General case where a buffer has a different size than the chunksize
if (!compat_chunks_blocks || buffersize % chunksize != 0 || nchunks_append != 1) {
BLOSC_ERROR(b2nd_insert(array, buffer, buffersize, axis, array->shape[axis]));
return BLOSC2_ERROR_SUCCESS;
}
// Accelerated path for buffers that are of the same size as the chunksize
// printf("accelerated path\n");
// Append the buffer to the underlying schunk. This is very fast, as
// it doesn't need to do internal partitioning.
BLOSC_ERROR(blosc2_schunk_append_buffer(array->sc, (void*)buffer, buffersize));
// Finally, resize the array
int64_t newshape[B2ND_MAX_DIM];
memcpy(newshape, array->shape, array->ndim * sizeof(int64_t));
newshape[axis] += nchunks_append * array->chunkshape[axis];
BLOSC_ERROR(b2nd_resize(array, newshape, NULL));
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_delete(b2nd_array_t *array, const int8_t axis,
int64_t delete_start, int64_t delete_len) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale(array));
if (axis >= array->ndim) {
BLOSC_TRACE_ERROR("axis cannot be greater than the number of dimensions");
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
int64_t newshape[B2ND_MAX_DIM];
memcpy(newshape, array->shape, array->ndim * sizeof(int64_t));
newshape[axis] -= delete_len;
int64_t start[B2ND_MAX_DIM] = {0};
start[axis] = delete_start;
if (delete_start == (array->shape[axis] - delete_len)) {
BLOSC_ERROR(b2nd_resize(array, newshape, NULL));
} else {
BLOSC_ERROR(b2nd_resize(array, newshape, start));
}
return BLOSC2_ERROR_SUCCESS;
}
// Indexing
typedef struct {
int64_t value;
int64_t index;
} b2nd_selection_t;
int compare_selection(const void *a, const void *b) {
int res = (int) (((b2nd_selection_t *) a)->value - ((b2nd_selection_t *) b)->value);
// In case values are equal, sort by index
if (res == 0) {
res = (int) (((b2nd_selection_t *) a)->index - ((b2nd_selection_t *) b)->index);
}
return res;
}
int copy_block_buffer_data(b2nd_array_t *array,
int8_t ndim,
int64_t *block_selection_size,
b2nd_selection_t **chunk_selection,
b2nd_selection_t **p_block_selection_0,
b2nd_selection_t **p_block_selection_1,
uint8_t *block,
uint8_t *buffer,
int64_t *buffershape,
int64_t *bufferstrides,
bool get) {
p_block_selection_0[ndim] = chunk_selection[ndim];
p_block_selection_1[ndim] = chunk_selection[ndim];
while (p_block_selection_1[ndim] - p_block_selection_0[ndim] < block_selection_size[ndim]) {
if (ndim == array->ndim - 1) {
// --- leaf: batch consecutive innermost-dimension elements ---------
// Compute start position of the first element in the batch.
int64_t index_in_block_n[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
index_in_block_n[i] = p_block_selection_1[i]->value % array->chunkshape[i] % array->blockshape[i];
}
int64_t start_block = 0;
for (int i = 0; i < array->ndim; ++i) {
start_block += index_in_block_n[i] * array->item_block_strides[i];
}
int64_t index_in_buffer_n[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
index_in_buffer_n[i] = p_block_selection_1[i]->index;
}
int64_t start_buffer = 0;
for (int i = 0; i < array->ndim; ++i) {
start_buffer += index_in_buffer_n[i] * bufferstrides[i];
}
int64_t count = 1;
int32_t typesize = array->sc->typesize;
p_block_selection_1[ndim]++;
while (p_block_selection_1[ndim] - p_block_selection_0[ndim] < block_selection_size[ndim]) {
b2nd_selection_t *prev = p_block_selection_1[ndim] - 1;
b2nd_selection_t *curr = p_block_selection_1[ndim];
// Check if this element is adjacent to the previous one in both
// the block buffer (value) and the output buffer (index).
if (curr->value == prev->value + 1 && curr->index == prev->index + 1) {
count++;
} else {
// Flush the current batch.
if (get) {
memcpy(&buffer[start_buffer * typesize], &block[start_block * typesize], (size_t)(count * typesize));
} else {
memcpy(&block[start_block * typesize], &buffer[start_buffer * typesize], (size_t)(count * typesize));
}
// Recompute start position for the new batch.
// Only the innermost dimension advances here; outer dimensions
// must keep using their current recursive selection pointers.
for (int i = 0; i < array->ndim; ++i) {
if (i == ndim) {
index_in_block_n[i] = curr->value % array->chunkshape[i] % array->blockshape[i];
index_in_buffer_n[i] = curr->index;
} else {
index_in_block_n[i] = p_block_selection_1[i]->value % array->chunkshape[i] % array->blockshape[i];
index_in_buffer_n[i] = p_block_selection_1[i]->index;
}
}
start_block = 0;
for (int i = 0; i < array->ndim; ++i) {
start_block += index_in_block_n[i] * array->item_block_strides[i];
}
start_buffer = 0;
for (int i = 0; i < array->ndim; ++i) {
start_buffer += index_in_buffer_n[i] * bufferstrides[i];
}
count = 1;
}
p_block_selection_1[ndim]++;
}
// Flush the last batch.
if (get) {
memcpy(&buffer[start_buffer * typesize], &block[start_block * typesize], (size_t)(count * typesize));
} else {
memcpy(&block[start_block * typesize], &buffer[start_buffer * typesize], (size_t)(count * typesize));
}
} else {
BLOSC_ERROR(copy_block_buffer_data(array, (int8_t) (ndim + 1), block_selection_size,
chunk_selection,
p_block_selection_0, p_block_selection_1, block,
buffer, buffershape, bufferstrides, get)
);
p_block_selection_1[ndim]++;
}
}
return BLOSC2_ERROR_SUCCESS;
}
int iter_block_copy(b2nd_array_t *array, int8_t ndim,
int64_t *chunk_selection_size,
b2nd_selection_t **ordered_selection,
b2nd_selection_t **chunk_selection_0,
b2nd_selection_t **chunk_selection_1,
uint8_t *data,
uint8_t *buffer,
int64_t *buffershape,
int64_t *bufferstrides,
bool get) {
chunk_selection_0[ndim] = ordered_selection[ndim];
chunk_selection_1[ndim] = ordered_selection[ndim];
while (chunk_selection_1[ndim] - ordered_selection[ndim] < chunk_selection_size[ndim]) {
int64_t block_index_ndim = ((*chunk_selection_1[ndim]).value % array->chunkshape[ndim]) / array->blockshape[ndim];
while (chunk_selection_1[ndim] - ordered_selection[ndim] < chunk_selection_size[ndim] &&
block_index_ndim == ((*chunk_selection_1[ndim]).value % array->chunkshape[ndim]) / array->blockshape[ndim]) {
chunk_selection_1[ndim]++;
}
if (ndim == array->ndim - 1) {
int64_t block_chunk_strides[B2ND_MAX_DIM];
block_chunk_strides[array->ndim - 1] = 1;
for (int i = array->ndim - 2; i >= 0; --i) {
block_chunk_strides[i] = block_chunk_strides[i + 1] * (array->extchunkshape[i + 1] / array->blockshape[i + 1]);
}
int64_t block_index[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
block_index[i] = ((*chunk_selection_0[i]).value % array->chunkshape[i]) / array->blockshape[i];
}
int64_t nblock = 0;
for (int i = 0; i < array->ndim; ++i) {
nblock += block_index[i] * block_chunk_strides[i];
}
b2nd_selection_t **p_block_selection_0 = malloc(array->ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(p_block_selection_0, BLOSC2_ERROR_MEMORY_ALLOC);
b2nd_selection_t **p_block_selection_1 = malloc(array->ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(p_block_selection_1, BLOSC2_ERROR_MEMORY_ALLOC);
int64_t *block_selection_size = malloc(array->ndim * sizeof(int64_t));
BLOSC_ERROR_NULL(block_selection_size, BLOSC2_ERROR_MEMORY_ALLOC);
for (int i = 0; i < array->ndim; ++i) {
block_selection_size[i] = chunk_selection_1[i] - chunk_selection_0[i];
}
BLOSC_ERROR(copy_block_buffer_data(array,
(int8_t) 0,
block_selection_size,
chunk_selection_0,
p_block_selection_0,
p_block_selection_1,
&data[nblock * array->blocknitems * array->sc->typesize],
buffer,
buffershape,
bufferstrides,
get)
);
free(p_block_selection_0);
free(p_block_selection_1);
free(block_selection_size);
} else {
BLOSC_ERROR(iter_block_copy(array, (int8_t) (ndim + 1), chunk_selection_size,
ordered_selection, chunk_selection_0, chunk_selection_1,
data, buffer, buffershape, bufferstrides, get)
);
}
chunk_selection_0[ndim] = chunk_selection_1[ndim];
}
return BLOSC2_ERROR_SUCCESS;
}
int iter_block_maskout(b2nd_array_t *array, int8_t ndim,
int64_t *sel_block_size,
b2nd_selection_t **o_selection,
b2nd_selection_t **p_o_sel_block_0,
b2nd_selection_t **p_o_sel_block_1,
bool *maskout) {
p_o_sel_block_0[ndim] = o_selection[ndim];
p_o_sel_block_1[ndim] = o_selection[ndim];
while (p_o_sel_block_1[ndim] - o_selection[ndim] < sel_block_size[ndim]) {
int64_t block_index_ndim = ((*p_o_sel_block_1[ndim]).value % array->chunkshape[ndim]) / array->blockshape[ndim];
while (p_o_sel_block_1[ndim] - o_selection[ndim] < sel_block_size[ndim] &&
block_index_ndim == ((*p_o_sel_block_1[ndim]).value % array->chunkshape[ndim]) / array->blockshape[ndim]) {
p_o_sel_block_1[ndim]++;
}
if (ndim == array->ndim - 1) {
int64_t block_chunk_strides[B2ND_MAX_DIM];
block_chunk_strides[array->ndim - 1] = 1;
for (int i = array->ndim - 2; i >= 0; --i) {
block_chunk_strides[i] = block_chunk_strides[i + 1] * (array->extchunkshape[i + 1] / array->blockshape[i + 1]);
}
int64_t block_index[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
block_index[i] = ((*p_o_sel_block_0[i]).value % array->chunkshape[i]) / array->blockshape[i];
}
int64_t nblock = 0;
for (int i = 0; i < array->ndim; ++i) {
nblock += block_index[i] * block_chunk_strides[i];
}
maskout[nblock] = false;
} else {
BLOSC_ERROR(iter_block_maskout(array, (int8_t) (ndim + 1), sel_block_size,
o_selection, p_o_sel_block_0, p_o_sel_block_1,
maskout)
);
}
p_o_sel_block_0[ndim] = p_o_sel_block_1[ndim];
}
return BLOSC2_ERROR_SUCCESS;
}
int iter_chunk(b2nd_array_t *array, int8_t ndim,
int64_t *selection_size,
b2nd_selection_t **ordered_selection,
b2nd_selection_t **p_ordered_selection_0,
b2nd_selection_t **p_ordered_selection_1,
uint8_t *buffer,
int64_t *buffershape,
int64_t *bufferstrides,
bool get,
uint8_t *chunk_data,
int32_t chunk_data_nbytes) {
p_ordered_selection_0[ndim] = ordered_selection[ndim];
p_ordered_selection_1[ndim] = ordered_selection[ndim];
while (p_ordered_selection_1[ndim] - ordered_selection[ndim] < selection_size[ndim]) {
int64_t chunk_index_ndim = (*p_ordered_selection_1[ndim]).value / array->chunkshape[ndim];
while (p_ordered_selection_1[ndim] - ordered_selection[ndim] < selection_size[ndim] &&
chunk_index_ndim == (*p_ordered_selection_1[ndim]).value / array->chunkshape[ndim]) {
p_ordered_selection_1[ndim]++;
}
if (ndim == array->ndim - 1) {
int64_t chunk_array_strides[B2ND_MAX_DIM];
chunk_array_strides[array->ndim - 1] = 1;
for (int i = array->ndim - 2; i >= 0; --i) {
chunk_array_strides[i] = chunk_array_strides[i + 1] *
(array->extshape[i + 1] / array->chunkshape[i + 1]);
}
int64_t chunk_index[B2ND_MAX_DIM];
for (int i = 0; i < array->ndim; ++i) {
chunk_index[i] = (*p_ordered_selection_0[i]).value / array->chunkshape[i];
}
int64_t nchunk = 0;
for (int i = 0; i < array->ndim; ++i) {
nchunk += chunk_index[i] * chunk_array_strides[i];
}
int64_t nblocks = array->extchunknitems / array->blocknitems;
b2nd_selection_t **p_chunk_selection_0 = malloc(array->ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(p_chunk_selection_0, BLOSC2_ERROR_MEMORY_ALLOC);
b2nd_selection_t **p_chunk_selection_1 = malloc(array->ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(p_chunk_selection_1, BLOSC2_ERROR_MEMORY_ALLOC);
int64_t *chunk_selection_size = malloc(array->ndim * sizeof(int64_t));
BLOSC_ERROR_NULL(chunk_selection_size, BLOSC2_ERROR_MEMORY_ALLOC);
for (int i = 0; i < array->ndim; ++i) {
chunk_selection_size[i] = p_ordered_selection_1[i] - p_ordered_selection_0[i];
}
if (get) {
bool *maskout = calloc(nblocks, sizeof(bool));
for (int i = 0; i < nblocks; ++i) {
maskout[i] = true;
}
BLOSC_ERROR(iter_block_maskout(array, (int8_t) 0,
chunk_selection_size,
p_ordered_selection_0,
p_chunk_selection_0,
p_chunk_selection_1,
maskout));
if (blosc2_set_maskout(array->sc->dctx, maskout, (int) nblocks) !=
BLOSC2_ERROR_SUCCESS) {
BLOSC_TRACE_ERROR("Error setting the maskout");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
free(maskout);
}
int data_nbytes = chunk_data_nbytes;
uint8_t *data = chunk_data;
int err = blosc2_schunk_decompress_chunk(array->sc, nchunk, data, data_nbytes);
if (err < 0) {
BLOSC_TRACE_ERROR("Error decompressing chunk");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
BLOSC_ERROR(iter_block_copy(array, 0, chunk_selection_size,
p_ordered_selection_0, p_chunk_selection_0, p_chunk_selection_1,
data, buffer, buffershape, bufferstrides, get));
if (!get) {
int32_t chunk_size = data_nbytes + BLOSC_EXTENDED_HEADER_LENGTH;
uint8_t *chunk = malloc(chunk_size);
BLOSC_ERROR_NULL(chunk, BLOSC2_ERROR_MEMORY_ALLOC);
err = blosc2_compress_ctx(array->sc->cctx, data, data_nbytes, chunk, chunk_size);
if (err < 0) {
BLOSC_TRACE_ERROR("Error compressing data");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
err = (int) blosc2_schunk_update_chunk(array->sc, nchunk, chunk, false);
if (err < 0) {
BLOSC_TRACE_ERROR("Error updating chunk");
BLOSC_ERROR(BLOSC2_ERROR_FAILURE);
}
}
free(chunk_selection_size);
free(p_chunk_selection_0);
free(p_chunk_selection_1);
} else {
BLOSC_ERROR(iter_chunk(array, (int8_t) (ndim + 1), selection_size,
ordered_selection, p_ordered_selection_0, p_ordered_selection_1,
buffer, buffershape, bufferstrides, get, chunk_data, chunk_data_nbytes));
}
p_ordered_selection_0[ndim] = p_ordered_selection_1[ndim];
}
return BLOSC2_ERROR_SUCCESS;
}
int orthogonal_selection(b2nd_array_t *array, int64_t **selection, int64_t *selection_size, void *buffer,
int64_t *buffershape, int64_t buffersize, bool get) {
BLOSC_ERROR_NULL(array, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(selection, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR_NULL(selection_size, BLOSC2_ERROR_NULL_POINTER);
BLOSC_ERROR(refresh_if_stale(array));
int8_t ndim = array->ndim;
for (int i = 0; i < ndim; ++i) {
BLOSC_ERROR_NULL(selection[i], BLOSC2_ERROR_NULL_POINTER);
// Check that indexes are not larger than array shape
for (int j = 0; j < selection_size[i]; ++j) {
if (selection[i][j] > array->shape[i]) {
BLOSC_ERROR(BLOSC2_ERROR_INVALID_INDEX);
}
}
}
// Check buffer size
int64_t sel_size = array->sc->typesize;
for (int i = 0; i < ndim; ++i) {
sel_size *= selection_size[i];
}
if (sel_size < buffersize) {
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// Size of the reusable chunk buffer allocated further down. Computed here,
// before any allocation, so the error path has nothing to unwind: the cast
// used to truncate an int64 product into the malloc size while the copy
// offsets kept using the full extent (see #795).
int64_t chunk_data_nbytes_64 = 0;
if (b2nd_mul_overflow_int64(array->extchunknitems, array->sc->typesize, &chunk_data_nbytes_64) ||
chunk_data_nbytes_64 > INT32_MAX) {
BLOSC_TRACE_ERROR("Chunk buffer size overflows the maximum of %d bytes", INT32_MAX);
BLOSC_ERROR(BLOSC2_ERROR_INVALID_PARAM);
}
// Sort selections
b2nd_selection_t **ordered_selection = malloc(ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(ordered_selection, BLOSC2_ERROR_MEMORY_ALLOC);
for (int i = 0; i < ndim; ++i) {
ordered_selection[i] = malloc(selection_size[i] * sizeof(b2nd_selection_t));
for (int j = 0; j < selection_size[i]; ++j) {
ordered_selection[i][j].index = j;
ordered_selection[i][j].value = selection[i][j];
}
qsort(ordered_selection[i], selection_size[i], sizeof(b2nd_selection_t), compare_selection);
}
// Define pointers to iterate over ordered_selection data
b2nd_selection_t **p_ordered_selection_0 = malloc(ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(p_ordered_selection_0, BLOSC2_ERROR_MEMORY_ALLOC);
b2nd_selection_t **p_ordered_selection_1 = malloc(ndim * sizeof(b2nd_selection_t *));
BLOSC_ERROR_NULL(p_ordered_selection_1, BLOSC2_ERROR_MEMORY_ALLOC);
int64_t bufferstrides[B2ND_MAX_DIM];
bufferstrides[array->ndim - 1] = 1;
for (int i = array->ndim - 2; i >= 0; --i) {
bufferstrides[i] = bufferstrides[i + 1] * buffershape[i + 1];
}
// Pre-allocate a single chunk decompression buffer, reused for every
// chunk visited by iter_chunk, instead of malloc/free per chunk.
int32_t chunk_data_nbytes = (int32_t) chunk_data_nbytes_64;
uint8_t *chunk_data = malloc(chunk_data_nbytes);
BLOSC_ERROR_NULL(chunk_data, BLOSC2_ERROR_MEMORY_ALLOC);
BLOSC_ERROR(iter_chunk(array, 0,
selection_size, ordered_selection,
p_ordered_selection_0,
p_ordered_selection_1,
buffer, buffershape, bufferstrides, get,
chunk_data, chunk_data_nbytes));
free(chunk_data);
// Free allocated memory
free(p_ordered_selection_0);
free(p_ordered_selection_1);
for (int i = 0; i < ndim; ++i) {
free(ordered_selection[i]);
}
free(ordered_selection);
return BLOSC2_ERROR_SUCCESS;
}
int b2nd_get_orthogonal_selection(const b2nd_array_t *array, int64_t **selection, int64_t *selection_size, void *buffer,
int64_t *buffershape, int64_t buffersize) {
return orthogonal_selection((b2nd_array_t *)array, selection, selection_size, buffer, buffershape, buffersize, true);
}
int b2nd_set_orthogonal_selection(b2nd_array_t *array, int64_t **selection, int64_t *selection_size, const void *buffer,
int64_t *buffershape, int64_t buffersize) {
return orthogonal_selection(array, selection, selection_size, (void*)buffer, buffershape, buffersize, false);
}
b2nd_context_t *
b2nd_create_ctx(const blosc2_storage *b2_storage, int8_t ndim, const int64_t *shape, const int32_t *chunkshape,
const int32_t *blockshape, const char *dtype, int8_t dtype_format, const blosc2_metalayer *metalayers,
int32_t nmetalayers) {
if (ndim < 0 || ndim > B2ND_MAX_DIM) {
BLOSC_TRACE_ERROR("ndim must be in [0, %d]", B2ND_MAX_DIM);
return NULL;
}
if (ndim > 0) {
if (shape == NULL || chunkshape == NULL || blockshape == NULL) {
BLOSC_TRACE_ERROR("shape, chunkshape and blockshape cannot be NULL when ndim > 0");
return NULL;
}
if (validate_shape_chunkshape_blockshape(ndim, shape, chunkshape, blockshape) < 0) {
return NULL;
}
}
b2nd_context_t *ctx = malloc(sizeof(b2nd_context_t));
BLOSC_ERROR_NULL(ctx, NULL);
blosc2_storage *params_b2_storage = malloc(sizeof(blosc2_storage));
BLOSC_ERROR_NULL(params_b2_storage, NULL);
if (b2_storage == NULL) {
memcpy(params_b2_storage, &BLOSC2_STORAGE_DEFAULTS, sizeof(blosc2_storage));
}
else {
memcpy(params_b2_storage, b2_storage, sizeof(blosc2_storage));
}
blosc2_cparams *cparams = malloc(sizeof(blosc2_cparams));
BLOSC_ERROR_NULL(cparams, NULL);
// We need a copy of cparams mainly to be able to modify blocksize
if (params_b2_storage->cparams == NULL) {
memcpy(cparams, &BLOSC2_CPARAMS_DEFAULTS, sizeof(blosc2_cparams));
}
else {
memcpy(cparams, params_b2_storage->cparams, sizeof(blosc2_cparams));
}
if (dtype == NULL) {
// ctx->dtype = strdup(B2ND_DEFAULT_DTYPE);
char buf[16] = {0};
snprintf(buf, sizeof(buf), "|S%d", cparams->typesize);
ctx->dtype = strdup(buf);
}
else {
ctx->dtype = strdup(dtype);
}
ctx->dtype_format = dtype_format;
params_b2_storage->cparams = cparams;
ctx->b2_storage = params_b2_storage;
ctx->ndim = ndim;
int32_t blocknitems = 1;
bool blocksize_overflow = false;
for (int i = 0; i < ndim; i++) {
ctx->shape[i] = shape[i];
ctx->chunkshape[i] = chunkshape[i];
ctx->blockshape[i] = blockshape[i];
if (b2nd_mul_overflow_int32(blocknitems, ctx->blockshape[i], &blocknitems)) {
blocksize_overflow = true;
break;
}
}
// update_shape_struct() rejects this too, but only once an array is built out
// of the context; catching it here keeps cparams->blocksize from wrapping
if (blocksize_overflow ||
b2nd_mul_overflow_int32(blocknitems, cparams->typesize, &cparams->blocksize)) {
BLOSC_TRACE_ERROR("blockshape product overflows the maximum blocksize");
free(ctx->dtype);
free(cparams);
free(params_b2_storage);
free(ctx);
return NULL;
}
ctx->nmetalayers = nmetalayers;
for (int i = 0; i < nmetalayers; ++i) {
ctx->metalayers[i] = metalayers[i];
}
#if defined(HAVE_ZFP)
#include "blosc2/codecs-registry.h"
if ((ctx->b2_storage->cparams->compcode >= BLOSC_CODEC_ZFP_FIXED_ACCURACY) &&
(ctx->b2_storage->cparams->compcode <= BLOSC_CODEC_ZFP_FIXED_RATE)) {
for (int i = 0; i < BLOSC2_MAX_FILTERS; ++i) {
if ((ctx->b2_storage->cparams->filters[i] == BLOSC_SHUFFLE) ||
(ctx->b2_storage->cparams->filters[i] == BLOSC_BITSHUFFLE)) {
BLOSC_TRACE_ERROR("ZFP cannot be run in presence of SHUFFLE / BITSHUFFLE");
return NULL;
}
}
}
#endif /* HAVE_ZFP */
return ctx;
}
int b2nd_free_ctx(b2nd_context_t *ctx) {
ctx->b2_storage->cparams->schunk = NULL;
free(ctx->b2_storage->cparams);
free(ctx->b2_storage);
free(ctx->dtype);
free(ctx);
return BLOSC2_ERROR_SUCCESS;
}
Blosc-c-blosc2-d3bee7c/blosc/b2nd_utils.c 0000664 0000000 0000000 00000027367 15255167342 0020300 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
#include "b2nd.h"
#include
// copyNdim where N = {2-8} - specializations of copy loops to be used by b2nd_copy_buffer
// since we don't have c++ templates, substitute manual specializations for up to known B2ND_MAX_DIM (8)
// it's not pretty, but it substantially reduces overhead vs. the generic method
void copy8dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[7] * itemsize;
int64_t copy_start[7] = {0};
do {
do {
do {
do {
do {
do {
do {
int64_t src_copy_start = 0;
int64_t dst_copy_start = 0;
for (int j = 0; j < 7; ++j) {
src_copy_start += copy_start[j] * src_strides[j];
dst_copy_start += copy_start[j] * dst_strides[j];
}
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start[6];
} while (copy_start[6] < copy_shape[6]);
++copy_start[5];
copy_start[6] = 0;
} while (copy_start[5] < copy_shape[5]);
++copy_start[4];
copy_start[5] = 0;
} while (copy_start[4] < copy_shape[4]);
++copy_start[3];
copy_start[4] = 0;
} while (copy_start[3] < copy_shape[3]);
++copy_start[2];
copy_start[3] = 0;
} while (copy_start[2] < copy_shape[2]);
++copy_start[1];
copy_start[2] = 0;
} while (copy_start[1] < copy_shape[1]);
++copy_start[0];
copy_start[1] = 0;
} while (copy_start[0] < copy_shape[0]);
}
void copy7dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[6] * itemsize;
int64_t copy_start[6] = {0};
do {
do {
do {
do {
do {
do {
int64_t src_copy_start = 0;
int64_t dst_copy_start = 0;
for (int j = 0; j < 6; ++j) {
src_copy_start += copy_start[j] * src_strides[j];
dst_copy_start += copy_start[j] * dst_strides[j];
}
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start[5];
} while (copy_start[5] < copy_shape[5]);
++copy_start[4];
copy_start[5] = 0;
} while (copy_start[4] < copy_shape[4]);
++copy_start[3];
copy_start[4] = 0;
} while (copy_start[3] < copy_shape[3]);
++copy_start[2];
copy_start[3] = 0;
} while (copy_start[2] < copy_shape[2]);
++copy_start[1];
copy_start[2] = 0;
} while (copy_start[1] < copy_shape[1]);
++copy_start[0];
copy_start[1] = 0;
} while (copy_start[0] < copy_shape[0]);
}
void copy6dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[5] * itemsize;
int64_t copy_start[5] = {0};
do {
do {
do {
do {
do {
int64_t src_copy_start = 0;
int64_t dst_copy_start = 0;
for (int j = 0; j < 5; ++j) {
src_copy_start += copy_start[j] * src_strides[j];
dst_copy_start += copy_start[j] * dst_strides[j];
}
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start[4];
} while (copy_start[4] < copy_shape[4]);
++copy_start[3];
copy_start[4] = 0;
} while (copy_start[3] < copy_shape[3]);
++copy_start[2];
copy_start[3] = 0;
} while (copy_start[2] < copy_shape[2]);
++copy_start[1];
copy_start[2] = 0;
} while (copy_start[1] < copy_shape[1]);
++copy_start[0];
copy_start[1] = 0;
} while (copy_start[0] < copy_shape[0]);
}
void copy5dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[4] * itemsize;
int64_t copy_start[4] = {0};
do {
do {
do {
do {
int64_t src_copy_start = 0;
int64_t dst_copy_start = 0;
for (int j = 0; j < 4; ++j) {
src_copy_start += copy_start[j] * src_strides[j];
dst_copy_start += copy_start[j] * dst_strides[j];
}
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start[3];
} while (copy_start[3] < copy_shape[3]);
++copy_start[2];
copy_start[3] = 0;
} while (copy_start[2] < copy_shape[2]);
++copy_start[1];
copy_start[2] = 0;
} while (copy_start[1] < copy_shape[1]);
++copy_start[0];
copy_start[1] = 0;
} while (copy_start[0] < copy_shape[0]);
}
void copy4dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[3] * itemsize;
int64_t copy_start[3] = {0};
do {
do {
do {
int64_t src_copy_start = 0;
int64_t dst_copy_start = 0;
for (int j = 0; j < 3; ++j) {
src_copy_start += copy_start[j] * src_strides[j];
dst_copy_start += copy_start[j] * dst_strides[j];
}
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start[2];
} while (copy_start[2] < copy_shape[2]);
++copy_start[1];
copy_start[2] = 0;
} while (copy_start[1] < copy_shape[1]);
++copy_start[0];
copy_start[1] = 0;
} while (copy_start[0] < copy_shape[0]);
}
void copy3dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[2] * itemsize;
int64_t copy_start[2] = {0};
do {
do {
int64_t src_copy_start = 0;
int64_t dst_copy_start = 0;
for (int j = 0; j < 2; ++j) {
src_copy_start += copy_start[j] * src_strides[j];
dst_copy_start += copy_start[j] * dst_strides[j];
}
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start[1];
} while (copy_start[1] < copy_shape[1]);
++copy_start[0];
copy_start[1] = 0;
} while (copy_start[0] < copy_shape[0]);
}
void copy2dim(const int32_t itemsize,
const int64_t *copy_shape,
const uint8_t *bsrc, const int64_t *src_strides,
uint8_t *bdst, const int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[1] * itemsize;
int64_t copy_start = 0;
do {
int64_t src_copy_start = copy_start * src_strides[0];
int64_t dst_copy_start = copy_start * dst_strides[0];
memcpy(&bdst[dst_copy_start * itemsize], &bsrc[src_copy_start * itemsize], copy_nbytes);
++copy_start;
} while (copy_start < copy_shape[0]);
}
void copy_ndim_fallback(const int8_t ndim,
const int32_t itemsize,
int64_t *copy_shape,
const uint8_t *bsrc, int64_t *src_strides,
uint8_t *bdst, int64_t *dst_strides) {
int64_t copy_nbytes = copy_shape[ndim - 1] * itemsize;
int64_t number_of_copies = 1;
for (int i = 0; i < ndim - 1; ++i) {
number_of_copies *= copy_shape[i];
}
for (int ncopy = 0; ncopy < number_of_copies; ++ncopy) {
// Compute the start of the copy
int64_t copy_start[B2ND_MAX_DIM] = {0};
blosc2_unidim_to_multidim((int8_t) (ndim - 1), copy_shape, ncopy, copy_start);
// Translate this index to the src buffer
int64_t src_copy_start;
blosc2_multidim_to_unidim(copy_start, (int8_t) (ndim - 1), src_strides, &src_copy_start);
// Translate this index to the dst buffer
int64_t dst_copy_start;
blosc2_multidim_to_unidim(copy_start, (int8_t) (ndim - 1), dst_strides, &dst_copy_start);
// Perform the copy
memcpy(&bdst[dst_copy_start * itemsize],
&bsrc[src_copy_start * itemsize],
copy_nbytes);
}
}
int b2nd_copy_buffer2(int8_t ndim,
int32_t itemsize,
const void *src, const int64_t *src_pad_shape,
const int64_t *src_start, const int64_t *src_stop,
void *dst, const int64_t *dst_pad_shape,
const int64_t *dst_start) {
// Compute the shape of the copy
int64_t copy_shape[B2ND_MAX_DIM] = {0};
for (int i = 0; i < ndim; ++i) {
copy_shape[i] = src_stop[i] - src_start[i];
if (copy_shape[i] == 0) {
return BLOSC2_ERROR_SUCCESS;
}
}
// Compute the strides
int64_t src_strides[B2ND_MAX_DIM] = {0};
src_strides[ndim - 1] = 1;
for (int i = ndim - 2; i >= 0; --i) {
src_strides[i] = src_strides[i + 1] * src_pad_shape[i + 1];
}
int64_t dst_strides[B2ND_MAX_DIM] = {0};
dst_strides[ndim - 1] = 1;
for (int i = ndim - 2; i >= 0; --i) {
dst_strides[i] = dst_strides[i + 1] * dst_pad_shape[i + 1];
}
// Align the buffers removing unnecessary data
int64_t src_start_n;
blosc2_multidim_to_unidim(src_start, ndim, src_strides, &src_start_n);
uint8_t *bsrc = (uint8_t *) src;
bsrc = &bsrc[src_start_n * itemsize];
int64_t dst_start_n;
blosc2_multidim_to_unidim(dst_start, ndim, dst_strides, &dst_start_n);
uint8_t *bdst = (uint8_t *) dst;
bdst = &bdst[dst_start_n * itemsize];
switch (ndim) {
case 1:
memcpy(&bdst[0], &bsrc[0], copy_shape[0] * itemsize);
break;
case 2:
copy2dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
case 3:
copy3dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
case 4:
copy4dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
case 5:
copy5dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
case 6:
copy6dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
case 7:
copy7dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
case 8:
copy8dim(itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
default:
// guard against potential future increase to B2ND_MAX_DIM
copy_ndim_fallback(ndim, itemsize, copy_shape, bsrc, src_strides, bdst, dst_strides);
break;
}
return BLOSC2_ERROR_SUCCESS;
}
// Keep the old signature for API compatibility
int b2nd_copy_buffer(int8_t ndim,
uint8_t itemsize,
const void *src, const int64_t *src_pad_shape,
const int64_t *src_start, const int64_t *src_stop,
void *dst, const int64_t *dst_pad_shape,
const int64_t *dst_start) {
// Simply cast itemsize to int32_t and delegate
return b2nd_copy_buffer2(ndim, (int32_t)itemsize, src, src_pad_shape,
src_start, src_stop, dst, dst_pad_shape, dst_start);
}
Blosc-c-blosc2-d3bee7c/blosc/bitshuffle-altivec.c 0000664 0000000 0000000 00000054377 15255167342 0022014 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
/*********************************************************************
Bitshuffle - Filter for improving compression of typed binary data.
Author: Kiyoshi Masui
Website: https://github.com/kiyo-masui/bitshuffle
Note: Adapted for c-blosc by Francesc Alted
Altivec/VSX version by Jerome Kieffer.
See LICENSES/BITSHUFFLE.txt file for details about copyright and
rights to use.
**********************************************************************/
#include "bitshuffle-altivec.h"
#include "bitshuffle-generic.h"
#include
/* Make sure ALTIVEC is available for the compilation target and compiler.
vec_bperm requires POWER8 (ISA 2.07). VSX is needed for vec_xl/vec_xst.
Exclude __APPLE__ because some GCC versions falsely define __VSX__ on
powerpc-apple-darwin targets (GCC Bug #121696, cf. Eigen's workaround). */
#if defined(__ALTIVEC__) && defined(__VSX__) && defined(_ARCH_PWR8) && !defined(__APPLE__)
/* vec_bperm places its result in different elements depending on byte order.
On big-endian the result is in element 0; on little-endian it is in
element 4 (of a uint16_t vector). */
#if defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#define BLOSC_BPERM_RESULT_IDX 0
#else
#define BLOSC_BPERM_RESULT_IDX 4
#endif
#include "transpose-altivec.h"
#include
#include
/* The next is useful for debugging purposes */
#if 0
#include
#include
static void helper_print(__vector uint8_t v, char* txt){
printf("%s %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",txt,
v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8], v[9], v[10], v[11], v[12], v[13], v[14], v[15]);
}
#endif
static inline __vector uint8_t gen_save_mask(size_t offset){
__vector uint8_t mask;
size_t k;
for (k = 0; k < 16; k++)
mask[k] = (k> 3];
*oui16 = tmp[BLOSC_BPERM_RESULT_IDX];
}
}
count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size,
nbyte - nbyte % 16);
return count;
}
/* Transpose bits within elements. */
int64_t bshuf_trans_bit_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size) {
int64_t count;
CHECK_MULT_EIGHT(size);
void* tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
count = bshuf_trans_byte_elem_altivec(in, out, size, elem_size, tmp_buf);
CHECK_ERR(count);
// bshuf_trans_bit_byte_altivec / bitshuffle1_altivec
count = bshuf_trans_bit_byte_altivec(out, tmp_buf, size, elem_size);
CHECK_ERR(count);
count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size);
free(tmp_buf);
return count;
}
/* For data organized into a row for each bit (8 * elem_size rows), transpose
* the bytes. */
int64_t bshuf_trans_byte_bitrow_altivec(const void* in, void* out, const size_t size,
const size_t elem_size) {
static const __vector uint8_t epi8_low = (const __vector uint8_t) {
0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13,
0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17};
static const __vector uint8_t epi8_hi = (const __vector uint8_t) {
0x08, 0x18, 0x09, 0x19, 0x0a, 0x1a, 0x0b, 0x1b,
0x0c, 0x1c, 0x0d, 0x1d, 0x0e, 0x1e, 0x0f, 0x1f};
static const __vector uint8_t epi16_low = (const __vector uint8_t) {
0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17};
static const __vector uint8_t epi16_hi = (const __vector uint8_t) {
0x08, 0x09, 0x18, 0x19, 0x0a, 0x0b, 0x1a, 0x1b,
0x0c, 0x0d, 0x1c, 0x1d, 0x0e, 0x0f, 0x1e, 0x1f};
static const __vector uint8_t epi32_low = (const __vector uint8_t) {
0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17};
static const __vector uint8_t epi32_hi = (const __vector uint8_t) {
0x08, 0x09, 0x0a, 0x0b, 0x18, 0x19, 0x1a, 0x1b,
0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
static const __vector uint8_t epi64_low = (const __vector uint8_t) {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17};
static const __vector uint8_t epi64_hi = (const __vector uint8_t) {
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f};
const uint8_t* in_b = (const uint8_t*)in;
uint8_t* out_b = (uint8_t*)out;
size_t nrows = 8 * elem_size;
size_t nbyte_row = size / 8;
__vector uint8_t xmm0[16], xmm1[16];
CHECK_MULT_EIGHT(size);
// The optimized algorithms can only deal with even values or 1 for elem_size
if ((elem_size > 1) && (elem_size % 2)) {
return bshuf_trans_byte_bitrow_scal(in, out, size, elem_size);
}
int nvectors = (elem_size == 1) ? 8 : 16;
for (size_t ii = 0; ii + (nvectors - 1) < nrows; ii += nvectors) {
for (size_t jj = 0; jj + 15 < nbyte_row; jj += 16) { // vectors of 16 elements
if (elem_size == 1) {
for (int k = 0; k < 8; k++) {
xmm0[k] = vec_xl((ii + k) * nbyte_row + jj, in_b);
}
xmm1[0] = vec_perm(xmm0[0], xmm0[1], epi8_low);
xmm1[1] = vec_perm(xmm0[2], xmm0[3], epi8_low);
xmm1[2] = vec_perm(xmm0[4], xmm0[5], epi8_low);
xmm1[3] = vec_perm(xmm0[6], xmm0[7], epi8_low);
xmm1[4] = vec_perm(xmm0[0], xmm0[1], epi8_hi);
xmm1[5] = vec_perm(xmm0[2], xmm0[3], epi8_hi);
xmm1[6] = vec_perm(xmm0[4], xmm0[5], epi8_hi);
xmm1[7] = vec_perm(xmm0[6], xmm0[7], epi8_hi);
xmm0[0] = vec_perm(xmm1[0], xmm1[1], epi16_low);
xmm0[1] = vec_perm(xmm1[2], xmm1[3], epi16_low);
xmm0[2] = vec_perm(xmm1[0], xmm1[1], epi16_hi);
xmm0[3] = vec_perm(xmm1[2], xmm1[3], epi16_hi);
xmm0[4] = vec_perm(xmm1[4], xmm1[5], epi16_low);
xmm0[5] = vec_perm(xmm1[6], xmm1[7], epi16_low);
xmm0[6] = vec_perm(xmm1[4], xmm1[5], epi16_hi);
xmm0[7] = vec_perm(xmm1[6], xmm1[7], epi16_hi);
xmm1[0] = vec_perm(xmm0[0], xmm0[1], epi32_low);
xmm1[1] = vec_perm(xmm0[0], xmm0[1], epi32_hi);
xmm1[2] = vec_perm(xmm0[2], xmm0[3], epi32_low);
xmm1[3] = vec_perm(xmm0[2], xmm0[3], epi32_hi);
xmm1[4] = vec_perm(xmm0[4], xmm0[5], epi32_low);
xmm1[5] = vec_perm(xmm0[4], xmm0[5], epi32_hi);
xmm1[6] = vec_perm(xmm0[6], xmm0[7], epi32_low);
xmm1[7] = vec_perm(xmm0[6], xmm0[7], epi32_hi);
for (int k = 0; k < 8; k++) {
vec_xst(xmm1[k], (jj + k * 2) * nrows + ii, out_b);
}
continue;
}
for (int k = 0; k < 16; k++) {
xmm0[k] = vec_xl((ii + k) * nbyte_row + jj, in_b);
}
for (int k = 0; k < 16; k += 8) {
xmm1[k + 0] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi8_low);
xmm1[k + 1] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi8_low);
xmm1[k + 2] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi8_low);
xmm1[k + 3] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi8_low);
xmm1[k + 4] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi8_hi);
xmm1[k + 5] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi8_hi);
xmm1[k + 6] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi8_hi);
xmm1[k + 7] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi8_hi);
}
for (int k = 0; k < 16; k += 8) {
xmm0[k + 0] = vec_perm(xmm1[k + 0], xmm1[k + 1], epi16_low);
xmm0[k + 1] = vec_perm(xmm1[k + 2], xmm1[k + 3], epi16_low);
xmm0[k + 2] = vec_perm(xmm1[k + 0], xmm1[k + 1], epi16_hi);
xmm0[k + 3] = vec_perm(xmm1[k + 2], xmm1[k + 3], epi16_hi);
xmm0[k + 4] = vec_perm(xmm1[k + 4], xmm1[k + 5], epi16_low);
xmm0[k + 5] = vec_perm(xmm1[k + 6], xmm1[k + 7], epi16_low);
xmm0[k + 6] = vec_perm(xmm1[k + 4], xmm1[k + 5], epi16_hi);
xmm0[k + 7] = vec_perm(xmm1[k + 6], xmm1[k + 7], epi16_hi);
}
for (int k = 0; k < 16; k += 8) {
xmm1[k + 0] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi32_low);
xmm1[k + 1] = vec_perm(xmm0[k + 0], xmm0[k + 1], epi32_hi);
xmm1[k + 2] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi32_low);
xmm1[k + 3] = vec_perm(xmm0[k + 2], xmm0[k + 3], epi32_hi);
xmm1[k + 4] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi32_low);
xmm1[k + 5] = vec_perm(xmm0[k + 4], xmm0[k + 5], epi32_hi);
xmm1[k + 6] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi32_low);
xmm1[k + 7] = vec_perm(xmm0[k + 6], xmm0[k + 7], epi32_hi);
}
for (int k = 0; k < 8; k += 4) {
xmm0[k * 2 + 0] = vec_perm(xmm1[k + 0], xmm1[k + 8], epi64_low);
xmm0[k * 2 + 1] = vec_perm(xmm1[k + 0], xmm1[k + 8], epi64_hi);
xmm0[k * 2 + 2] = vec_perm(xmm1[k + 1], xmm1[k + 9], epi64_low);
xmm0[k * 2 + 3] = vec_perm(xmm1[k + 1], xmm1[k + 9], epi64_hi);
xmm0[k * 2 + 4] = vec_perm(xmm1[k + 2], xmm1[k + 10], epi64_low);
xmm0[k * 2 + 5] = vec_perm(xmm1[k + 2], xmm1[k + 10], epi64_hi);
xmm0[k * 2 + 6] = vec_perm(xmm1[k + 3], xmm1[k + 11], epi64_low);
xmm0[k * 2 + 7] = vec_perm(xmm1[k + 3], xmm1[k + 11], epi64_hi);
}
for (int k = 0; k < 16; k++) {
vec_xst(xmm0[k], (jj + k) * nrows + ii, out_b);
}
}
// Copy the remainder
for (size_t jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj++) {
for (int k = 0; k < nvectors; k++) {
out_b[jj * nrows + ii + k] = in_b[(ii + k) * nbyte_row + jj];
}
}
}
return size * elem_size;
}
/* Shuffle bits within the bytes of eight element blocks. */
int64_t bshuf_shuffle_bit_eightelem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size) {
/* With a bit of care, this could be written such that such that it is */
/* in_buf = out_buf safe. */
const uint8_t* in_b = (const uint8_t*)in;
uint8_t* out_b = (uint8_t*)out;
size_t nbyte = elem_size * size;
__vector uint8_t masks[8], data;
CHECK_MULT_EIGHT(size);
// Generate all 8 needed masks
for (int kk = 0; kk < 8; kk++){
masks[kk] = make_bitperm_mask(1, kk);
}
if (elem_size % 2) {
bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size);
} else {
for (size_t ii = 0; ii + 8 * elem_size - 1 < nbyte;
ii += 8 * elem_size) {
for (size_t jj = 0; jj + 15 < 8 * elem_size; jj += 16) {
data = vec_xl(ii + jj, in_b);
for (size_t kk = 0; kk < 8; kk++) {
__vector uint16_t tmp;
uint16_t* oui16;
tmp = (__vector uint16_t) vec_bperm(data, masks[kk]);
oui16 = (uint16_t*)&out_b[ii + (jj>>3) + kk * elem_size];
*oui16 = tmp[BLOSC_BPERM_RESULT_IDX];
}
}
}
}
return size * elem_size;
}
/* Untranspose bits within elements. */
int64_t bshuf_untrans_bit_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size) {
int64_t count;
CHECK_MULT_EIGHT(size);
void* tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
count = bshuf_trans_byte_bitrow_altivec(in, tmp_buf, size, elem_size);
CHECK_ERR(count);
count = bshuf_shuffle_bit_eightelem_altivec(tmp_buf, out, size, elem_size);
free(tmp_buf);
return count;
}
const bool is_bshuf_altivec = true;
#else /* VSX + POWER8 path */
const bool is_bshuf_altivec = false;
int64_t
bshuf_trans_bit_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size) {
abort();
}
int64_t
bshuf_untrans_bit_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size) {
abort();
}
#endif /* defined(__ALTIVEC__) && defined(__VSX__) && defined(_ARCH_PWR8) && !defined(__APPLE__) */
Blosc-c-blosc2-d3bee7c/blosc/bitshuffle-altivec.h 0000664 0000000 0000000 00000003253 15255167342 0022004 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
/* ALTIVEC-accelerated shuffle/unshuffle routines. */
#ifndef BLOSC_BITSHUFFLE_ALTIVEC_H
#define BLOSC_BITSHUFFLE_ALTIVEC_H
#include "blosc2/blosc2-common.h"
#include
#include
#include
/**
* ALTIVEC-accelerated bit(un)shuffle routines availability.
*/
extern const bool is_bshuf_altivec;
BLOSC_NO_EXPORT int64_t
bshuf_trans_byte_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size, void* tmp_buf);
BLOSC_NO_EXPORT int64_t
bshuf_trans_byte_bitrow_altivec(const void* in, void* out, const size_t size,
const size_t elem_size);
BLOSC_NO_EXPORT int64_t
bshuf_shuffle_bit_eightelem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size);
/**
ALTIVEC-accelerated bitshuffle routine.
*/
BLOSC_NO_EXPORT int64_t
bshuf_trans_bit_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size);
/**
ALTIVEC-accelerated bitunshuffle routine.
*/
BLOSC_NO_EXPORT int64_t
bshuf_untrans_bit_elem_altivec(const void* in, void* out, const size_t size,
const size_t elem_size);
#endif /* BLOSC_BITSHUFFLE_ALTIVEC_H */
Blosc-c-blosc2-d3bee7c/blosc/bitshuffle-avx2.c 0000664 0000000 0000000 00000020671 15255167342 0021233 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
/*********************************************************************
Bitshuffle - Filter for improving compression of typed binary data.
Author: Kiyoshi Masui
Website: https://github.com/kiyo-masui/bitshuffle
Note: Adapted for c-blosc by Francesc Alted.
See LICENSES/BITSHUFFLE.txt file for details about copyright and
rights to use.
**********************************************************************/
#include "bitshuffle-avx2.h"
#include "bitshuffle-sse2.h"
#include "bitshuffle-generic.h"
#include
/* Make sure AVX2 is available for the compilation target and compiler. */
#if defined(__AVX2__)
#include
/* The next is useful for debugging purposes */
#if 0
#include
#include
static void printymm(__m256i ymm0)
{
uint8_t buf[32];
((__m256i *)buf)[0] = ymm0;
printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n",
buf[0], buf[1], buf[2], buf[3],
buf[4], buf[5], buf[6], buf[7],
buf[8], buf[9], buf[10], buf[11],
buf[12], buf[13], buf[14], buf[15],
buf[16], buf[17], buf[18], buf[19],
buf[20], buf[21], buf[22], buf[23],
buf[24], buf[25], buf[26], buf[27],
buf[28], buf[29], buf[30], buf[31]);
}
#endif
/* ---- Code that requires AVX2. Intel Haswell (2013) and later. ---- */
/* Transpose bits within bytes. */
int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
size_t ii, kk;
const char* in_b = (const char*) in;
char* out_b = (char*) out;
int32_t* out_i32;
size_t nbyte = elem_size * size;
int64_t count;
__m256i ymm;
int32_t bt;
for (ii = 0; ii + 31 < nbyte; ii += 32) {
ymm = _mm256_loadu_si256((__m256i *) &in_b[ii]);
for (kk = 0; kk < 8; kk++) {
bt = _mm256_movemask_epi8(ymm);
ymm = _mm256_slli_epi16(ymm, 1);
out_i32 = (int32_t*) &out_b[((7 - kk) * nbyte + ii) / 8];
*out_i32 = bt;
}
}
count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size,
nbyte - nbyte % 32);
return count;
}
/* Transpose bits within elements. */
int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
int64_t count;
CHECK_MULT_EIGHT(size);
void* tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size);
CHECK_ERR_FREE(count, tmp_buf);
count = bshuf_trans_bit_byte_AVX(out, tmp_buf, size, elem_size);
CHECK_ERR_FREE(count, tmp_buf);
count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size);
free(tmp_buf);
return count;
}
/* For data organized into a row for each bit (8 * elem_size rows), transpose
* the bytes. */
int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
size_t hh, ii, jj, kk, mm;
const char* in_b = (const char*) in;
char* out_b = (char*) out;
CHECK_MULT_EIGHT(size);
size_t nrows = 8 * elem_size;
size_t nbyte_row = size / 8;
if (elem_size % 4) return bshuf_trans_byte_bitrow_SSE(in, out, size,
elem_size);
__m256i ymm_0[8];
__m256i ymm_1[8];
__m256i ymm_storeage[8][4];
for (jj = 0; jj + 31 < nbyte_row; jj += 32) {
for (ii = 0; ii + 3 < elem_size; ii += 4) {
for (hh = 0; hh < 4; hh ++) {
for (kk = 0; kk < 8; kk ++){
ymm_0[kk] = _mm256_loadu_si256((__m256i *) &in_b[
(ii * 8 + hh * 8 + kk) * nbyte_row + jj]);
}
for (kk = 0; kk < 4; kk ++){
ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
}
for (kk = 0; kk < 2; kk ++){
for (mm = 0; mm < 2; mm ++){
ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16(
ymm_1[kk * 4 + mm * 2],
ymm_1[kk * 4 + mm * 2 + 1]);
ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16(
ymm_1[kk * 4 + mm * 2],
ymm_1[kk * 4 + mm * 2 + 1]);
}
}
for (kk = 0; kk < 4; kk ++){
ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2],
ymm_0[kk * 2 + 1]);
}
for (kk = 0; kk < 8; kk ++){
ymm_storeage[kk][hh] = ymm_1[kk];
}
}
for (mm = 0; mm < 8; mm ++) {
for (kk = 0; kk < 4; kk ++){
ymm_0[kk] = ymm_storeage[mm][kk];
}
ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]);
ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]);
ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]);
ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]);
ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32);
ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32);
ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49);
ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]);
_mm256_storeu_si256((__m256i *) &out_b[
(jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]);
}
}
}
for (ii = 0; ii < nrows; ii ++ ) {
for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj ++) {
out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj];
}
}
return size * elem_size;
}
/* Shuffle bits within the bytes of eight element blocks. */
int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
CHECK_MULT_EIGHT(size);
// With a bit of care, this could be written such that such that it is
// in_buf = out_buf safe.
const char* in_b = (const char*) in;
char* out_b = (char*) out;
size_t ii, jj, kk;
size_t nbyte = elem_size * size;
__m256i ymm;
int32_t bt;
if (elem_size % 4) {
return bshuf_shuffle_bit_eightelem_SSE(in, out, size, elem_size);
} else {
for (jj = 0; jj + 31 < 8 * elem_size; jj += 32) {
for (ii = 0; ii + 8 * elem_size - 1 < nbyte;
ii += 8 * elem_size) {
ymm = _mm256_loadu_si256((__m256i *) &in_b[ii + jj]);
for (kk = 0; kk < 8; kk++) {
bt = _mm256_movemask_epi8(ymm);
ymm = _mm256_slli_epi16(ymm, 1);
size_t ind = (ii + jj / 8 + (7 - kk) * elem_size);
* (int32_t *) &out_b[ind] = bt;
}
}
}
}
return size * elem_size;
}
/* Untranspose bits within elements. */
int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
int64_t count;
CHECK_MULT_EIGHT(size);
void* tmp_buf = malloc(size * elem_size);
if (tmp_buf == NULL) return -1;
count = bshuf_trans_byte_bitrow_AVX(in, tmp_buf, size, elem_size);
CHECK_ERR_FREE(count, tmp_buf);
count = bshuf_shuffle_bit_eightelem_AVX(tmp_buf, out, size, elem_size);
free(tmp_buf);
return count;
}
const bool is_bshuf_AVX = true;
#else /* defined(__AVX2__) */
const bool is_bshuf_AVX = false;
int64_t
bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
abort();
}
int64_t
bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size) {
abort();
}
#endif /* defined(__AVX2__) */
Blosc-c-blosc2-d3bee7c/blosc/bitshuffle-avx2.h 0000664 0000000 0000000 00000002721 15255167342 0021234 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
/* AVX2-accelerated shuffle/unshuffle routines. */
#ifndef BLOSC_BITSHUFFLE_AVX2_H
#define BLOSC_BITSHUFFLE_AVX2_H
#include "blosc2/blosc2-common.h"
#include
#include
#include
/**
* AVX2-accelerated bit(un)shuffle routines availability.
*/
extern const bool is_bshuf_AVX;
/**
* AVX2-accelerated bitshuffle routine.
*/
BLOSC_NO_EXPORT int64_t
bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size);
/**
* AVX2-accelerated bitunshuffle routine.
*/
BLOSC_NO_EXPORT int64_t
bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size);
/**
* AVX2 utils used by AVX512 functions
*/
int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size,
const size_t elem_size);
int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size,
const size_t elem_size);
#endif /* BLOSC_BITSHUFFLE_AVX2_H */
Blosc-c-blosc2-d3bee7c/blosc/bitshuffle-avx512.c 0000664 0000000 0000000 00000012211 15255167342 0021370 0 ustar 00root root 0000000 0000000 /*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team