blake2b_simd-1.0.0/.cargo_vcs_info.json0000644000000001120000000000100133420ustar { "git": { "sha1": "4e35d8bd27542737278c14ee550d5cec9dfc35c7" } } blake2b_simd-1.0.0/Cargo.toml0000644000000020200000000000100113400ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "blake2b_simd" version = "1.0.0" authors = ["Jack O'Connor"] description = "a pure Rust BLAKE2b implementation with dynamic SIMD" documentation = "https://docs.rs/blake2b_simd" readme = "README.md" keywords = ["blake2b", "blake2bp", "blake2"] license = "MIT" repository = "https://github.com/oconnor663/blake2_simd" [dependencies.arrayref] version = "0.3.5" [dependencies.arrayvec] version = "0.7.0" default-features = false [dependencies.constant_time_eq] version = "0.1.3" [features] default = ["std"] std = [] uninline_portable = [] blake2b_simd-1.0.0/Cargo.toml.orig000064400000000000000000000014600072674642500150600ustar 00000000000000[package] name = "blake2b_simd" version = "1.0.0" authors = ["Jack O'Connor"] description = "a pure Rust BLAKE2b implementation with dynamic SIMD" license = "MIT" repository = "https://github.com/oconnor663/blake2_simd" documentation = "https://docs.rs/blake2b_simd" readme = "README.md" keywords = ["blake2b", "blake2bp", "blake2"] edition = "2018" [features] default = ["std"] std = [] # This crate does a lot of #[inline(always)]. For BLAKE2b on ARM Cortex-M0 (and # presumably other tiny chips), some of that inlining actually hurts # performance. This feature disables some inlining, improving the performance # of the portable implementation in that specific case. uninline_portable = [] [dependencies] arrayref = "0.3.5" arrayvec = { version = "0.7.0", default-features = false } constant_time_eq = "0.1.3" blake2b_simd-1.0.0/README.md000064400000000000000000000036770072674642500134640ustar 00000000000000# blake2b_simd [![GitHub](https://img.shields.io/github/tag/oconnor663/blake2_simd.svg?label=GitHub)](https://github.com/oconnor663/blake2_simd) [![crates.io](https://img.shields.io/crates/v/blake2b_simd.svg)](https://crates.io/crates/blake2b_simd) [![Actions Status](https://github.com/oconnor663/blake2_simd/workflows/tests/badge.svg)](https://github.com/oconnor663/blake2_simd/actions) An implementation of the BLAKE2b and BLAKE2bp hash functions. See also [`blake2s_simd`](../blake2s). This crate includes: - 100% stable Rust. - SIMD implementations based on Samuel Neves' [`blake2-avx2`](https://github.com/sneves/blake2-avx2). These are very fast. For benchmarks, see [the Performance section of the README](https://github.com/oconnor663/blake2_simd#performance). - Portable, safe implementations for other platforms. - Dynamic CPU feature detection. Binaries include multiple implementations by default and choose the fastest one the processor supports at runtime. - All the features from the [the BLAKE2 spec](https://blake2.net/blake2.pdf), like adjustable length, keying, and associated data for tree hashing. - `no_std` support. The `std` Cargo feature is on by default, for CPU feature detection and for implementing `std::io::Write`. - Support for computing multiple BLAKE2b hashes in parallel, matching the efficiency of BLAKE2bp. See the [`many`](many/index.html) module. # Example ``` use blake2b_simd::{blake2b, Params}; let expected = "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6d\ c1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d"; let hash = blake2b(b"foo"); assert_eq!(expected, &hash.to_hex()); let hash = Params::new() .hash_length(16) .key(b"The Magic Words are Squeamish Ossifrage") .personal(b"L. P. Waterhouse") .to_state() .update(b"foo") .update(b"bar") .update(b"baz") .finalize(); assert_eq!("ee8ff4e9be887297cf79348dc35dab56", &hash.to_hex()); ``` blake2b_simd-1.0.0/src/avx2.rs000064400000000000000000001003640072674642500142110ustar 00000000000000#[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; use crate::guts::{ assemble_count, count_high, count_low, final_block, flag_word, input_debug_asserts, Finalize, Job, LastNode, Stride, }; use crate::{Count, Word, BLOCKBYTES, IV, SIGMA}; use arrayref::{array_refs, mut_array_refs}; use core::cmp; use core::mem; pub const DEGREE: usize = 4; #[inline(always)] unsafe fn loadu(src: *const [Word; DEGREE]) -> __m256i { // This is an unaligned load, so the pointer cast is allowed. _mm256_loadu_si256(src as *const __m256i) } #[inline(always)] unsafe fn storeu(src: __m256i, dest: *mut [Word; DEGREE]) { // This is an unaligned store, so the pointer cast is allowed. _mm256_storeu_si256(dest as *mut __m256i, src) } #[inline(always)] unsafe fn loadu_128(mem_addr: &[u8; 16]) -> __m128i { _mm_loadu_si128(mem_addr.as_ptr() as *const __m128i) } #[inline(always)] unsafe fn add(a: __m256i, b: __m256i) -> __m256i { _mm256_add_epi64(a, b) } #[inline(always)] unsafe fn eq(a: __m256i, b: __m256i) -> __m256i { _mm256_cmpeq_epi64(a, b) } #[inline(always)] unsafe fn and(a: __m256i, b: __m256i) -> __m256i { _mm256_and_si256(a, b) } #[inline(always)] unsafe fn negate_and(a: __m256i, b: __m256i) -> __m256i { // Note that "and not" implies the reverse of the actual arg order. _mm256_andnot_si256(a, b) } #[inline(always)] unsafe fn xor(a: __m256i, b: __m256i) -> __m256i { _mm256_xor_si256(a, b) } #[inline(always)] unsafe fn set1(x: u64) -> __m256i { _mm256_set1_epi64x(x as i64) } #[inline(always)] unsafe fn set4(a: u64, b: u64, c: u64, d: u64) -> __m256i { _mm256_setr_epi64x(a as i64, b as i64, c as i64, d as i64) } // Adapted from https://github.com/rust-lang-nursery/stdsimd/pull/479. macro_rules! _MM_SHUFFLE { ($z:expr, $y:expr, $x:expr, $w:expr) => { ($z << 6) | ($y << 4) | ($x << 2) | $w }; } // These rotations are the "simple version". For the "complicated version", see // https://github.com/sneves/blake2-avx2/blob/b3723921f668df09ece52dcd225a36d4a4eea1d9/blake2b-common.h#L43-L46. // For a discussion of the tradeoffs, see // https://github.com/sneves/blake2-avx2/pull/5. In short: // - Due to an LLVM bug (https://bugs.llvm.org/show_bug.cgi?id=44379), this // version performs better on recent x86 chips. // - LLVM is able to optimize this version to AVX-512 rotation instructions // when those are enabled. #[inline(always)] unsafe fn rot32(x: __m256i) -> __m256i { _mm256_or_si256(_mm256_srli_epi64(x, 32), _mm256_slli_epi64(x, 64 - 32)) } #[inline(always)] unsafe fn rot24(x: __m256i) -> __m256i { _mm256_or_si256(_mm256_srli_epi64(x, 24), _mm256_slli_epi64(x, 64 - 24)) } #[inline(always)] unsafe fn rot16(x: __m256i) -> __m256i { _mm256_or_si256(_mm256_srli_epi64(x, 16), _mm256_slli_epi64(x, 64 - 16)) } #[inline(always)] unsafe fn rot63(x: __m256i) -> __m256i { _mm256_or_si256(_mm256_srli_epi64(x, 63), _mm256_slli_epi64(x, 64 - 63)) } #[inline(always)] unsafe fn g1(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m256i, m: &mut __m256i) { *a = add(*a, *m); *a = add(*a, *b); *d = xor(*d, *a); *d = rot32(*d); *c = add(*c, *d); *b = xor(*b, *c); *b = rot24(*b); } #[inline(always)] unsafe fn g2(a: &mut __m256i, b: &mut __m256i, c: &mut __m256i, d: &mut __m256i, m: &mut __m256i) { *a = add(*a, *m); *a = add(*a, *b); *d = xor(*d, *a); *d = rot16(*d); *c = add(*c, *d); *b = xor(*b, *c); *b = rot63(*b); } // Note the optimization here of leaving b as the unrotated row, rather than a. // All the message loads below are adjusted to compensate for this. See // discussion at https://github.com/sneves/blake2-avx2/pull/4 #[inline(always)] unsafe fn diagonalize(a: &mut __m256i, _b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { *a = _mm256_permute4x64_epi64(*a, _MM_SHUFFLE!(2, 1, 0, 3)); *d = _mm256_permute4x64_epi64(*d, _MM_SHUFFLE!(1, 0, 3, 2)); *c = _mm256_permute4x64_epi64(*c, _MM_SHUFFLE!(0, 3, 2, 1)); } #[inline(always)] unsafe fn undiagonalize(a: &mut __m256i, _b: &mut __m256i, c: &mut __m256i, d: &mut __m256i) { *a = _mm256_permute4x64_epi64(*a, _MM_SHUFFLE!(0, 3, 2, 1)); *d = _mm256_permute4x64_epi64(*d, _MM_SHUFFLE!(1, 0, 3, 2)); *c = _mm256_permute4x64_epi64(*c, _MM_SHUFFLE!(2, 1, 0, 3)); } #[inline(always)] unsafe fn compress_block( block: &[u8; BLOCKBYTES], words: &mut [Word; 8], count: Count, last_block: Word, last_node: Word, ) { let (words_low, words_high) = mut_array_refs!(words, DEGREE, DEGREE); let (iv_low, iv_high) = array_refs!(&IV, DEGREE, DEGREE); let mut a = loadu(words_low); let mut b = loadu(words_high); let mut c = loadu(iv_low); let flags = set4(count_low(count), count_high(count), last_block, last_node); let mut d = xor(loadu(iv_high), flags); let msg_chunks = array_refs!(block, 16, 16, 16, 16, 16, 16, 16, 16); let m0 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.0)); let m1 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.1)); let m2 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.2)); let m3 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.3)); let m4 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.4)); let m5 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.5)); let m6 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.6)); let m7 = _mm256_broadcastsi128_si256(loadu_128(msg_chunks.7)); let iv0 = a; let iv1 = b; let mut t0; let mut t1; let mut b0; // round 1 t0 = _mm256_unpacklo_epi64(m0, m1); t1 = _mm256_unpacklo_epi64(m2, m3); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m0, m1); t1 = _mm256_unpackhi_epi64(m2, m3); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpacklo_epi64(m7, m4); t1 = _mm256_unpacklo_epi64(m5, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m7, m4); t1 = _mm256_unpackhi_epi64(m5, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 2 t0 = _mm256_unpacklo_epi64(m7, m2); t1 = _mm256_unpackhi_epi64(m4, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m5, m4); t1 = _mm256_alignr_epi8(m3, m7, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpackhi_epi64(m2, m0); t1 = _mm256_blend_epi32(m5, m0, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_alignr_epi8(m6, m1, 8); t1 = _mm256_blend_epi32(m3, m1, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 3 t0 = _mm256_alignr_epi8(m6, m5, 8); t1 = _mm256_unpackhi_epi64(m2, m7); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m4, m0); t1 = _mm256_blend_epi32(m6, m1, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_alignr_epi8(m5, m4, 8); t1 = _mm256_unpackhi_epi64(m1, m3); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m2, m7); t1 = _mm256_blend_epi32(m0, m3, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 4 t0 = _mm256_unpackhi_epi64(m3, m1); t1 = _mm256_unpackhi_epi64(m6, m5); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m4, m0); t1 = _mm256_unpacklo_epi64(m6, m7); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_alignr_epi8(m1, m7, 8); t1 = _mm256_shuffle_epi32(m2, _MM_SHUFFLE!(1, 0, 3, 2)); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m4, m3); t1 = _mm256_unpacklo_epi64(m5, m0); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 5 t0 = _mm256_unpackhi_epi64(m4, m2); t1 = _mm256_unpacklo_epi64(m1, m5); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_blend_epi32(m3, m0, 0x33); t1 = _mm256_blend_epi32(m7, m2, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_alignr_epi8(m7, m1, 8); t1 = _mm256_alignr_epi8(m3, m5, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m6, m0); t1 = _mm256_unpacklo_epi64(m6, m4); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 6 t0 = _mm256_unpacklo_epi64(m1, m3); t1 = _mm256_unpacklo_epi64(m0, m4); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m6, m5); t1 = _mm256_unpackhi_epi64(m5, m1); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_alignr_epi8(m2, m0, 8); t1 = _mm256_unpackhi_epi64(m3, m7); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m4, m6); t1 = _mm256_alignr_epi8(m7, m2, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 7 t0 = _mm256_blend_epi32(m0, m6, 0x33); t1 = _mm256_unpacklo_epi64(m7, m2); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m2, m7); t1 = _mm256_alignr_epi8(m5, m6, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpacklo_epi64(m4, m0); t1 = _mm256_blend_epi32(m4, m3, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m5, m3); t1 = _mm256_shuffle_epi32(m1, _MM_SHUFFLE!(1, 0, 3, 2)); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 8 t0 = _mm256_unpackhi_epi64(m6, m3); t1 = _mm256_blend_epi32(m1, m6, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_alignr_epi8(m7, m5, 8); t1 = _mm256_unpackhi_epi64(m0, m4); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_blend_epi32(m2, m1, 0x33); t1 = _mm256_alignr_epi8(m4, m7, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m5, m0); t1 = _mm256_unpacklo_epi64(m2, m3); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 9 t0 = _mm256_unpacklo_epi64(m3, m7); t1 = _mm256_alignr_epi8(m0, m5, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m7, m4); t1 = _mm256_alignr_epi8(m4, m1, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpacklo_epi64(m5, m6); t1 = _mm256_unpackhi_epi64(m6, m0); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_alignr_epi8(m1, m2, 8); t1 = _mm256_alignr_epi8(m2, m3, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 10 t0 = _mm256_unpacklo_epi64(m5, m4); t1 = _mm256_unpackhi_epi64(m3, m0); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m1, m2); t1 = _mm256_blend_epi32(m2, m3, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpackhi_epi64(m6, m7); t1 = _mm256_unpackhi_epi64(m4, m1); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_blend_epi32(m5, m0, 0x33); t1 = _mm256_unpacklo_epi64(m7, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 11 t0 = _mm256_unpacklo_epi64(m0, m1); t1 = _mm256_unpacklo_epi64(m2, m3); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m0, m1); t1 = _mm256_unpackhi_epi64(m2, m3); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpacklo_epi64(m7, m4); t1 = _mm256_unpacklo_epi64(m5, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpackhi_epi64(m7, m4); t1 = _mm256_unpackhi_epi64(m5, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); // round 12 t0 = _mm256_unpacklo_epi64(m7, m2); t1 = _mm256_unpackhi_epi64(m4, m6); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_unpacklo_epi64(m5, m4); t1 = _mm256_alignr_epi8(m3, m7, 8); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); diagonalize(&mut a, &mut b, &mut c, &mut d); t0 = _mm256_unpackhi_epi64(m2, m0); t1 = _mm256_blend_epi32(m5, m0, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g1(&mut a, &mut b, &mut c, &mut d, &mut b0); t0 = _mm256_alignr_epi8(m6, m1, 8); t1 = _mm256_blend_epi32(m3, m1, 0x33); b0 = _mm256_blend_epi32(t0, t1, 0xF0); g2(&mut a, &mut b, &mut c, &mut d, &mut b0); undiagonalize(&mut a, &mut b, &mut c, &mut d); a = xor(a, c); b = xor(b, d); a = xor(a, iv0); b = xor(b, iv1); storeu(a, words_low); storeu(b, words_high); } #[target_feature(enable = "avx2")] pub unsafe fn compress1_loop( input: &[u8], words: &mut [Word; 8], mut count: Count, last_node: LastNode, finalize: Finalize, stride: Stride, ) { input_debug_asserts(input, finalize); let mut local_words = *words; let mut fin_offset = input.len().saturating_sub(1); fin_offset -= fin_offset % stride.padded_blockbytes(); let mut buf = [0; BLOCKBYTES]; let (fin_block, fin_len, _) = final_block(input, fin_offset, &mut buf, stride); let fin_last_block = flag_word(finalize.yes()); let fin_last_node = flag_word(finalize.yes() && last_node.yes()); let mut offset = 0; loop { let block; let count_delta; let last_block; let last_node; if offset == fin_offset { block = fin_block; count_delta = fin_len; last_block = fin_last_block; last_node = fin_last_node; } else { // This unsafe cast avoids bounds checks. There's guaranteed to be // enough input because `offset < fin_offset`. block = &*(input.as_ptr().add(offset) as *const [u8; BLOCKBYTES]); count_delta = BLOCKBYTES; last_block = flag_word(false); last_node = flag_word(false); }; count = count.wrapping_add(count_delta as Count); compress_block(block, &mut local_words, count, last_block, last_node); // Check for termination before bumping the offset, to avoid overflow. if offset == fin_offset { break; } offset += stride.padded_blockbytes(); } *words = local_words; } // Performance note: Factoring out a G function here doesn't hurt performance, // unlike in the case of BLAKE2s where it hurts substantially. In fact, on my // machine, it helps a tiny bit. But the difference it tiny, so I'm going to // stick to the approach used by https://github.com/sneves/blake2-avx2 // until/unless I can be sure the (tiny) improvement is consistent across // different Intel microarchitectures. Smaller code size is nice, but a // divergence between the BLAKE2b and BLAKE2s implementations is less nice. #[inline(always)] unsafe fn round(v: &mut [__m256i; 16], m: &[__m256i; 16], r: usize) { v[0] = add(v[0], m[SIGMA[r][0] as usize]); v[1] = add(v[1], m[SIGMA[r][2] as usize]); v[2] = add(v[2], m[SIGMA[r][4] as usize]); v[3] = add(v[3], m[SIGMA[r][6] as usize]); v[0] = add(v[0], v[4]); v[1] = add(v[1], v[5]); v[2] = add(v[2], v[6]); v[3] = add(v[3], v[7]); v[12] = xor(v[12], v[0]); v[13] = xor(v[13], v[1]); v[14] = xor(v[14], v[2]); v[15] = xor(v[15], v[3]); v[12] = rot32(v[12]); v[13] = rot32(v[13]); v[14] = rot32(v[14]); v[15] = rot32(v[15]); v[8] = add(v[8], v[12]); v[9] = add(v[9], v[13]); v[10] = add(v[10], v[14]); v[11] = add(v[11], v[15]); v[4] = xor(v[4], v[8]); v[5] = xor(v[5], v[9]); v[6] = xor(v[6], v[10]); v[7] = xor(v[7], v[11]); v[4] = rot24(v[4]); v[5] = rot24(v[5]); v[6] = rot24(v[6]); v[7] = rot24(v[7]); v[0] = add(v[0], m[SIGMA[r][1] as usize]); v[1] = add(v[1], m[SIGMA[r][3] as usize]); v[2] = add(v[2], m[SIGMA[r][5] as usize]); v[3] = add(v[3], m[SIGMA[r][7] as usize]); v[0] = add(v[0], v[4]); v[1] = add(v[1], v[5]); v[2] = add(v[2], v[6]); v[3] = add(v[3], v[7]); v[12] = xor(v[12], v[0]); v[13] = xor(v[13], v[1]); v[14] = xor(v[14], v[2]); v[15] = xor(v[15], v[3]); v[12] = rot16(v[12]); v[13] = rot16(v[13]); v[14] = rot16(v[14]); v[15] = rot16(v[15]); v[8] = add(v[8], v[12]); v[9] = add(v[9], v[13]); v[10] = add(v[10], v[14]); v[11] = add(v[11], v[15]); v[4] = xor(v[4], v[8]); v[5] = xor(v[5], v[9]); v[6] = xor(v[6], v[10]); v[7] = xor(v[7], v[11]); v[4] = rot63(v[4]); v[5] = rot63(v[5]); v[6] = rot63(v[6]); v[7] = rot63(v[7]); v[0] = add(v[0], m[SIGMA[r][8] as usize]); v[1] = add(v[1], m[SIGMA[r][10] as usize]); v[2] = add(v[2], m[SIGMA[r][12] as usize]); v[3] = add(v[3], m[SIGMA[r][14] as usize]); v[0] = add(v[0], v[5]); v[1] = add(v[1], v[6]); v[2] = add(v[2], v[7]); v[3] = add(v[3], v[4]); v[15] = xor(v[15], v[0]); v[12] = xor(v[12], v[1]); v[13] = xor(v[13], v[2]); v[14] = xor(v[14], v[3]); v[15] = rot32(v[15]); v[12] = rot32(v[12]); v[13] = rot32(v[13]); v[14] = rot32(v[14]); v[10] = add(v[10], v[15]); v[11] = add(v[11], v[12]); v[8] = add(v[8], v[13]); v[9] = add(v[9], v[14]); v[5] = xor(v[5], v[10]); v[6] = xor(v[6], v[11]); v[7] = xor(v[7], v[8]); v[4] = xor(v[4], v[9]); v[5] = rot24(v[5]); v[6] = rot24(v[6]); v[7] = rot24(v[7]); v[4] = rot24(v[4]); v[0] = add(v[0], m[SIGMA[r][9] as usize]); v[1] = add(v[1], m[SIGMA[r][11] as usize]); v[2] = add(v[2], m[SIGMA[r][13] as usize]); v[3] = add(v[3], m[SIGMA[r][15] as usize]); v[0] = add(v[0], v[5]); v[1] = add(v[1], v[6]); v[2] = add(v[2], v[7]); v[3] = add(v[3], v[4]); v[15] = xor(v[15], v[0]); v[12] = xor(v[12], v[1]); v[13] = xor(v[13], v[2]); v[14] = xor(v[14], v[3]); v[15] = rot16(v[15]); v[12] = rot16(v[12]); v[13] = rot16(v[13]); v[14] = rot16(v[14]); v[10] = add(v[10], v[15]); v[11] = add(v[11], v[12]); v[8] = add(v[8], v[13]); v[9] = add(v[9], v[14]); v[5] = xor(v[5], v[10]); v[6] = xor(v[6], v[11]); v[7] = xor(v[7], v[8]); v[4] = xor(v[4], v[9]); v[5] = rot63(v[5]); v[6] = rot63(v[6]); v[7] = rot63(v[7]); v[4] = rot63(v[4]); } // We'd rather make this a regular function with #[inline(always)], but for // some reason that blows up compile times by about 10 seconds, at least in // some cases (BLAKE2b avx2.rs). This macro seems to get the same performance // result, without the compile time issue. macro_rules! compress4_transposed { ( $h_vecs:expr, $msg_vecs:expr, $count_low:expr, $count_high:expr, $lastblock:expr, $lastnode:expr, ) => { let h_vecs: &mut [__m256i; 8] = $h_vecs; let msg_vecs: &[__m256i; 16] = $msg_vecs; let count_low: __m256i = $count_low; let count_high: __m256i = $count_high; let lastblock: __m256i = $lastblock; let lastnode: __m256i = $lastnode; let mut v = [ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), xor(set1(IV[4]), count_low), xor(set1(IV[5]), count_high), xor(set1(IV[6]), lastblock), xor(set1(IV[7]), lastnode), ]; round(&mut v, &msg_vecs, 0); round(&mut v, &msg_vecs, 1); round(&mut v, &msg_vecs, 2); round(&mut v, &msg_vecs, 3); round(&mut v, &msg_vecs, 4); round(&mut v, &msg_vecs, 5); round(&mut v, &msg_vecs, 6); round(&mut v, &msg_vecs, 7); round(&mut v, &msg_vecs, 8); round(&mut v, &msg_vecs, 9); round(&mut v, &msg_vecs, 10); round(&mut v, &msg_vecs, 11); h_vecs[0] = xor(xor(h_vecs[0], v[0]), v[8]); h_vecs[1] = xor(xor(h_vecs[1], v[1]), v[9]); h_vecs[2] = xor(xor(h_vecs[2], v[2]), v[10]); h_vecs[3] = xor(xor(h_vecs[3], v[3]), v[11]); h_vecs[4] = xor(xor(h_vecs[4], v[4]), v[12]); h_vecs[5] = xor(xor(h_vecs[5], v[5]), v[13]); h_vecs[6] = xor(xor(h_vecs[6], v[6]), v[14]); h_vecs[7] = xor(xor(h_vecs[7], v[7]), v[15]); }; } #[inline(always)] unsafe fn interleave128(a: __m256i, b: __m256i) -> (__m256i, __m256i) { ( _mm256_permute2x128_si256(a, b, 0x20), _mm256_permute2x128_si256(a, b, 0x31), ) } // There are several ways to do a transposition. We could do it naively, with 8 separate // _mm256_set_epi64x instructions, referencing each of the 64 words explicitly. Or we could copy // the vecs into contiguous storage and then use gather instructions. This third approach is to use // a series of unpack instructions to interleave the vectors. In my benchmarks, interleaving is the // fastest approach. To test this, run `cargo +nightly bench --bench libtest load_4` in the // https://github.com/oconnor663/bao_experiments repo. #[inline(always)] unsafe fn transpose_vecs( vec_a: __m256i, vec_b: __m256i, vec_c: __m256i, vec_d: __m256i, ) -> [__m256i; DEGREE] { // Interleave 64-bit lates. The low unpack is lanes 00/22 and the high is 11/33. let ab_02 = _mm256_unpacklo_epi64(vec_a, vec_b); let ab_13 = _mm256_unpackhi_epi64(vec_a, vec_b); let cd_02 = _mm256_unpacklo_epi64(vec_c, vec_d); let cd_13 = _mm256_unpackhi_epi64(vec_c, vec_d); // Interleave 128-bit lanes. let (abcd_0, abcd_2) = interleave128(ab_02, cd_02); let (abcd_1, abcd_3) = interleave128(ab_13, cd_13); [abcd_0, abcd_1, abcd_2, abcd_3] } #[inline(always)] unsafe fn transpose_state_vecs(jobs: &[Job; DEGREE]) -> [__m256i; 8] { // Load all the state words into transposed vectors, where the first vector // has the first word of each state, etc. Transposing once at the beginning // and once at the end is more efficient that repeating it for each block. let words0 = array_refs!(&jobs[0].words, DEGREE, DEGREE); let words1 = array_refs!(&jobs[1].words, DEGREE, DEGREE); let words2 = array_refs!(&jobs[2].words, DEGREE, DEGREE); let words3 = array_refs!(&jobs[3].words, DEGREE, DEGREE); let [h0, h1, h2, h3] = transpose_vecs( loadu(words0.0), loadu(words1.0), loadu(words2.0), loadu(words3.0), ); let [h4, h5, h6, h7] = transpose_vecs( loadu(words0.1), loadu(words1.1), loadu(words2.1), loadu(words3.1), ); [h0, h1, h2, h3, h4, h5, h6, h7] } #[inline(always)] unsafe fn untranspose_state_vecs(h_vecs: &[__m256i; 8], jobs: &mut [Job; DEGREE]) { // Un-transpose the updated state vectors back into the caller's arrays. let [job0, job1, job2, job3] = jobs; let words0 = mut_array_refs!(&mut job0.words, DEGREE, DEGREE); let words1 = mut_array_refs!(&mut job1.words, DEGREE, DEGREE); let words2 = mut_array_refs!(&mut job2.words, DEGREE, DEGREE); let words3 = mut_array_refs!(&mut job3.words, DEGREE, DEGREE); let out = transpose_vecs(h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3]); storeu(out[0], words0.0); storeu(out[1], words1.0); storeu(out[2], words2.0); storeu(out[3], words3.0); let out = transpose_vecs(h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7]); storeu(out[0], words0.1); storeu(out[1], words1.1); storeu(out[2], words2.1); storeu(out[3], words3.1); } #[inline(always)] unsafe fn transpose_msg_vecs(blocks: [*const [u8; BLOCKBYTES]; DEGREE]) -> [__m256i; 16] { // These input arrays have no particular alignment, so we use unaligned // loads to read from them. let block0 = blocks[0] as *const [Word; DEGREE]; let block1 = blocks[1] as *const [Word; DEGREE]; let block2 = blocks[2] as *const [Word; DEGREE]; let block3 = blocks[3] as *const [Word; DEGREE]; let [m0, m1, m2, m3] = transpose_vecs( loadu(block0.add(0)), loadu(block1.add(0)), loadu(block2.add(0)), loadu(block3.add(0)), ); let [m4, m5, m6, m7] = transpose_vecs( loadu(block0.add(1)), loadu(block1.add(1)), loadu(block2.add(1)), loadu(block3.add(1)), ); let [m8, m9, m10, m11] = transpose_vecs( loadu(block0.add(2)), loadu(block1.add(2)), loadu(block2.add(2)), loadu(block3.add(2)), ); let [m12, m13, m14, m15] = transpose_vecs( loadu(block0.add(3)), loadu(block1.add(3)), loadu(block2.add(3)), loadu(block3.add(3)), ); [ m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15, ] } #[inline(always)] unsafe fn load_counts(jobs: &[Job; DEGREE]) -> (__m256i, __m256i) { ( set4( count_low(jobs[0].count), count_low(jobs[1].count), count_low(jobs[2].count), count_low(jobs[3].count), ), set4( count_high(jobs[0].count), count_high(jobs[1].count), count_high(jobs[2].count), count_high(jobs[3].count), ), ) } #[inline(always)] unsafe fn store_counts(jobs: &mut [Job; DEGREE], low: __m256i, high: __m256i) { let low_ints: [Word; DEGREE] = mem::transmute(low); let high_ints: [Word; DEGREE] = mem::transmute(high); for i in 0..DEGREE { jobs[i].count = assemble_count(low_ints[i], high_ints[i]); } } #[inline(always)] unsafe fn add_to_counts(lo: &mut __m256i, hi: &mut __m256i, delta: __m256i) { // If the low counts reach zero, that means they wrapped, unless the delta // was also zero. *lo = add(*lo, delta); let lo_reached_zero = eq(*lo, set1(0)); let delta_was_zero = eq(delta, set1(0)); let hi_inc = and(set1(1), negate_and(delta_was_zero, lo_reached_zero)); *hi = add(*hi, hi_inc); } #[inline(always)] unsafe fn flags_vec(flags: [bool; DEGREE]) -> __m256i { set4( flag_word(flags[0]), flag_word(flags[1]), flag_word(flags[2]), flag_word(flags[3]), ) } #[target_feature(enable = "avx2")] pub unsafe fn compress4_loop(jobs: &mut [Job; DEGREE], finalize: Finalize, stride: Stride) { // If we're not finalizing, there can't be a partial block at the end. for job in jobs.iter() { input_debug_asserts(job.input, finalize); } let msg_ptrs = [ jobs[0].input.as_ptr(), jobs[1].input.as_ptr(), jobs[2].input.as_ptr(), jobs[3].input.as_ptr(), ]; let mut h_vecs = transpose_state_vecs(&jobs); let (mut counts_lo, mut counts_hi) = load_counts(&jobs); // Prepare the final blocks (note, which could be empty if the input is // empty). Do all this before entering the main loop. let min_len = jobs.iter().map(|job| job.input.len()).min().unwrap(); let mut fin_offset = min_len.saturating_sub(1); fin_offset -= fin_offset % stride.padded_blockbytes(); // Performance note, making these buffers mem::uninitialized() seems to // cause problems in the optimizer. let mut buf0: [u8; BLOCKBYTES] = [0; BLOCKBYTES]; let mut buf1: [u8; BLOCKBYTES] = [0; BLOCKBYTES]; let mut buf2: [u8; BLOCKBYTES] = [0; BLOCKBYTES]; let mut buf3: [u8; BLOCKBYTES] = [0; BLOCKBYTES]; let (block0, len0, finalize0) = final_block(jobs[0].input, fin_offset, &mut buf0, stride); let (block1, len1, finalize1) = final_block(jobs[1].input, fin_offset, &mut buf1, stride); let (block2, len2, finalize2) = final_block(jobs[2].input, fin_offset, &mut buf2, stride); let (block3, len3, finalize3) = final_block(jobs[3].input, fin_offset, &mut buf3, stride); let fin_blocks: [*const [u8; BLOCKBYTES]; DEGREE] = [block0, block1, block2, block3]; let fin_counts_delta = set4(len0 as Word, len1 as Word, len2 as Word, len3 as Word); let fin_last_block; let fin_last_node; if finalize.yes() { fin_last_block = flags_vec([finalize0, finalize1, finalize2, finalize3]); fin_last_node = flags_vec([ finalize0 && jobs[0].last_node.yes(), finalize1 && jobs[1].last_node.yes(), finalize2 && jobs[2].last_node.yes(), finalize3 && jobs[3].last_node.yes(), ]); } else { fin_last_block = set1(0); fin_last_node = set1(0); } // The main loop. let mut offset = 0; loop { let blocks; let counts_delta; let last_block; let last_node; if offset == fin_offset { blocks = fin_blocks; counts_delta = fin_counts_delta; last_block = fin_last_block; last_node = fin_last_node; } else { blocks = [ msg_ptrs[0].add(offset) as *const [u8; BLOCKBYTES], msg_ptrs[1].add(offset) as *const [u8; BLOCKBYTES], msg_ptrs[2].add(offset) as *const [u8; BLOCKBYTES], msg_ptrs[3].add(offset) as *const [u8; BLOCKBYTES], ]; counts_delta = set1(BLOCKBYTES as Word); last_block = set1(0); last_node = set1(0); }; let m_vecs = transpose_msg_vecs(blocks); add_to_counts(&mut counts_lo, &mut counts_hi, counts_delta); compress4_transposed!( &mut h_vecs, &m_vecs, counts_lo, counts_hi, last_block, last_node, ); // Check for termination before bumping the offset, to avoid overflow. if offset == fin_offset { break; } offset += stride.padded_blockbytes(); } // Write out the results. untranspose_state_vecs(&h_vecs, &mut *jobs); store_counts(&mut *jobs, counts_lo, counts_hi); let max_consumed = offset.saturating_add(stride.padded_blockbytes()); for job in jobs.iter_mut() { let consumed = cmp::min(max_consumed, job.input.len()); job.input = &job.input[consumed..]; } } blake2b_simd-1.0.0/src/blake2bp.rs000064400000000000000000000507540072674642500150220ustar 00000000000000//! BLAKE2bp, a variant of BLAKE2b that uses SIMD more efficiently. //! //! The AVX2 implementation of BLAKE2bp is about twice as fast that of BLAKE2b. //! However, note that it's a different hash function, and it gives a different //! hash from BLAKE2b for the same input. //! //! # Example //! //! ``` //! use blake2b_simd::blake2bp; //! //! let hash = blake2bp::Params::new() //! .hash_length(16) //! .key(b"The Magic Words are Squeamish Ossifrage") //! .to_state() //! .update(b"foo") //! .update(b"bar") //! .update(b"baz") //! .finalize(); //! assert_eq!("e69c7d2c42a5ac14948772231c68c552", &hash.to_hex()); //! ``` use crate::guts::{Finalize, Implementation, Job, LastNode, Stride}; use crate::many; use crate::Count; use crate::Hash; use crate::Word; use crate::BLOCKBYTES; use crate::KEYBYTES; use crate::OUTBYTES; use core::cmp; use core::fmt; use core::mem::size_of; #[cfg(feature = "std")] use std; pub(crate) const DEGREE: usize = 4; /// Compute the BLAKE2bp hash of a slice of bytes all at once, using default /// parameters. /// /// # Example /// /// ``` /// # use blake2b_simd::blake2bp::blake2bp; /// let expected = "8ca9ccee7946afcb686fe7556628b5ba1bf9a691da37ca58cd049354d99f3704\ /// 2c007427e5f219b9ab5063707ec6823872dee413ee014b4d02f2ebb6abb5f643"; /// let hash = blake2bp(b"foo"); /// assert_eq!(expected, &hash.to_hex()); /// ``` pub fn blake2bp(input: &[u8]) -> Hash { Params::new().hash(input) } /// A parameter builder for BLAKE2bp, just like the [`Params`](../struct.Params.html) type for /// BLAKE2b. /// /// This builder only supports configuring the hash length and a secret key. This matches the /// options provided by the [reference /// implementation](https://github.com/BLAKE2/BLAKE2/blob/320c325437539ae91091ce62efec1913cd8093c2/ref/blake2.h#L162-L165). /// /// # Example /// /// ``` /// use blake2b_simd::blake2bp; /// let mut state = blake2bp::Params::new().hash_length(32).to_state(); /// ``` #[derive(Clone)] pub struct Params { hash_length: u8, key_length: u8, key: [u8; KEYBYTES], implementation: Implementation, } impl Params { /// Equivalent to `Params::default()`. pub fn new() -> Self { Self { hash_length: OUTBYTES as u8, key_length: 0, key: [0; KEYBYTES], implementation: Implementation::detect(), } } fn to_words(&self) -> ([[Word; 8]; DEGREE], [Word; 8]) { let mut base_params = crate::Params::new(); base_params .hash_length(self.hash_length as usize) .key(&self.key[..self.key_length as usize]) .fanout(DEGREE as u8) .max_depth(2) .max_leaf_length(0) // Note that inner_hash_length is always OUTBYTES, regardless of the hash_length // parameter. This isn't documented in the spec, but it matches the behavior of the // reference implementation: https://github.com/BLAKE2/BLAKE2/blob/320c325437539ae91091ce62efec1913cd8093c2/ref/blake2bp-ref.c#L55 .inner_hash_length(OUTBYTES); let leaf_words = |worker_index| { base_params .clone() .node_offset(worker_index) .node_depth(0) // Note that setting the last_node flag here has no effect, // because it isn't included in the state words. .to_words() }; let leaf_words = [leaf_words(0), leaf_words(1), leaf_words(2), leaf_words(3)]; let root_words = base_params .clone() .node_offset(0) .node_depth(1) // Note that setting the last_node flag here has no effect, because // it isn't included in the state words. Also note that because // we're only preserving its state words, the root node won't hash // any key bytes. .to_words(); (leaf_words, root_words) } /// Hash an input all at once with these parameters. pub fn hash(&self, input: &[u8]) -> Hash { // If there's a key, just fall back to using the State. if self.key_length > 0 { return self.to_state().update(input).finalize(); } let (mut leaf_words, mut root_words) = self.to_words(); // Hash each leaf in parallel. let jobs = leaf_words.iter_mut().enumerate().map(|(i, words)| { let input_start = cmp::min(input.len(), i * BLOCKBYTES); Job { input: &input[input_start..], words, count: 0, last_node: if i == DEGREE - 1 { LastNode::Yes } else { LastNode::No }, } }); many::compress_many(jobs, self.implementation, Finalize::Yes, Stride::Parallel); // Hash each leaf into the root. finalize_root_words( &leaf_words, &mut root_words, self.hash_length, self.implementation, ) } /// Construct a BLAKE2bp `State` object based on these parameters. pub fn to_state(&self) -> State { State::with_params(self) } /// Set the length of the final hash, from 1 to `OUTBYTES` (64). Apart from controlling the /// length of the final `Hash`, this is also associated data, and changing it will result in a /// totally different hash. pub fn hash_length(&mut self, length: usize) -> &mut Self { assert!( 1 <= length && length <= OUTBYTES, "Bad hash length: {}", length ); self.hash_length = length as u8; self } /// Use a secret key, so that BLAKE2bp acts as a MAC. The maximum key length is `KEYBYTES` /// (64). An empty key is equivalent to having no key at all. pub fn key(&mut self, key: &[u8]) -> &mut Self { assert!(key.len() <= KEYBYTES, "Bad key length: {}", key.len()); self.key_length = key.len() as u8; self.key = [0; KEYBYTES]; self.key[..key.len()].copy_from_slice(key); self } } impl Default for Params { fn default() -> Self { Self::new() } } impl fmt::Debug for Params { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, "Params {{ hash_length: {}, key_length: {} }}", self.hash_length, // NB: Don't print the key itself. Debug shouldn't leak secrets. self.key_length, ) } } /// An incremental hasher for BLAKE2bp, just like the [`State`](../struct.State.html) type for /// BLAKE2b. /// /// # Example /// /// ``` /// use blake2b_simd::blake2bp; /// /// let mut state = blake2bp::State::new(); /// state.update(b"foo"); /// state.update(b"bar"); /// let hash = state.finalize(); /// /// let expected = "e654427b6ef02949471712263e59071abbb6aa94855674c1daeed6cfaf127c33\ /// dfa3205f7f7f71e4f0673d25fa82a368488911f446bccd323af3ab03f53e56e5"; /// assert_eq!(expected, &hash.to_hex()); /// ``` #[derive(Clone)] pub struct State { leaf_words: [[Word; 8]; DEGREE], root_words: [Word; 8], // Note that this buffer is twice as large as what compress4 needs. That guarantees that we // have enough input when we compress to know we don't need to finalize any of the leaves. buf: [u8; 2 * DEGREE * BLOCKBYTES], buf_len: u16, // Note that this is the *per-leaf* count. count: Count, hash_length: u8, implementation: Implementation, is_keyed: bool, } impl State { /// Equivalent to `State::default()` or `Params::default().to_state()`. pub fn new() -> Self { Self::with_params(&Params::default()) } fn with_params(params: &Params) -> Self { let (leaf_words, root_words) = params.to_words(); // If a key is set, initalize the buffer to contain the key bytes. Note // that only the leaves hash key bytes. The root doesn't, even though // the key length it still set in its parameters. Again this isn't // documented in the spec, but it matches the behavior of the reference // implementation: // https://github.com/BLAKE2/BLAKE2/blob/320c325437539ae91091ce62efec1913cd8093c2/ref/blake2bp-ref.c#L128 // This particular behavior (though not the inner hash length behavior // above) is also corroborated by the official test vectors; see // tests/vector_tests.rs. let mut buf = [0; 2 * DEGREE * BLOCKBYTES]; let mut buf_len = 0; if params.key_length > 0 { for i in 0..DEGREE { let keybytes = ¶ms.key[..params.key_length as usize]; buf[i * BLOCKBYTES..][..keybytes.len()].copy_from_slice(keybytes); buf_len = BLOCKBYTES * DEGREE; } } Self { leaf_words, root_words, buf, buf_len: buf_len as u16, count: 0, // count gets updated in self.compress() hash_length: params.hash_length, implementation: params.implementation, is_keyed: params.key_length > 0, } } fn fill_buf(&mut self, input: &mut &[u8]) { let take = cmp::min(self.buf.len() - self.buf_len as usize, input.len()); self.buf[self.buf_len as usize..][..take].copy_from_slice(&input[..take]); self.buf_len += take as u16; *input = &input[take..]; } fn compress_to_leaves( leaves: &mut [[Word; 8]; DEGREE], input: &[u8], count: &mut Count, implementation: Implementation, ) { // Input is assumed to be an even number of blocks for each leaf. Since // we're not finilizing, debug asserts will fire otherwise. let jobs = leaves.iter_mut().enumerate().map(|(i, words)| { Job { input: &input[i * BLOCKBYTES..], words, count: *count, last_node: LastNode::No, // irrelevant when not finalizing } }); many::compress_many(jobs, implementation, Finalize::No, Stride::Parallel); // Note that count is the bytes input *per-leaf*. *count = count.wrapping_add((input.len() / DEGREE) as Count); } /// Add input to the hash. You can call `update` any number of times. pub fn update(&mut self, mut input: &[u8]) -> &mut Self { // If we have a partial buffer, try to complete it. If we complete it and there's more // input waiting, we need to compress to make more room. However, because we need to be // sure that *none* of the leaves would need to be finalized as part of this round of // compression, we need to buffer more than we would for BLAKE2b. if self.buf_len > 0 { self.fill_buf(&mut input); // The buffer is large enough for two compressions. If we've filled // the buffer and there's still more input coming, then we have to // do at least one compression. If there's enough input still // coming that all the leaves are guaranteed to get more, do both // compressions in the buffer. Otherwise, do just one and shift the // back half of the buffer to the front. if !input.is_empty() { if input.len() > (DEGREE - 1) * BLOCKBYTES { // Enough input coming to do both compressions. Self::compress_to_leaves( &mut self.leaf_words, &self.buf, &mut self.count, self.implementation, ); self.buf_len = 0; } else { // Only enough input coming for one compression. Self::compress_to_leaves( &mut self.leaf_words, &self.buf[..DEGREE * BLOCKBYTES], &mut self.count, self.implementation, ); self.buf_len = (DEGREE * BLOCKBYTES) as u16; let (buf_front, buf_back) = self.buf.split_at_mut(DEGREE * BLOCKBYTES); buf_front.copy_from_slice(buf_back); } } } // Now we directly compress as much input as possible, without copying // it into the buffer. We need to make sure we buffer at least one byte // for each of the leaves, so that we know we don't need to finalize // them. let needed_tail = (DEGREE - 1) * BLOCKBYTES + 1; let mut bulk_bytes = input.len().saturating_sub(needed_tail); bulk_bytes -= bulk_bytes % (DEGREE * BLOCKBYTES); if bulk_bytes > 0 { Self::compress_to_leaves( &mut self.leaf_words, &input[..bulk_bytes], &mut self.count, self.implementation, ); input = &input[bulk_bytes..]; } // Buffer any remaining input, to be either compressed or finalized in // a subsequent call. self.fill_buf(&mut input); debug_assert_eq!(0, input.len()); self } /// Finalize the state and return a `Hash`. This method is idempotent, and calling it multiple /// times will give the same result. It's also possible to `update` with more input in between. pub fn finalize(&self) -> Hash { // Hash whatever's remaining in the buffer and finalize the leaves. let buf_len = self.buf_len as usize; let mut leaves_copy = self.leaf_words; let jobs = leaves_copy .iter_mut() .enumerate() .map(|(leaf_index, leaf_words)| { let input = &self.buf[cmp::min(leaf_index * BLOCKBYTES, buf_len)..buf_len]; Job { input, words: leaf_words, count: self.count, last_node: if leaf_index == DEGREE - 1 { LastNode::Yes } else { LastNode::No }, } }); many::compress_many(jobs, self.implementation, Finalize::Yes, Stride::Parallel); // Concatenate each leaf into the root and hash that. let mut root_words_copy = self.root_words; finalize_root_words( &leaves_copy, &mut root_words_copy, self.hash_length, self.implementation, ) } /// Return the total number of bytes input so far. /// /// Note that `count` doesn't include the bytes of the key block, if any. /// It's exactly the total number of input bytes fed to `update`. pub fn count(&self) -> Count { // Remember that self.count is *per-leaf*. let mut ret = self .count .wrapping_mul(DEGREE as Count) .wrapping_add(self.buf_len as Count); if self.is_keyed { ret -= (DEGREE * BLOCKBYTES) as Count; } ret } } #[cfg(feature = "std")] impl std::io::Write for State { fn write(&mut self, buf: &[u8]) -> std::io::Result { self.update(buf); Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } impl fmt::Debug for State { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, "State {{ count: {}, hash_length: {} }}", self.count(), self.hash_length, ) } } impl Default for State { fn default() -> Self { Self::with_params(&Params::default()) } } // Compress each of the four finalized hashes into the root words as input, // using two compressions. Note that even if a future version of this // implementation supports the hash_length parameter and sets it as associated // data for all nodes, this step must still use the untruncated output of each // leaf. Note also that, as mentioned above, the root node doesn't hash any key // bytes. fn finalize_root_words( leaf_words: &[[Word; 8]; DEGREE], root_words: &mut [Word; 8], hash_length: u8, imp: Implementation, ) -> Hash { debug_assert_eq!(OUTBYTES, 8 * size_of::()); let mut block = [0; DEGREE * OUTBYTES]; for (word, chunk) in leaf_words .iter() .flat_map(|words| words.iter()) .zip(block.chunks_exact_mut(size_of::())) { chunk.copy_from_slice(&word.to_le_bytes()); } imp.compress1_loop( &block, root_words, 0, LastNode::Yes, Finalize::Yes, Stride::Serial, ); Hash { bytes: crate::state_words_to_bytes(&root_words), len: hash_length, } } pub(crate) fn force_portable(params: &mut Params) { params.implementation = Implementation::portable(); } #[cfg(test)] pub(crate) mod test { use super::*; use crate::paint_test_input; // This is a simple reference implementation without the complicated buffering or parameter // support of the real implementation. We need this because the official test vectors don't // include any inputs large enough to exercise all the branches in the buffering logic. fn blake2bp_reference(input: &[u8]) -> Hash { let mut leaves = arrayvec::ArrayVec::<_, DEGREE>::new(); for leaf_index in 0..DEGREE { leaves.push( crate::Params::new() .fanout(DEGREE as u8) .max_depth(2) .node_offset(leaf_index as u64) .inner_hash_length(OUTBYTES) .to_state(), ); } leaves[DEGREE - 1].set_last_node(true); for (i, chunk) in input.chunks(BLOCKBYTES).enumerate() { leaves[i % DEGREE].update(chunk); } let mut root = crate::Params::new() .fanout(DEGREE as u8) .max_depth(2) .node_depth(1) .inner_hash_length(OUTBYTES) .last_node(true) .to_state(); for leaf in &mut leaves { root.update(leaf.finalize().as_bytes()); } root.finalize() } #[test] fn test_against_reference() { let mut buf = [0; 21 * BLOCKBYTES]; paint_test_input(&mut buf); // - 8 blocks is just enought to fill the double buffer. // - 9 blocks triggers the "perform one compression on the double buffer" case. // - 11 blocks is the largest input where only one compression may be performed, on the // first half of the buffer, because there's not enough input to avoid needing to // finalize the second half. // - 12 blocks triggers the "perform both compressions in the double buffer" case. // - 15 blocks is the largest input where, after compressing 8 blocks from the buffer, // there's not enough input to hash directly from memory. // - 16 blocks triggers "after emptying the buffer, hash directly from memory". for num_blocks in 0..=20 { for &extra in &[0, 1, BLOCKBYTES - 1] { for &portable in &[false, true] { // eprintln!("\ncase -----"); // dbg!(num_blocks); // dbg!(extra); // dbg!(portable); // First hash the input all at once, as a sanity check. let mut params = Params::new(); if portable { force_portable(&mut params); } let input = &buf[..num_blocks * BLOCKBYTES + extra]; let expected = blake2bp_reference(&input); let mut state = params.to_state(); let found = state.update(input).finalize(); assert_eq!(expected, found); // Then, do it again, but buffer 1 byte of input first. That causes the buffering // branch to trigger. let mut state = params.to_state(); let maybe_one = cmp::min(1, input.len()); state.update(&input[..maybe_one]); assert_eq!(maybe_one as Count, state.count()); // Do a throwaway finalize here to check for idempotency. state.finalize(); state.update(&input[maybe_one..]); assert_eq!(input.len() as Count, state.count()); let found = state.finalize(); assert_eq!(expected, found); // Finally, do it again with the all-at-once interface. assert_eq!(expected, blake2bp(input)); } } } } } blake2b_simd-1.0.0/src/guts.rs000064400000000000000000000454140072674642500143170ustar 00000000000000use crate::*; use arrayref::array_ref; use core::cmp; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub const MAX_DEGREE: usize = 4; #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] pub const MAX_DEGREE: usize = 1; // Variants other than Portable are unreachable in no_std, unless CPU features // are explicitly enabled for the build with e.g. RUSTFLAGS="-C target-feature=avx2". // This might change in the future if is_x86_feature_detected moves into libcore. #[allow(dead_code)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum Platform { Portable, #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] SSE41, #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] AVX2, } #[derive(Clone, Copy, Debug)] pub struct Implementation(Platform); impl Implementation { pub fn detect() -> Self { // Try the different implementations in order of how fast/modern they // are. Currently on non-x86, everything just uses portable. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { if let Some(avx2_impl) = Self::avx2_if_supported() { return avx2_impl; } } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { if let Some(sse41_impl) = Self::sse41_if_supported() { return sse41_impl; } } Self::portable() } pub fn portable() -> Self { Implementation(Platform::Portable) } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[allow(unreachable_code)] pub fn sse41_if_supported() -> Option { // Check whether SSE4.1 support is assumed by the build. #[cfg(target_feature = "sse4.1")] { return Some(Implementation(Platform::SSE41)); } // Otherwise dynamically check for support if we can. #[cfg(feature = "std")] { if is_x86_feature_detected!("sse4.1") { return Some(Implementation(Platform::SSE41)); } } None } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[allow(unreachable_code)] pub fn avx2_if_supported() -> Option { // Check whether AVX2 support is assumed by the build. #[cfg(target_feature = "avx2")] { return Some(Implementation(Platform::AVX2)); } // Otherwise dynamically check for support if we can. #[cfg(feature = "std")] { if is_x86_feature_detected!("avx2") { return Some(Implementation(Platform::AVX2)); } } None } pub fn degree(&self) -> usize { match self.0 { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] Platform::AVX2 => avx2::DEGREE, #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] Platform::SSE41 => sse41::DEGREE, Platform::Portable => 1, } } pub fn compress1_loop( &self, input: &[u8], words: &mut [Word; 8], count: Count, last_node: LastNode, finalize: Finalize, stride: Stride, ) { match self.0 { #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] Platform::AVX2 => unsafe { avx2::compress1_loop(input, words, count, last_node, finalize, stride); }, // Note that there's an SSE version of compress1 in the official C // implementation, but I haven't ported it yet. _ => { portable::compress1_loop(input, words, count, last_node, finalize, stride); } } } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub fn compress2_loop(&self, jobs: &mut [Job; 2], finalize: Finalize, stride: Stride) { match self.0 { Platform::AVX2 | Platform::SSE41 => unsafe { sse41::compress2_loop(jobs, finalize, stride) }, _ => panic!("unsupported"), } } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub fn compress4_loop(&self, jobs: &mut [Job; 4], finalize: Finalize, stride: Stride) { match self.0 { Platform::AVX2 => unsafe { avx2::compress4_loop(jobs, finalize, stride) }, _ => panic!("unsupported"), } } } pub struct Job<'a, 'b> { pub input: &'a [u8], pub words: &'b mut [Word; 8], pub count: Count, pub last_node: LastNode, } impl<'a, 'b> core::fmt::Debug for Job<'a, 'b> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // NB: Don't print the words. Leaking them would allow length extension. write!( f, "Job {{ input_len: {}, count: {}, last_node: {} }}", self.input.len(), self.count, self.last_node.yes(), ) } } // Finalize could just be a bool, but this is easier to read at callsites. #[derive(Clone, Copy, Debug)] pub enum Finalize { Yes, No, } impl Finalize { pub fn yes(&self) -> bool { match self { Finalize::Yes => true, Finalize::No => false, } } } // Like Finalize, this is easier to read at callsites. #[derive(Clone, Copy, Debug)] pub enum LastNode { Yes, No, } impl LastNode { pub fn yes(&self) -> bool { match self { LastNode::Yes => true, LastNode::No => false, } } } #[derive(Clone, Copy, Debug)] pub enum Stride { Serial, // BLAKE2b/BLAKE2s Parallel, // BLAKE2bp/BLAKE2sp } impl Stride { pub fn padded_blockbytes(&self) -> usize { match self { Stride::Serial => BLOCKBYTES, Stride::Parallel => blake2bp::DEGREE * BLOCKBYTES, } } } pub(crate) fn count_low(count: Count) -> Word { count as Word } pub(crate) fn count_high(count: Count) -> Word { (count >> 8 * size_of::()) as Word } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] pub(crate) fn assemble_count(low: Word, high: Word) -> Count { low as Count + ((high as Count) << 8 * size_of::()) } pub(crate) fn flag_word(flag: bool) -> Word { if flag { !0 } else { 0 } } // Pull a array reference at the given offset straight from the input, if // there's a full block of input available. If there's only a partial block, // copy it into the provided buffer, and return an array reference that. Along // with the array, return the number of bytes of real input, and whether the // input can be finalized (i.e. whether there aren't any more bytes after this // block). Note that this is written so that the optimizer can elide bounds // checks, see: https://godbolt.org/z/0hH2bC pub fn final_block<'a>( input: &'a [u8], offset: usize, buffer: &'a mut [u8; BLOCKBYTES], stride: Stride, ) -> (&'a [u8; BLOCKBYTES], usize, bool) { let capped_offset = cmp::min(offset, input.len()); let offset_slice = &input[capped_offset..]; if offset_slice.len() >= BLOCKBYTES { let block = array_ref!(offset_slice, 0, BLOCKBYTES); let should_finalize = offset_slice.len() <= stride.padded_blockbytes(); (block, BLOCKBYTES, should_finalize) } else { // Copy the final block to the front of the block buffer. The rest of // the buffer is assumed to be initialized to zero. buffer[..offset_slice.len()].copy_from_slice(offset_slice); (buffer, offset_slice.len(), true) } } pub fn input_debug_asserts(input: &[u8], finalize: Finalize) { // If we're not finalizing, the input must not be empty, and it must be an // even multiple of the block size. if !finalize.yes() { debug_assert!(!input.is_empty()); debug_assert_eq!(0, input.len() % BLOCKBYTES); } } #[cfg(test)] mod test { use super::*; use core::mem::size_of; #[test] fn test_detection() { assert_eq!(Platform::Portable, Implementation::portable().0); #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[cfg(feature = "std")] { if is_x86_feature_detected!("avx2") { assert_eq!(Platform::AVX2, Implementation::detect().0); assert_eq!( Platform::AVX2, Implementation::avx2_if_supported().unwrap().0 ); assert_eq!( Platform::SSE41, Implementation::sse41_if_supported().unwrap().0 ); } else if is_x86_feature_detected!("sse4.1") { assert_eq!(Platform::SSE41, Implementation::detect().0); assert!(Implementation::avx2_if_supported().is_none()); assert_eq!( Platform::SSE41, Implementation::sse41_if_supported().unwrap().0 ); } else { assert_eq!(Platform::Portable, Implementation::detect().0); assert!(Implementation::avx2_if_supported().is_none()); assert!(Implementation::sse41_if_supported().is_none()); } } } // TODO: Move all of these case tests into the implementation files. fn exercise_cases(mut f: F) where F: FnMut(Stride, usize, LastNode, Finalize, Count), { // Chose counts to hit the relevant overflow cases. let counts = &[ (0 as Count), ((1 as Count) << (8 * size_of::())) - BLOCKBYTES as Count, (0 as Count).wrapping_sub(BLOCKBYTES as Count), ]; for &stride in &[Stride::Serial, Stride::Parallel] { let lengths = [ 0, 1, BLOCKBYTES - 1, BLOCKBYTES, BLOCKBYTES + 1, 2 * BLOCKBYTES - 1, 2 * BLOCKBYTES, 2 * BLOCKBYTES + 1, stride.padded_blockbytes() - 1, stride.padded_blockbytes(), stride.padded_blockbytes() + 1, 2 * stride.padded_blockbytes() - 1, 2 * stride.padded_blockbytes(), 2 * stride.padded_blockbytes() + 1, ]; for &length in &lengths { for &last_node in &[LastNode::No, LastNode::Yes] { for &finalize in &[Finalize::No, Finalize::Yes] { if !finalize.yes() && (length == 0 || length % BLOCKBYTES != 0) { // Skip these cases, they're invalid. continue; } for &count in counts { // eprintln!("\ncase -----"); // dbg!(stride); // dbg!(length); // dbg!(last_node); // dbg!(finalize); // dbg!(count); f(stride, length, last_node, finalize, count); } } } } } } fn initial_test_words(input_index: usize) -> [Word; 8] { crate::Params::new() .node_offset(input_index as u64) .to_words() } // Use the portable implementation, one block at a time, to compute the // final state words expected for a given test case. fn reference_compression( input: &[u8], stride: Stride, last_node: LastNode, finalize: Finalize, mut count: Count, input_index: usize, ) -> [Word; 8] { let mut words = initial_test_words(input_index); let mut offset = 0; while offset == 0 || offset < input.len() { let block_size = cmp::min(BLOCKBYTES, input.len() - offset); let maybe_finalize = if offset + stride.padded_blockbytes() < input.len() { Finalize::No } else { finalize }; portable::compress1_loop( &input[offset..][..block_size], &mut words, count, last_node, maybe_finalize, Stride::Serial, ); offset += stride.padded_blockbytes(); count = count.wrapping_add(BLOCKBYTES as Count); } words } // For various loop lengths and finalization parameters, make sure that the // implementation gives the same answer as the portable implementation does // when invoked one block at a time. (So even the portable implementation // itself is being tested here, to make sure its loop is correct.) Note // that this doesn't include any fixed test vectors; those are taken from // the blake2-kat.json file (copied from upstream) and tested elsewhere. fn exercise_compress1_loop(implementation: Implementation) { let mut input = [0; 100 * BLOCKBYTES]; paint_test_input(&mut input); exercise_cases(|stride, length, last_node, finalize, count| { let reference_words = reference_compression(&input[..length], stride, last_node, finalize, count, 0); let mut test_words = initial_test_words(0); implementation.compress1_loop( &input[..length], &mut test_words, count, last_node, finalize, stride, ); assert_eq!(reference_words, test_words); }); } #[test] fn test_compress1_loop_portable() { exercise_compress1_loop(Implementation::portable()); } #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn test_compress1_loop_sse41() { // Currently this just falls back to portable, but we test it anyway. if let Some(imp) = Implementation::sse41_if_supported() { exercise_compress1_loop(imp); } } #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn test_compress1_loop_avx2() { if let Some(imp) = Implementation::avx2_if_supported() { exercise_compress1_loop(imp); } } // I use ArrayVec everywhere in here becuase currently these tests pass // under no_std. I might decide that's not worth maintaining at some point, // since really all we care about with no_std is that the library builds, // but for now it's here. Everything is keyed off of this N constant so // that it's easy to copy the code to exercise_compress4_loop. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn exercise_compress2_loop(implementation: Implementation) { const N: usize = 2; let mut input_buffer = [0; 100 * BLOCKBYTES]; paint_test_input(&mut input_buffer); let mut inputs = arrayvec::ArrayVec::<_, N>::new(); for i in 0..N { inputs.push(&input_buffer[i..]); } exercise_cases(|stride, length, last_node, finalize, count| { let mut reference_words = arrayvec::ArrayVec::<_, N>::new(); for i in 0..N { let words = reference_compression( &inputs[i][..length], stride, last_node, finalize, count.wrapping_add((i * BLOCKBYTES) as Count), i, ); reference_words.push(words); } let mut test_words = arrayvec::ArrayVec::<_, N>::new(); for i in 0..N { test_words.push(initial_test_words(i)); } let mut jobs = arrayvec::ArrayVec::<_, N>::new(); for (i, words) in test_words.iter_mut().enumerate() { jobs.push(Job { input: &inputs[i][..length], words, count: count.wrapping_add((i * BLOCKBYTES) as Count), last_node, }); } let mut jobs = jobs.into_inner().expect("full"); implementation.compress2_loop(&mut jobs, finalize, stride); for i in 0..N { assert_eq!(reference_words[i], test_words[i], "words {} unequal", i); } }); } #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn test_compress2_loop_sse41() { if let Some(imp) = Implementation::sse41_if_supported() { exercise_compress2_loop(imp); } } #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn test_compress2_loop_avx2() { // Currently this just falls back to SSE4.1, but we test it anyway. if let Some(imp) = Implementation::avx2_if_supported() { exercise_compress2_loop(imp); } } // Copied from exercise_compress2_loop, with a different value of N and an // interior call to compress4_loop. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn exercise_compress4_loop(implementation: Implementation) { const N: usize = 4; let mut input_buffer = [0; 100 * BLOCKBYTES]; paint_test_input(&mut input_buffer); let mut inputs = arrayvec::ArrayVec::<_, N>::new(); for i in 0..N { inputs.push(&input_buffer[i..]); } exercise_cases(|stride, length, last_node, finalize, count| { let mut reference_words = arrayvec::ArrayVec::<_, N>::new(); for i in 0..N { let words = reference_compression( &inputs[i][..length], stride, last_node, finalize, count.wrapping_add((i * BLOCKBYTES) as Count), i, ); reference_words.push(words); } let mut test_words = arrayvec::ArrayVec::<_, N>::new(); for i in 0..N { test_words.push(initial_test_words(i)); } let mut jobs = arrayvec::ArrayVec::<_, N>::new(); for (i, words) in test_words.iter_mut().enumerate() { jobs.push(Job { input: &inputs[i][..length], words, count: count.wrapping_add((i * BLOCKBYTES) as Count), last_node, }); } let mut jobs = jobs.into_inner().expect("full"); implementation.compress4_loop(&mut jobs, finalize, stride); for i in 0..N { assert_eq!(reference_words[i], test_words[i], "words {} unequal", i); } }); } #[test] #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn test_compress4_loop_avx2() { if let Some(imp) = Implementation::avx2_if_supported() { exercise_compress4_loop(imp); } } #[test] fn sanity_check_count_size() { assert_eq!(size_of::(), 2 * size_of::()); } } blake2b_simd-1.0.0/src/lib.rs000064400000000000000000000551560072674642500141070ustar 00000000000000//! [![GitHub](https://img.shields.io/github/tag/oconnor663/blake2_simd.svg?label=GitHub)](https://github.com/oconnor663/blake2_simd) [![crates.io](https://img.shields.io/crates/v/blake2b_simd.svg)](https://crates.io/crates/blake2b_simd) [![Actions Status](https://github.com/oconnor663/blake2_simd/workflows/tests/badge.svg)](https://github.com/oconnor663/blake2_simd/actions) //! //! An implementation of the BLAKE2b and BLAKE2bp hash functions. See also //! [`blake2s_simd`](https://docs.rs/blake2s_simd). //! //! This crate includes: //! //! - 100% stable Rust. //! - SIMD implementations based on Samuel Neves' [`blake2-avx2`](https://github.com/sneves/blake2-avx2). //! These are very fast. For benchmarks, see [the Performance section of the //! README](https://github.com/oconnor663/blake2_simd#performance). //! - Portable, safe implementations for other platforms. //! - Dynamic CPU feature detection. Binaries include multiple implementations by default and //! choose the fastest one the processor supports at runtime. //! - All the features from the [the BLAKE2 spec](https://blake2.net/blake2.pdf), like adjustable //! length, keying, and associated data for tree hashing. //! - `no_std` support. The `std` Cargo feature is on by default, for CPU feature detection and //! for implementing `std::io::Write`. //! - Support for computing multiple BLAKE2b hashes in parallel, matching the efficiency of //! BLAKE2bp. See the [`many`](many/index.html) module. //! //! # Example //! //! ``` //! use blake2b_simd::{blake2b, Params}; //! //! let expected = "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6d\ //! c1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d"; //! let hash = blake2b(b"foo"); //! assert_eq!(expected, &hash.to_hex()); //! //! let hash = Params::new() //! .hash_length(16) //! .key(b"The Magic Words are Squeamish Ossifrage") //! .personal(b"L. P. Waterhouse") //! .to_state() //! .update(b"foo") //! .update(b"bar") //! .update(b"baz") //! .finalize(); //! assert_eq!("ee8ff4e9be887297cf79348dc35dab56", &hash.to_hex()); //! ``` #![cfg_attr(not(feature = "std"), no_std)] use arrayref::{array_refs, mut_array_refs}; use core::cmp; use core::fmt; use core::mem::size_of; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod avx2; mod portable; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] mod sse41; pub mod blake2bp; mod guts; pub mod many; #[cfg(test)] mod test; type Word = u64; type Count = u128; /// The max hash length. pub const OUTBYTES: usize = 8 * size_of::(); /// The max key length. pub const KEYBYTES: usize = 8 * size_of::(); /// The max salt length. pub const SALTBYTES: usize = 2 * size_of::(); /// The max personalization length. pub const PERSONALBYTES: usize = 2 * size_of::(); /// The number input bytes passed to each call to the compression function. Small benchmarks need /// to use an even multiple of `BLOCKBYTES`, or else their apparent throughput will be low. pub const BLOCKBYTES: usize = 16 * size_of::(); const IV: [Word; 8] = [ 0x6A09E667F3BCC908, 0xBB67AE8584CAA73B, 0x3C6EF372FE94F82B, 0xA54FF53A5F1D36F1, 0x510E527FADE682D1, 0x9B05688C2B3E6C1F, 0x1F83D9ABFB41BD6B, 0x5BE0CD19137E2179, ]; const SIGMA: [[u8; 16]; 12] = [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8], [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9], [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10], [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], ]; /// Compute the BLAKE2b hash of a slice of bytes all at once, using default /// parameters. /// /// # Example /// /// ``` /// # use blake2b_simd::{blake2b, Params}; /// let expected = "ca002330e69d3e6b84a46a56a6533fd79d51d97a3bb7cad6c2ff43b354185d6d\ /// c1e723fb3db4ae0737e120378424c714bb982d9dc5bbd7a0ab318240ddd18f8d"; /// let hash = blake2b(b"foo"); /// assert_eq!(expected, &hash.to_hex()); /// ``` pub fn blake2b(input: &[u8]) -> Hash { Params::new().hash(input) } /// A parameter builder that exposes all the non-default BLAKE2 features. /// /// Apart from `hash_length`, which controls the length of the final `Hash`, /// all of these parameters are just associated data that gets mixed with the /// input. For more details, see [the BLAKE2 spec](https://blake2.net/blake2.pdf). /// /// Several of the parameters have a valid range defined in the spec and /// documented below. Trying to set an invalid parameter will panic. /// /// # Example /// /// ``` /// # use blake2b_simd::Params; /// // Create a Params object with a secret key and a non-default length. /// let mut params = Params::new(); /// params.key(b"my secret key"); /// params.hash_length(16); /// /// // Use those params to hash an input all at once. /// let hash = params.hash(b"my input"); /// /// // Or use those params to build an incremental State. /// let mut state = params.to_state(); /// ``` #[derive(Clone)] pub struct Params { hash_length: u8, key_length: u8, key_block: [u8; BLOCKBYTES], salt: [u8; SALTBYTES], personal: [u8; PERSONALBYTES], fanout: u8, max_depth: u8, max_leaf_length: u32, node_offset: u64, node_depth: u8, inner_hash_length: u8, last_node: guts::LastNode, implementation: guts::Implementation, } impl Params { /// Equivalent to `Params::default()`. #[inline] pub fn new() -> Self { Self { hash_length: OUTBYTES as u8, key_length: 0, key_block: [0; BLOCKBYTES], salt: [0; SALTBYTES], personal: [0; PERSONALBYTES], // NOTE: fanout and max_depth don't default to zero! fanout: 1, max_depth: 1, max_leaf_length: 0, node_offset: 0, node_depth: 0, inner_hash_length: 0, last_node: guts::LastNode::No, implementation: guts::Implementation::detect(), } } #[inline(always)] fn to_words(&self) -> [Word; 8] { let (salt_left, salt_right) = array_refs!(&self.salt, SALTBYTES / 2, SALTBYTES / 2); let (personal_left, personal_right) = array_refs!(&self.personal, PERSONALBYTES / 2, PERSONALBYTES / 2); [ IV[0] ^ self.hash_length as u64 ^ (self.key_length as u64) << 8 ^ (self.fanout as u64) << 16 ^ (self.max_depth as u64) << 24 ^ (self.max_leaf_length as u64) << 32, IV[1] ^ self.node_offset, IV[2] ^ self.node_depth as u64 ^ (self.inner_hash_length as u64) << 8, IV[3], IV[4] ^ Word::from_le_bytes(*salt_left), IV[5] ^ Word::from_le_bytes(*salt_right), IV[6] ^ Word::from_le_bytes(*personal_left), IV[7] ^ Word::from_le_bytes(*personal_right), ] } /// Hash an input all at once with these parameters. #[inline] pub fn hash(&self, input: &[u8]) -> Hash { // If there's a key, just fall back to using the State. if self.key_length > 0 { return self.to_state().update(input).finalize(); } let mut words = self.to_words(); self.implementation.compress1_loop( input, &mut words, 0, self.last_node, guts::Finalize::Yes, guts::Stride::Serial, ); Hash { bytes: state_words_to_bytes(&words), len: self.hash_length, } } /// Construct a `State` object based on these parameters, for hashing input /// incrementally. pub fn to_state(&self) -> State { State::with_params(self) } /// Set the length of the final hash in bytes, from 1 to `OUTBYTES` (64). Apart from /// controlling the length of the final `Hash`, this is also associated data, and changing it /// will result in a totally different hash. #[inline] pub fn hash_length(&mut self, length: usize) -> &mut Self { assert!( 1 <= length && length <= OUTBYTES, "Bad hash length: {}", length ); self.hash_length = length as u8; self } /// Use a secret key, so that BLAKE2 acts as a MAC. The maximum key length is `KEYBYTES` (64). /// An empty key is equivalent to having no key at all. #[inline] pub fn key(&mut self, key: &[u8]) -> &mut Self { assert!(key.len() <= KEYBYTES, "Bad key length: {}", key.len()); self.key_length = key.len() as u8; self.key_block = [0; BLOCKBYTES]; self.key_block[..key.len()].copy_from_slice(key); self } /// At most `SALTBYTES` (16). Shorter salts are padded with null bytes. An empty salt is /// equivalent to having no salt at all. #[inline] pub fn salt(&mut self, salt: &[u8]) -> &mut Self { assert!(salt.len() <= SALTBYTES, "Bad salt length: {}", salt.len()); self.salt = [0; SALTBYTES]; self.salt[..salt.len()].copy_from_slice(salt); self } /// At most `PERSONALBYTES` (16). Shorter personalizations are padded with null bytes. An empty /// personalization is equivalent to having no personalization at all. #[inline] pub fn personal(&mut self, personalization: &[u8]) -> &mut Self { assert!( personalization.len() <= PERSONALBYTES, "Bad personalization length: {}", personalization.len() ); self.personal = [0; PERSONALBYTES]; self.personal[..personalization.len()].copy_from_slice(personalization); self } /// From 0 (meaning unlimited) to 255. The default is 1 (meaning sequential). #[inline] pub fn fanout(&mut self, fanout: u8) -> &mut Self { self.fanout = fanout; self } /// From 0 (meaning BLAKE2X B2 hashes), through 1 (the default, meaning sequential) to 255 (meaning unlimited). #[inline] pub fn max_depth(&mut self, depth: u8) -> &mut Self { self.max_depth = depth; self } /// From 0 (the default, meaning unlimited or sequential) to `2^32 - 1`. #[inline] pub fn max_leaf_length(&mut self, length: u32) -> &mut Self { self.max_leaf_length = length; self } /// From 0 (the default, meaning first, leftmost, leaf, or sequential) to `2^64 - 1`. #[inline] pub fn node_offset(&mut self, offset: u64) -> &mut Self { self.node_offset = offset; self } /// From 0 (the default, meaning leaf or sequential) to 255. #[inline] pub fn node_depth(&mut self, depth: u8) -> &mut Self { self.node_depth = depth; self } /// From 0 (the default, meaning sequential) to `OUTBYTES` (64). #[inline] pub fn inner_hash_length(&mut self, length: usize) -> &mut Self { assert!(length <= OUTBYTES, "Bad inner hash length: {}", length); self.inner_hash_length = length as u8; self } /// Indicates the rightmost node in a row. This can also be changed on the /// `State` object, potentially after hashing has begun. See /// [`State::set_last_node`]. /// /// [`State::set_last_node`]: struct.State.html#method.set_last_node #[inline] pub fn last_node(&mut self, last_node: bool) -> &mut Self { self.last_node = if last_node { guts::LastNode::Yes } else { guts::LastNode::No }; self } } impl Default for Params { fn default() -> Self { Self::new() } } impl fmt::Debug for Params { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, "Params {{ hash_length: {}, key_length: {}, salt: {:?}, personal: {:?}, fanout: {}, \ max_depth: {}, max_leaf_length: {}, node_offset: {}, node_depth: {}, \ inner_hash_length: {}, last_node: {} }}", self.hash_length, // NB: Don't print the key itself. Debug shouldn't leak secrets. self.key_length, &self.salt, &self.personal, self.fanout, self.max_depth, self.max_leaf_length, self.node_offset, self.node_depth, self.inner_hash_length, self.last_node.yes(), ) } } /// An incremental hasher for BLAKE2b. /// /// To construct a `State` with non-default parameters, see `Params::to_state`. /// /// # Example /// /// ``` /// use blake2b_simd::{State, blake2b}; /// /// let mut state = blake2b_simd::State::new(); /// /// state.update(b"foo"); /// assert_eq!(blake2b(b"foo"), state.finalize()); /// /// state.update(b"bar"); /// assert_eq!(blake2b(b"foobar"), state.finalize()); /// ``` #[derive(Clone)] pub struct State { words: [Word; 8], count: Count, buf: [u8; BLOCKBYTES], buflen: u8, last_node: guts::LastNode, hash_length: u8, implementation: guts::Implementation, is_keyed: bool, } impl State { /// Equivalent to `State::default()` or `Params::default().to_state()`. pub fn new() -> Self { Self::with_params(&Params::default()) } fn with_params(params: &Params) -> Self { let mut state = Self { words: params.to_words(), count: 0, buf: [0; BLOCKBYTES], buflen: 0, last_node: params.last_node, hash_length: params.hash_length, implementation: params.implementation, is_keyed: params.key_length > 0, }; if state.is_keyed { state.buf = params.key_block; state.buflen = state.buf.len() as u8; } state } fn fill_buf(&mut self, input: &mut &[u8]) { let take = cmp::min(BLOCKBYTES - self.buflen as usize, input.len()); self.buf[self.buflen as usize..self.buflen as usize + take].copy_from_slice(&input[..take]); self.buflen += take as u8; *input = &input[take..]; } // If the state already has some input in its buffer, try to fill the buffer and perform a // compression. However, only do the compression if there's more input coming, otherwise it // will give the wrong hash it the caller finalizes immediately after. fn compress_buffer_if_possible(&mut self, input: &mut &[u8]) { if self.buflen > 0 { self.fill_buf(input); if !input.is_empty() { self.implementation.compress1_loop( &self.buf, &mut self.words, self.count, self.last_node, guts::Finalize::No, guts::Stride::Serial, ); self.count = self.count.wrapping_add(BLOCKBYTES as Count); self.buflen = 0; } } } /// Add input to the hash. You can call `update` any number of times. pub fn update(&mut self, mut input: &[u8]) -> &mut Self { // If we have a partial buffer, try to complete it. self.compress_buffer_if_possible(&mut input); // While there's more than a block of input left (which also means we cleared the buffer // above), compress blocks directly without copying. let mut end = input.len().saturating_sub(1); end -= end % BLOCKBYTES; if end > 0 { self.implementation.compress1_loop( &input[..end], &mut self.words, self.count, self.last_node, guts::Finalize::No, guts::Stride::Serial, ); self.count = self.count.wrapping_add(end as Count); input = &input[end..]; } // Buffer any remaining input, to be either compressed or finalized in a subsequent call. // Note that this represents some copying overhead, which in theory we could avoid in // all-at-once setting. A function hardcoded for exactly BLOCKSIZE input bytes is about 10% // faster than using this implementation for the same input. self.fill_buf(&mut input); self } /// Finalize the state and return a `Hash`. This method is idempotent, and calling it multiple /// times will give the same result. It's also possible to `update` with more input in between. pub fn finalize(&self) -> Hash { let mut words_copy = self.words; self.implementation.compress1_loop( &self.buf[..self.buflen as usize], &mut words_copy, self.count, self.last_node, guts::Finalize::Yes, guts::Stride::Serial, ); Hash { bytes: state_words_to_bytes(&words_copy), len: self.hash_length, } } /// Set a flag indicating that this is the last node of its level in a tree hash. This is /// equivalent to [`Params::last_node`], except that it can be set at any time before calling /// `finalize`. That allows callers to begin hashing a node without knowing ahead of time /// whether it's the last in its level. For more details about the intended use of this flag /// [the BLAKE2 spec]. /// /// [`Params::last_node`]: struct.Params.html#method.last_node /// [the BLAKE2 spec]: https://blake2.net/blake2.pdf pub fn set_last_node(&mut self, last_node: bool) -> &mut Self { self.last_node = if last_node { guts::LastNode::Yes } else { guts::LastNode::No }; self } /// Return the total number of bytes input so far. /// /// Note that `count` doesn't include the bytes of the key block, if any. /// It's exactly the total number of input bytes fed to `update`. pub fn count(&self) -> Count { let mut ret = self.count.wrapping_add(self.buflen as Count); if self.is_keyed { ret -= BLOCKBYTES as Count; } ret } } #[inline(always)] fn state_words_to_bytes(state_words: &[Word; 8]) -> [u8; OUTBYTES] { let mut bytes = [0; OUTBYTES]; { const W: usize = size_of::(); let refs = mut_array_refs!(&mut bytes, W, W, W, W, W, W, W, W); *refs.0 = state_words[0].to_le_bytes(); *refs.1 = state_words[1].to_le_bytes(); *refs.2 = state_words[2].to_le_bytes(); *refs.3 = state_words[3].to_le_bytes(); *refs.4 = state_words[4].to_le_bytes(); *refs.5 = state_words[5].to_le_bytes(); *refs.6 = state_words[6].to_le_bytes(); *refs.7 = state_words[7].to_le_bytes(); } bytes } #[cfg(feature = "std")] impl std::io::Write for State { fn write(&mut self, buf: &[u8]) -> std::io::Result { self.update(buf); Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } impl fmt::Debug for State { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // NB: Don't print the words. Leaking them would allow length extension. write!( f, "State {{ count: {}, hash_length: {}, last_node: {} }}", self.count(), self.hash_length, self.last_node.yes(), ) } } impl Default for State { fn default() -> Self { Self::with_params(&Params::default()) } } type HexString = arrayvec::ArrayString<{ 2 * OUTBYTES }>; /// A finalized BLAKE2 hash, with constant-time equality. #[derive(Clone, Copy)] pub struct Hash { bytes: [u8; OUTBYTES], len: u8, } impl Hash { /// Convert the hash to a byte slice. Note that if you're using BLAKE2 as a MAC, you need /// constant time equality, which `&[u8]` doesn't provide. pub fn as_bytes(&self) -> &[u8] { &self.bytes[..self.len as usize] } /// Convert the hash to a byte array. Note that if you're using BLAKE2 as a /// MAC, you need constant time equality, which arrays don't provide. This /// panics in debug mode if the length of the hash isn't `OUTBYTES`. #[inline] pub fn as_array(&self) -> &[u8; OUTBYTES] { debug_assert_eq!(self.len as usize, OUTBYTES); &self.bytes } /// Convert the hash to a lowercase hexadecimal /// [`ArrayString`](https://docs.rs/arrayvec/0.7/arrayvec/struct.ArrayString.html). pub fn to_hex(&self) -> HexString { bytes_to_hex(self.as_bytes()) } } fn bytes_to_hex(bytes: &[u8]) -> HexString { let mut s = arrayvec::ArrayString::new(); let table = b"0123456789abcdef"; for &b in bytes { s.push(table[(b >> 4) as usize] as char); s.push(table[(b & 0xf) as usize] as char); } s } impl From<[u8; OUTBYTES]> for Hash { fn from(bytes: [u8; OUTBYTES]) -> Self { Self { bytes, len: OUTBYTES as u8, } } } impl From<&[u8; OUTBYTES]> for Hash { fn from(bytes: &[u8; OUTBYTES]) -> Self { Self::from(*bytes) } } /// This implementation is constant time, if the two hashes are the same length. impl PartialEq for Hash { fn eq(&self, other: &Hash) -> bool { constant_time_eq::constant_time_eq(&self.as_bytes(), &other.as_bytes()) } } /// This implementation is constant time, if the slice is the same length as the hash. impl PartialEq<[u8]> for Hash { fn eq(&self, other: &[u8]) -> bool { constant_time_eq::constant_time_eq(&self.as_bytes(), other) } } impl Eq for Hash {} impl AsRef<[u8]> for Hash { fn as_ref(&self) -> &[u8] { self.as_bytes() } } impl fmt::Debug for Hash { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "Hash(0x{})", self.to_hex()) } } // Paint a byte pattern that won't repeat, so that we don't accidentally miss // buffer offset bugs. This is the same as what Bao uses in its tests. #[cfg(test)] fn paint_test_input(buf: &mut [u8]) { let mut offset = 0; let mut counter: u32 = 1; while offset < buf.len() { let bytes = counter.to_le_bytes(); let take = cmp::min(bytes.len(), buf.len() - offset); buf[offset..][..take].copy_from_slice(&bytes[..take]); counter += 1; offset += take; } } // This module is pub for internal benchmarks only. Please don't use it. #[doc(hidden)] pub mod benchmarks { use super::*; pub fn force_portable(params: &mut Params) { params.implementation = guts::Implementation::portable(); } pub fn force_portable_blake2bp(params: &mut blake2bp::Params) { blake2bp::force_portable(params); } } blake2b_simd-1.0.0/src/many.rs000064400000000000000000000434030072674642500142750ustar 00000000000000//! Interfaces for hashing multiple inputs at once, using SIMD more //! efficiently. //! //! The throughput of these interfaces is comparable to BLAKE2bp, about twice //! the throughput of regular BLAKE2b when AVX2 is available. //! //! These interfaces can accept any number of inputs, and the implementation //! does its best to parallelize them. In general, the more inputs you can pass //! in at once the better. If you need to batch your inputs in smaller groups, //! see the [`degree`](fn.degree.html) function for a good batch size. //! //! The implementation keeps working in parallel even when inputs are of //! different lengths, by managing a working set of jobs whose input isn't yet //! exhausted. However, if one or two inputs are much longer than the others, //! and they're encountered only at the end, there might not be any remaining //! work to parallelize them with. In this case, sorting the inputs //! longest-first can improve parallelism. //! //! # Example //! //! ``` //! use blake2b_simd::{blake2b, State, many::update_many}; //! //! let mut states = [ //! State::new(), //! State::new(), //! State::new(), //! State::new(), //! ]; //! //! let inputs = [ //! &b"foo"[..], //! &b"bar"[..], //! &b"baz"[..], //! &b"bing"[..], //! ]; //! //! update_many(states.iter_mut().zip(inputs.iter())); //! //! for (state, input) in states.iter_mut().zip(inputs.iter()) { //! assert_eq!(blake2b(input), state.finalize()); //! } //! ``` use crate::guts::{self, Finalize, Implementation, Job, LastNode, Stride}; use crate::state_words_to_bytes; use crate::Count; use crate::Hash; use crate::Params; use crate::State; use crate::Word; use crate::BLOCKBYTES; use arrayvec::ArrayVec; use core::fmt; /// The largest possible value of [`degree`](fn.degree.html) on the target /// platform. /// /// Note that this constant reflects the parallelism degree supported by this /// crate, so it will change over time as support is added or removed. For /// example, when Rust stabilizes AVX-512 support and this crate adds an /// AVX-512 implementation, this constant will double on x86 targets. If that /// implementation is an optional feature (e.g. because it's nightly-only), the /// value of this constant will depend on that optional feature also. pub const MAX_DEGREE: usize = guts::MAX_DEGREE; /// The parallelism degree of the implementation, detected at runtime. If you /// hash your inputs in small batches, making the batch size a multiple of /// `degree` will generally give good performance. /// /// For example, an x86 processor that supports AVX2 can compute four BLAKE2b /// hashes in parallel, so `degree` returns 4 on that machine. If you call /// [`hash_many`] with only three inputs, that's not enough to use the AVX2 /// implementation, and your average throughput will be lower. Likewise if you /// call it with five inputs of equal length, the first four will be hashed in /// parallel with AVX2, but the last one will have to be hashed by itself, and /// again your average throughput will be lower. /// /// As noted in the module level docs, performance is more complicated if your /// inputs are of different lengths. When parallelizing long and short inputs /// together, the longer ones will have bytes left over, and the implementation /// will try to parallelize those leftover bytes with subsequent inputs. The /// more inputs available in that case, the more the implementation will be /// able to parallelize. /// /// If you need a constant batch size, for example to collect inputs in an /// array, see [`MAX_DEGREE`]. /// /// [`hash_many`]: fn.hash_many.html /// [`MAX_DEGREE`]: constant.MAX_DEGREE.html pub fn degree() -> usize { guts::Implementation::detect().degree() } type JobsVec<'a, 'b> = ArrayVec, { guts::MAX_DEGREE }>; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[inline(always)] fn fill_jobs_vec<'a, 'b>( jobs_iter: &mut impl Iterator>, vec: &mut JobsVec<'a, 'b>, target_len: usize, ) { while vec.len() < target_len { if let Some(job) = jobs_iter.next() { vec.push(job); } else { break; } } } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[inline(always)] fn evict_finished<'a, 'b>(vec: &mut JobsVec<'a, 'b>, num_jobs: usize) { // Iterate backwards so that removal doesn't cause an out-of-bounds panic. for i in (0..num_jobs).rev() { // Note that is_empty() is only valid because we know all these jobs // have been run at least once. Otherwise we could confuse the empty // input for a finished job, which would be incorrect. // // Avoid a panic branch here in release mode. debug_assert!(vec.len() > i); if vec.len() > i && vec[i].input.is_empty() { // Note that calling pop_at() repeatedly has some overhead, because // later elements need to be shifted up. However, the JobsVec is // small, and this approach guarantees that jobs are encountered in // order. vec.pop_at(i); } } } pub(crate) fn compress_many<'a, 'b, I>( jobs: I, imp: Implementation, finalize: Finalize, stride: Stride, ) where I: IntoIterator>, { // Fuse is important for correctness, since each of these blocks tries to // advance the iterator, even if a previous block emptied it. #[allow(unused_mut)] let mut jobs_iter = jobs.into_iter().fuse(); #[allow(unused_mut)] let mut jobs_vec = JobsVec::new(); #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] if imp.degree() >= 4 { loop { fill_jobs_vec(&mut jobs_iter, &mut jobs_vec, 4); if jobs_vec.len() < 4 { break; } let jobs_array = arrayref::array_mut_ref!(jobs_vec, 0, 4); imp.compress4_loop(jobs_array, finalize, stride); evict_finished(&mut jobs_vec, 4); } } #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] if imp.degree() >= 2 { loop { fill_jobs_vec(&mut jobs_iter, &mut jobs_vec, 2); if jobs_vec.len() < 2 { break; } let jobs_array = arrayref::array_mut_ref!(jobs_vec, 0, 2); imp.compress2_loop(jobs_array, finalize, stride); evict_finished(&mut jobs_vec, 2); } } for job in jobs_vec.into_iter().chain(jobs_iter) { let Job { input, words, count, last_node, } = job; imp.compress1_loop(input, words, count, last_node, finalize, stride); } } /// Update any number of `State` objects at once. /// /// # Example /// /// ``` /// use blake2b_simd::{blake2b, State, many::update_many}; /// /// let mut states = [ /// State::new(), /// State::new(), /// State::new(), /// State::new(), /// ]; /// /// let inputs = [ /// &b"foo"[..], /// &b"bar"[..], /// &b"baz"[..], /// &b"bing"[..], /// ]; /// /// update_many(states.iter_mut().zip(inputs.iter())); /// /// for (state, input) in states.iter_mut().zip(inputs.iter()) { /// assert_eq!(blake2b(input), state.finalize()); /// } /// ``` pub fn update_many<'a, 'b, I, T>(pairs: I) where I: IntoIterator, T: 'b + AsRef<[u8]> + ?Sized, { // Get the guts::Implementation from the first state, if any. let mut peekable_pairs = pairs.into_iter().peekable(); let implementation = if let Some((state, _)) = peekable_pairs.peek() { state.implementation } else { // No work items, just short circuit. return; }; // Adapt the pairs iterator into a Jobs iterator, but skip over the Jobs // where there's not actually any work to do (e.g. because there's not much // input and it's all just going in the State buffer). let jobs = peekable_pairs.flat_map(|(state, input_t)| { let mut input = input_t.as_ref(); // For each pair, if the State has some input in its buffer, try to // finish that buffer. If there wasn't enough input to do that -- // or if the input was empty to begin with -- skip this pair. state.compress_buffer_if_possible(&mut input); if input.is_empty() { return None; } // Now we know the buffer is empty and there's more input. Make sure we // buffer the final block, because update() doesn't finalize. let mut last_block_start = input.len() - 1; last_block_start -= last_block_start % BLOCKBYTES; let (blocks, last_block) = input.split_at(last_block_start); state.buf[..last_block.len()].copy_from_slice(last_block); state.buflen = last_block.len() as u8; // Finally, if the full blocks slice is non-empty, prepare that job for // compression, and bump the State count. if blocks.is_empty() { None } else { let count = state.count; state.count = state.count.wrapping_add(blocks.len() as Count); Some(Job { input: blocks, words: &mut state.words, count, last_node: state.last_node, }) } }); // Run all the Jobs in the iterator. compress_many(jobs, implementation, Finalize::No, Stride::Serial); } /// A job for the [`hash_many`] function. After calling [`hash_many`] on a /// collection of `HashManyJob` objects, you can call [`to_hash`] on each job /// to get the result. /// /// [`hash_many`]: fn.hash_many.html /// [`to_hash`]: struct.HashManyJob.html#method.to_hash #[derive(Clone)] pub struct HashManyJob<'a> { words: [Word; 8], count: Count, last_node: LastNode, hash_length: u8, input: &'a [u8], finished: bool, implementation: guts::Implementation, } impl<'a> HashManyJob<'a> { /// Construct a new `HashManyJob` from a set of hashing parameters and an /// input. #[inline] pub fn new(params: &Params, input: &'a [u8]) -> Self { let mut words = params.to_words(); let mut count = 0; let mut finished = false; // If we have key bytes, compress them into the state words. If there's // no additional input, this compression needs to finalize and set // finished=true. if params.key_length > 0 { let mut finalization = Finalize::No; if input.is_empty() { finalization = Finalize::Yes; finished = true; } params.implementation.compress1_loop( ¶ms.key_block, &mut words, 0, params.last_node, finalization, Stride::Serial, ); count = BLOCKBYTES as Count; } Self { words, count, last_node: params.last_node, hash_length: params.hash_length, input, finished, implementation: params.implementation, } } /// Get the hash from a finished job. If you call this before calling /// [`hash_many`], it will panic in debug mode. /// /// [`hash_many`]: fn.hash_many.html #[inline] pub fn to_hash(&self) -> Hash { debug_assert!(self.finished, "job hasn't been run yet"); Hash { bytes: state_words_to_bytes(&self.words), len: self.hash_length, } } } impl<'a> fmt::Debug for HashManyJob<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // NB: Don't print the words. Leaking them would allow length extension. write!( f, "HashManyJob {{ count: {}, hash_length: {}, last_node: {}, input_len: {} }}", self.count, self.hash_length, self.last_node.yes(), self.input.len(), ) } } /// Hash any number of complete inputs all at once. /// /// This is slightly more efficient than using `update_many` with `State` /// objects, because it doesn't need to do any buffering. /// /// Running `hash_many` on the same `HashManyJob` object more than once has no /// effect. /// /// # Example /// /// ``` /// use blake2b_simd::{blake2b, Params, many::{HashManyJob, hash_many}}; /// /// let inputs = [ /// &b"foo"[..], /// &b"bar"[..], /// &b"baz"[..], /// &b"bing"[..], /// ]; /// /// let mut params = Params::new(); /// params.hash_length(16); /// /// let mut jobs = [ /// HashManyJob::new(¶ms, inputs[0]), /// HashManyJob::new(¶ms, inputs[1]), /// HashManyJob::new(¶ms, inputs[2]), /// HashManyJob::new(¶ms, inputs[3]), /// ]; /// /// hash_many(jobs.iter_mut()); /// /// for (input, job) in inputs.iter().zip(jobs.iter()) { /// let expected = params.hash(input); /// assert_eq!(expected, job.to_hash()); /// } /// ``` pub fn hash_many<'a, 'b, I>(hash_many_jobs: I) where 'b: 'a, I: IntoIterator>, { // Get the guts::Implementation from the first job, if any. let mut peekable_jobs = hash_many_jobs.into_iter().peekable(); let implementation = if let Some(job) = peekable_jobs.peek() { job.implementation } else { // No work items, just short circuit. return; }; // In the jobs iterator, skip HashManyJobs that have already been run. This // is less because we actually expect callers to call hash_many twice // (though they're allowed to if they want), and more because // HashManyJob::new might need to finalize if there are key bytes but no // input. Tying the job lifetime to the Params reference is an alternative, // but I've found it too constraining in practice. We could also put key // bytes in every HashManyJob, but that would add unnecessary storage and // zeroing for all callers. let unfinished_jobs = peekable_jobs.into_iter().filter(|j| !j.finished); let jobs = unfinished_jobs.map(|j| { j.finished = true; Job { input: j.input, words: &mut j.words, count: j.count, last_node: j.last_node, } }); compress_many(jobs, implementation, Finalize::Yes, Stride::Serial); } #[cfg(test)] mod test { use super::*; use crate::guts; use crate::paint_test_input; use crate::BLOCKBYTES; use arrayvec::ArrayVec; #[test] fn test_degree() { assert!(degree() <= MAX_DEGREE); #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] #[cfg(feature = "std")] { if is_x86_feature_detected!("avx2") { assert!(degree() >= 4); } if is_x86_feature_detected!("sse4.1") { assert!(degree() >= 2); } } } #[test] fn test_hash_many() { // Use a length of inputs that will exercise all of the power-of-two loops. const LEN: usize = 2 * guts::MAX_DEGREE - 1; // Rerun LEN inputs LEN different times, with the empty input starting in a // different spot each time. let mut input = [0; LEN * BLOCKBYTES]; paint_test_input(&mut input); for start_offset in 0..LEN { let mut inputs: [&[u8]; LEN] = [&[]; LEN]; for i in 0..LEN { let chunks = (i + start_offset) % LEN; inputs[i] = &input[..chunks * BLOCKBYTES]; } let mut params: ArrayVec = ArrayVec::new(); for i in 0..LEN { let mut p = Params::new(); p.node_offset(i as u64); if i % 2 == 1 { p.last_node(true); p.key(b"foo"); } params.push(p); } let mut jobs: ArrayVec = ArrayVec::new(); for i in 0..LEN { jobs.push(HashManyJob::new(¶ms[i], inputs[i])); } hash_many(&mut jobs); // Check the outputs. for i in 0..LEN { let expected = params[i].hash(inputs[i]); assert_eq!(expected, jobs[i].to_hash()); } } } #[test] fn test_update_many() { // Use a length of inputs that will exercise all of the power-of-two loops. const LEN: usize = 2 * guts::MAX_DEGREE - 1; // Rerun LEN inputs LEN different times, with the empty input starting in a // different spot each time. let mut input = [0; LEN * BLOCKBYTES]; paint_test_input(&mut input); for start_offset in 0..LEN { let mut inputs: [&[u8]; LEN] = [&[]; LEN]; for i in 0..LEN { let chunks = (i + start_offset) % LEN; inputs[i] = &input[..chunks * BLOCKBYTES]; } let mut params: ArrayVec = ArrayVec::new(); for i in 0..LEN { let mut p = Params::new(); p.node_offset(i as u64); if i % 2 == 1 { p.last_node(true); p.key(b"foo"); } params.push(p); } let mut states: ArrayVec = ArrayVec::new(); for i in 0..LEN { states.push(params[i].to_state()); } // Run each input twice through, to exercise buffering. update_many(states.iter_mut().zip(inputs.iter())); update_many(states.iter_mut().zip(inputs.iter())); // Check the outputs. for i in 0..LEN { let mut reference_state = params[i].to_state(); // Again, run the input twice. reference_state.update(inputs[i]); reference_state.update(inputs[i]); assert_eq!(reference_state.finalize(), states[i].finalize()); assert_eq!(2 * inputs[i].len() as Count, states[i].count()); } } } } blake2b_simd-1.0.0/src/portable.rs000064400000000000000000000124030072674642500151350ustar 00000000000000use arrayref::{array_ref, array_refs}; use super::*; use crate::guts::{ count_high, count_low, final_block, flag_word, input_debug_asserts, Finalize, LastNode, Stride, }; // G is the mixing function, called eight times per round in the compression // function. V is the 16-word state vector of the compression function, usually // described as a 4x4 matrix. A, B, C, and D are the mixing indices, set by the // caller first to the four columns of V, and then to its four diagonals. X and // Y are words of input, chosen by the caller according to the message // schedule, SIGMA. #[inline(always)] fn g(v: &mut [Word; 16], a: usize, b: usize, c: usize, d: usize, x: Word, y: Word) { v[a] = v[a].wrapping_add(v[b]).wrapping_add(x); v[d] = (v[d] ^ v[a]).rotate_right(32); v[c] = v[c].wrapping_add(v[d]); v[b] = (v[b] ^ v[c]).rotate_right(24); v[a] = v[a].wrapping_add(v[b]).wrapping_add(y); v[d] = (v[d] ^ v[a]).rotate_right(16); v[c] = v[c].wrapping_add(v[d]); v[b] = (v[b] ^ v[c]).rotate_right(63); } // This is too much inlining for some small chips like ARM Cortex-M0, so the // uninline_portable feature is provided to disable it. #[cfg_attr(not(feature = "uninline_portable"), inline(always))] fn round(r: usize, m: &[Word; 16], v: &mut [Word; 16]) { // Select the message schedule based on the round. let s = SIGMA[r]; // Mix the columns. g(v, 0, 4, 8, 12, m[s[0] as usize], m[s[1] as usize]); g(v, 1, 5, 9, 13, m[s[2] as usize], m[s[3] as usize]); g(v, 2, 6, 10, 14, m[s[4] as usize], m[s[5] as usize]); g(v, 3, 7, 11, 15, m[s[6] as usize], m[s[7] as usize]); // Mix the rows. g(v, 0, 5, 10, 15, m[s[8] as usize], m[s[9] as usize]); g(v, 1, 6, 11, 12, m[s[10] as usize], m[s[11] as usize]); g(v, 2, 7, 8, 13, m[s[12] as usize], m[s[13] as usize]); g(v, 3, 4, 9, 14, m[s[14] as usize], m[s[15] as usize]); } #[inline(always)] fn compress_block( block: &[u8; BLOCKBYTES], words: &mut [Word; 8], count: Count, last_block: Word, last_node: Word, ) { // Initialize the compression state. let mut v = [ words[0], words[1], words[2], words[3], words[4], words[5], words[6], words[7], IV[0], IV[1], IV[2], IV[3], IV[4] ^ count_low(count), IV[5] ^ count_high(count), IV[6] ^ last_block, IV[7] ^ last_node, ]; // Parse the message bytes as ints in little endian order. const W: usize = size_of::(); let msg_refs = array_refs!(block, W, W, W, W, W, W, W, W, W, W, W, W, W, W, W, W); let m = [ Word::from_le_bytes(*msg_refs.0), Word::from_le_bytes(*msg_refs.1), Word::from_le_bytes(*msg_refs.2), Word::from_le_bytes(*msg_refs.3), Word::from_le_bytes(*msg_refs.4), Word::from_le_bytes(*msg_refs.5), Word::from_le_bytes(*msg_refs.6), Word::from_le_bytes(*msg_refs.7), Word::from_le_bytes(*msg_refs.8), Word::from_le_bytes(*msg_refs.9), Word::from_le_bytes(*msg_refs.10), Word::from_le_bytes(*msg_refs.11), Word::from_le_bytes(*msg_refs.12), Word::from_le_bytes(*msg_refs.13), Word::from_le_bytes(*msg_refs.14), Word::from_le_bytes(*msg_refs.15), ]; round(0, &m, &mut v); round(1, &m, &mut v); round(2, &m, &mut v); round(3, &m, &mut v); round(4, &m, &mut v); round(5, &m, &mut v); round(6, &m, &mut v); round(7, &m, &mut v); round(8, &m, &mut v); round(9, &m, &mut v); round(10, &m, &mut v); round(11, &m, &mut v); words[0] ^= v[0] ^ v[8]; words[1] ^= v[1] ^ v[9]; words[2] ^= v[2] ^ v[10]; words[3] ^= v[3] ^ v[11]; words[4] ^= v[4] ^ v[12]; words[5] ^= v[5] ^ v[13]; words[6] ^= v[6] ^ v[14]; words[7] ^= v[7] ^ v[15]; } pub fn compress1_loop( input: &[u8], words: &mut [Word; 8], mut count: Count, last_node: LastNode, finalize: Finalize, stride: Stride, ) { input_debug_asserts(input, finalize); let mut local_words = *words; let mut fin_offset = input.len().saturating_sub(1); fin_offset -= fin_offset % stride.padded_blockbytes(); let mut buf = [0; BLOCKBYTES]; let (fin_block, fin_len, _) = final_block(input, fin_offset, &mut buf, stride); let fin_last_block = flag_word(finalize.yes()); let fin_last_node = flag_word(finalize.yes() && last_node.yes()); let mut offset = 0; loop { let block; let count_delta; let last_block; let last_node; if offset == fin_offset { block = fin_block; count_delta = fin_len; last_block = fin_last_block; last_node = fin_last_node; } else { block = array_ref!(input, offset, BLOCKBYTES); count_delta = BLOCKBYTES; last_block = flag_word(false); last_node = flag_word(false); }; count = count.wrapping_add(count_delta as Count); compress_block(block, &mut local_words, count, last_block, last_node); // Check for termination before bumping the offset, to avoid overflow. if offset == fin_offset { break; } offset += stride.padded_blockbytes(); } *words = local_words; } blake2b_simd-1.0.0/src/sse41.rs000064400000000000000000000366000072674642500142710ustar 00000000000000#[cfg(target_arch = "x86")] use core::arch::x86::*; #[cfg(target_arch = "x86_64")] use core::arch::x86_64::*; use crate::guts::{ assemble_count, count_high, count_low, final_block, flag_word, input_debug_asserts, Finalize, Job, Stride, }; use crate::{Word, BLOCKBYTES, IV, SIGMA}; use arrayref::{array_refs, mut_array_refs}; use core::cmp; use core::mem; pub const DEGREE: usize = 2; #[inline(always)] unsafe fn loadu(src: *const [Word; DEGREE]) -> __m128i { // This is an unaligned load, so the pointer cast is allowed. _mm_loadu_si128(src as *const __m128i) } #[inline(always)] unsafe fn storeu(src: __m128i, dest: *mut [Word; DEGREE]) { // This is an unaligned store, so the pointer cast is allowed. _mm_storeu_si128(dest as *mut __m128i, src) } #[inline(always)] unsafe fn add(a: __m128i, b: __m128i) -> __m128i { _mm_add_epi64(a, b) } #[inline(always)] unsafe fn eq(a: __m128i, b: __m128i) -> __m128i { _mm_cmpeq_epi64(a, b) } #[inline(always)] unsafe fn and(a: __m128i, b: __m128i) -> __m128i { _mm_and_si128(a, b) } #[inline(always)] unsafe fn negate_and(a: __m128i, b: __m128i) -> __m128i { // Note that "and not" implies the reverse of the actual arg order. _mm_andnot_si128(a, b) } #[inline(always)] unsafe fn xor(a: __m128i, b: __m128i) -> __m128i { _mm_xor_si128(a, b) } #[inline(always)] unsafe fn set1(x: u64) -> __m128i { _mm_set1_epi64x(x as i64) } #[inline(always)] unsafe fn set2(a: u64, b: u64) -> __m128i { // There's no _mm_setr_epi64x, so note the arg order is backwards. _mm_set_epi64x(b as i64, a as i64) } // Adapted from https://github.com/rust-lang-nursery/stdsimd/pull/479. macro_rules! _MM_SHUFFLE { ($z:expr, $y:expr, $x:expr, $w:expr) => { ($z << 6) | ($y << 4) | ($x << 2) | $w }; } // These rotations are the "simple version". For the "complicated version", see // https://github.com/sneves/blake2-avx2/blob/b3723921f668df09ece52dcd225a36d4a4eea1d9/blake2b-common.h#L43-L46. // For a discussion of the tradeoffs, see // https://github.com/sneves/blake2-avx2/pull/5. In short: // - Due to an LLVM bug (https://bugs.llvm.org/show_bug.cgi?id=44379), this // version performs better on recent x86 chips. // - LLVM is able to optimize this version to AVX-512 rotation instructions // when those are enabled. #[inline(always)] unsafe fn rot32(x: __m128i) -> __m128i { _mm_or_si128(_mm_srli_epi64(x, 32), _mm_slli_epi64(x, 64 - 32)) } #[inline(always)] unsafe fn rot24(x: __m128i) -> __m128i { _mm_or_si128(_mm_srli_epi64(x, 24), _mm_slli_epi64(x, 64 - 24)) } #[inline(always)] unsafe fn rot16(x: __m128i) -> __m128i { _mm_or_si128(_mm_srli_epi64(x, 16), _mm_slli_epi64(x, 64 - 16)) } #[inline(always)] unsafe fn rot63(x: __m128i) -> __m128i { _mm_or_si128(_mm_srli_epi64(x, 63), _mm_slli_epi64(x, 64 - 63)) } #[inline(always)] unsafe fn round(v: &mut [__m128i; 16], m: &[__m128i; 16], r: usize) { v[0] = add(v[0], m[SIGMA[r][0] as usize]); v[1] = add(v[1], m[SIGMA[r][2] as usize]); v[2] = add(v[2], m[SIGMA[r][4] as usize]); v[3] = add(v[3], m[SIGMA[r][6] as usize]); v[0] = add(v[0], v[4]); v[1] = add(v[1], v[5]); v[2] = add(v[2], v[6]); v[3] = add(v[3], v[7]); v[12] = xor(v[12], v[0]); v[13] = xor(v[13], v[1]); v[14] = xor(v[14], v[2]); v[15] = xor(v[15], v[3]); v[12] = rot32(v[12]); v[13] = rot32(v[13]); v[14] = rot32(v[14]); v[15] = rot32(v[15]); v[8] = add(v[8], v[12]); v[9] = add(v[9], v[13]); v[10] = add(v[10], v[14]); v[11] = add(v[11], v[15]); v[4] = xor(v[4], v[8]); v[5] = xor(v[5], v[9]); v[6] = xor(v[6], v[10]); v[7] = xor(v[7], v[11]); v[4] = rot24(v[4]); v[5] = rot24(v[5]); v[6] = rot24(v[6]); v[7] = rot24(v[7]); v[0] = add(v[0], m[SIGMA[r][1] as usize]); v[1] = add(v[1], m[SIGMA[r][3] as usize]); v[2] = add(v[2], m[SIGMA[r][5] as usize]); v[3] = add(v[3], m[SIGMA[r][7] as usize]); v[0] = add(v[0], v[4]); v[1] = add(v[1], v[5]); v[2] = add(v[2], v[6]); v[3] = add(v[3], v[7]); v[12] = xor(v[12], v[0]); v[13] = xor(v[13], v[1]); v[14] = xor(v[14], v[2]); v[15] = xor(v[15], v[3]); v[12] = rot16(v[12]); v[13] = rot16(v[13]); v[14] = rot16(v[14]); v[15] = rot16(v[15]); v[8] = add(v[8], v[12]); v[9] = add(v[9], v[13]); v[10] = add(v[10], v[14]); v[11] = add(v[11], v[15]); v[4] = xor(v[4], v[8]); v[5] = xor(v[5], v[9]); v[6] = xor(v[6], v[10]); v[7] = xor(v[7], v[11]); v[4] = rot63(v[4]); v[5] = rot63(v[5]); v[6] = rot63(v[6]); v[7] = rot63(v[7]); v[0] = add(v[0], m[SIGMA[r][8] as usize]); v[1] = add(v[1], m[SIGMA[r][10] as usize]); v[2] = add(v[2], m[SIGMA[r][12] as usize]); v[3] = add(v[3], m[SIGMA[r][14] as usize]); v[0] = add(v[0], v[5]); v[1] = add(v[1], v[6]); v[2] = add(v[2], v[7]); v[3] = add(v[3], v[4]); v[15] = xor(v[15], v[0]); v[12] = xor(v[12], v[1]); v[13] = xor(v[13], v[2]); v[14] = xor(v[14], v[3]); v[15] = rot32(v[15]); v[12] = rot32(v[12]); v[13] = rot32(v[13]); v[14] = rot32(v[14]); v[10] = add(v[10], v[15]); v[11] = add(v[11], v[12]); v[8] = add(v[8], v[13]); v[9] = add(v[9], v[14]); v[5] = xor(v[5], v[10]); v[6] = xor(v[6], v[11]); v[7] = xor(v[7], v[8]); v[4] = xor(v[4], v[9]); v[5] = rot24(v[5]); v[6] = rot24(v[6]); v[7] = rot24(v[7]); v[4] = rot24(v[4]); v[0] = add(v[0], m[SIGMA[r][9] as usize]); v[1] = add(v[1], m[SIGMA[r][11] as usize]); v[2] = add(v[2], m[SIGMA[r][13] as usize]); v[3] = add(v[3], m[SIGMA[r][15] as usize]); v[0] = add(v[0], v[5]); v[1] = add(v[1], v[6]); v[2] = add(v[2], v[7]); v[3] = add(v[3], v[4]); v[15] = xor(v[15], v[0]); v[12] = xor(v[12], v[1]); v[13] = xor(v[13], v[2]); v[14] = xor(v[14], v[3]); v[15] = rot16(v[15]); v[12] = rot16(v[12]); v[13] = rot16(v[13]); v[14] = rot16(v[14]); v[10] = add(v[10], v[15]); v[11] = add(v[11], v[12]); v[8] = add(v[8], v[13]); v[9] = add(v[9], v[14]); v[5] = xor(v[5], v[10]); v[6] = xor(v[6], v[11]); v[7] = xor(v[7], v[8]); v[4] = xor(v[4], v[9]); v[5] = rot63(v[5]); v[6] = rot63(v[6]); v[7] = rot63(v[7]); v[4] = rot63(v[4]); } // We'd rather make this a regular function with #[inline(always)], but for // some reason that blows up compile times by about 10 seconds, at least in // some cases (BLAKE2b avx2.rs). This macro seems to get the same performance // result, without the compile time issue. macro_rules! compress2_transposed { ( $h_vecs:expr, $msg_vecs:expr, $count_low:expr, $count_high:expr, $lastblock:expr, $lastnode:expr, ) => { let h_vecs: &mut [__m128i; 8] = $h_vecs; let msg_vecs: &[__m128i; 16] = $msg_vecs; let count_low: __m128i = $count_low; let count_high: __m128i = $count_high; let lastblock: __m128i = $lastblock; let lastnode: __m128i = $lastnode; let mut v = [ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], set1(IV[0]), set1(IV[1]), set1(IV[2]), set1(IV[3]), xor(set1(IV[4]), count_low), xor(set1(IV[5]), count_high), xor(set1(IV[6]), lastblock), xor(set1(IV[7]), lastnode), ]; round(&mut v, &msg_vecs, 0); round(&mut v, &msg_vecs, 1); round(&mut v, &msg_vecs, 2); round(&mut v, &msg_vecs, 3); round(&mut v, &msg_vecs, 4); round(&mut v, &msg_vecs, 5); round(&mut v, &msg_vecs, 6); round(&mut v, &msg_vecs, 7); round(&mut v, &msg_vecs, 8); round(&mut v, &msg_vecs, 9); round(&mut v, &msg_vecs, 10); round(&mut v, &msg_vecs, 11); h_vecs[0] = xor(xor(h_vecs[0], v[0]), v[8]); h_vecs[1] = xor(xor(h_vecs[1], v[1]), v[9]); h_vecs[2] = xor(xor(h_vecs[2], v[2]), v[10]); h_vecs[3] = xor(xor(h_vecs[3], v[3]), v[11]); h_vecs[4] = xor(xor(h_vecs[4], v[4]), v[12]); h_vecs[5] = xor(xor(h_vecs[5], v[5]), v[13]); h_vecs[6] = xor(xor(h_vecs[6], v[6]), v[14]); h_vecs[7] = xor(xor(h_vecs[7], v[7]), v[15]); }; } #[inline(always)] unsafe fn transpose_vecs(a: __m128i, b: __m128i) -> [__m128i; DEGREE] { let a_words: [Word; DEGREE] = mem::transmute(a); let b_words: [Word; DEGREE] = mem::transmute(b); [set2(a_words[0], b_words[0]), set2(a_words[1], b_words[1])] } #[inline(always)] unsafe fn transpose_state_vecs(jobs: &[Job; DEGREE]) -> [__m128i; 8] { // Load all the state words into transposed vectors, where the first vector // has the first word of each state, etc. Transposing once at the beginning // and once at the end is more efficient that repeating it for each block. let words0 = array_refs!(&jobs[0].words, DEGREE, DEGREE, DEGREE, DEGREE); let words1 = array_refs!(&jobs[1].words, DEGREE, DEGREE, DEGREE, DEGREE); let [h0, h1] = transpose_vecs(loadu(words0.0), loadu(words1.0)); let [h2, h3] = transpose_vecs(loadu(words0.1), loadu(words1.1)); let [h4, h5] = transpose_vecs(loadu(words0.2), loadu(words1.2)); let [h6, h7] = transpose_vecs(loadu(words0.3), loadu(words1.3)); [h0, h1, h2, h3, h4, h5, h6, h7] } #[inline(always)] unsafe fn untranspose_state_vecs(h_vecs: &[__m128i; 8], jobs: &mut [Job; DEGREE]) { // Un-transpose the updated state vectors back into the caller's arrays. let [job0, job1] = jobs; let words0 = mut_array_refs!(&mut job0.words, DEGREE, DEGREE, DEGREE, DEGREE); let words1 = mut_array_refs!(&mut job1.words, DEGREE, DEGREE, DEGREE, DEGREE); let out = transpose_vecs(h_vecs[0], h_vecs[1]); storeu(out[0], words0.0); storeu(out[1], words1.0); let out = transpose_vecs(h_vecs[2], h_vecs[3]); storeu(out[0], words0.1); storeu(out[1], words1.1); let out = transpose_vecs(h_vecs[4], h_vecs[5]); storeu(out[0], words0.2); storeu(out[1], words1.2); let out = transpose_vecs(h_vecs[6], h_vecs[7]); storeu(out[0], words0.3); storeu(out[1], words1.3); } #[inline(always)] unsafe fn transpose_msg_vecs(blocks: [*const [u8; BLOCKBYTES]; DEGREE]) -> [__m128i; 16] { // These input arrays have no particular alignment, so we use unaligned // loads to read from them. let block0 = blocks[0] as *const [Word; DEGREE]; let block1 = blocks[1] as *const [Word; DEGREE]; let [m0, m1] = transpose_vecs(loadu(block0.add(0)), loadu(block1.add(0))); let [m2, m3] = transpose_vecs(loadu(block0.add(1)), loadu(block1.add(1))); let [m4, m5] = transpose_vecs(loadu(block0.add(2)), loadu(block1.add(2))); let [m6, m7] = transpose_vecs(loadu(block0.add(3)), loadu(block1.add(3))); let [m8, m9] = transpose_vecs(loadu(block0.add(4)), loadu(block1.add(4))); let [m10, m11] = transpose_vecs(loadu(block0.add(5)), loadu(block1.add(5))); let [m12, m13] = transpose_vecs(loadu(block0.add(6)), loadu(block1.add(6))); let [m14, m15] = transpose_vecs(loadu(block0.add(7)), loadu(block1.add(7))); [ m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15, ] } #[inline(always)] unsafe fn load_counts(jobs: &[Job; DEGREE]) -> (__m128i, __m128i) { ( set2(count_low(jobs[0].count), count_low(jobs[1].count)), set2(count_high(jobs[0].count), count_high(jobs[1].count)), ) } #[inline(always)] unsafe fn store_counts(jobs: &mut [Job; DEGREE], low: __m128i, high: __m128i) { let low_ints: [Word; DEGREE] = mem::transmute(low); let high_ints: [Word; DEGREE] = mem::transmute(high); for i in 0..DEGREE { jobs[i].count = assemble_count(low_ints[i], high_ints[i]); } } #[inline(always)] unsafe fn add_to_counts(lo: &mut __m128i, hi: &mut __m128i, delta: __m128i) { // If the low counts reach zero, that means they wrapped, unless the delta // was also zero. *lo = add(*lo, delta); let lo_reached_zero = eq(*lo, set1(0)); let delta_was_zero = eq(delta, set1(0)); let hi_inc = and(set1(1), negate_and(delta_was_zero, lo_reached_zero)); *hi = add(*hi, hi_inc); } #[inline(always)] unsafe fn flags_vec(flags: [bool; DEGREE]) -> __m128i { set2(flag_word(flags[0]), flag_word(flags[1])) } #[target_feature(enable = "sse4.1")] pub unsafe fn compress2_loop(jobs: &mut [Job; DEGREE], finalize: Finalize, stride: Stride) { // If we're not finalizing, there can't be a partial block at the end. for job in jobs.iter() { input_debug_asserts(job.input, finalize); } let msg_ptrs = [jobs[0].input.as_ptr(), jobs[1].input.as_ptr()]; let mut h_vecs = transpose_state_vecs(&jobs); let (mut counts_lo, mut counts_hi) = load_counts(&jobs); // Prepare the final blocks (note, which could be empty if the input is // empty). Do all this before entering the main loop. let min_len = jobs.iter().map(|job| job.input.len()).min().unwrap(); let mut fin_offset = min_len.saturating_sub(1); fin_offset -= fin_offset % stride.padded_blockbytes(); // Performance note, making these buffers mem::uninitialized() seems to // cause problems in the optimizer. let mut buf0: [u8; BLOCKBYTES] = [0; BLOCKBYTES]; let mut buf1: [u8; BLOCKBYTES] = [0; BLOCKBYTES]; let (block0, len0, finalize0) = final_block(jobs[0].input, fin_offset, &mut buf0, stride); let (block1, len1, finalize1) = final_block(jobs[1].input, fin_offset, &mut buf1, stride); let fin_blocks: [*const [u8; BLOCKBYTES]; DEGREE] = [block0, block1]; let fin_counts_delta = set2(len0 as Word, len1 as Word); let fin_last_block; let fin_last_node; if finalize.yes() { fin_last_block = flags_vec([finalize0, finalize1]); fin_last_node = flags_vec([ finalize0 && jobs[0].last_node.yes(), finalize1 && jobs[1].last_node.yes(), ]); } else { fin_last_block = set1(0); fin_last_node = set1(0); } // The main loop. let mut offset = 0; loop { let blocks; let counts_delta; let last_block; let last_node; if offset == fin_offset { blocks = fin_blocks; counts_delta = fin_counts_delta; last_block = fin_last_block; last_node = fin_last_node; } else { blocks = [ msg_ptrs[0].add(offset) as *const [u8; BLOCKBYTES], msg_ptrs[1].add(offset) as *const [u8; BLOCKBYTES], ]; counts_delta = set1(BLOCKBYTES as Word); last_block = set1(0); last_node = set1(0); }; let m_vecs = transpose_msg_vecs(blocks); add_to_counts(&mut counts_lo, &mut counts_hi, counts_delta); compress2_transposed!( &mut h_vecs, &m_vecs, counts_lo, counts_hi, last_block, last_node, ); // Check for termination before bumping the offset, to avoid overflow. if offset == fin_offset { break; } offset += stride.padded_blockbytes(); } // Write out the results. untranspose_state_vecs(&h_vecs, &mut *jobs); store_counts(&mut *jobs, counts_lo, counts_hi); let max_consumed = offset.saturating_add(stride.padded_blockbytes()); for job in jobs.iter_mut() { let consumed = cmp::min(max_consumed, job.input.len()); job.input = &job.input[consumed..]; } } blake2b_simd-1.0.0/src/test.rs000064400000000000000000000134060072674642500143100ustar 00000000000000use super::*; const EMPTY_HASH: &str = "786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419\ d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce"; const ABC_HASH: &str = "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1\ 7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923"; const ONE_BLOCK_HASH: &str = "865939e120e6805438478841afb739ae4250cf372653078a065cdcfffca4caf7\ 98e6d462b65d658fc165782640eded70963449ae1500fb0f24981d7727e22c41"; const THOUSAND_HASH: &str = "1ee4e51ecab5210a518f26150e882627ec839967f19d763e1508b12cfefed148\ 58f6a1c9d1f969bc224dc9440f5a6955277e755b9c513f9ba4421c5e50c8d787"; #[test] fn test_update_state() { let io = &[ (&b""[..], EMPTY_HASH), (&b"abc"[..], ABC_HASH), (&[0; BLOCKBYTES], ONE_BLOCK_HASH), (&[0; 1000], THOUSAND_HASH), ]; // Test each input all at once. for &(input, output) in io { let hash = blake2b(input); assert_eq!(&hash.to_hex(), output, "hash mismatch"); } // Now in two chunks. This is especially important for the ONE_BLOCK case, because it would be // a mistake for update() to call compress, even though the buffer is full. for &(input, output) in io { let mut state = State::new(); let split = input.len() / 2; state.update(&input[..split]); assert_eq!(split as Count, state.count()); state.update(&input[split..]); assert_eq!(input.len() as Count, state.count()); let hash = state.finalize(); assert_eq!(&hash.to_hex(), output, "hash mismatch"); } // Now one byte at a time. for &(input, output) in io { let mut state = State::new(); let mut count = 0; for &b in input { state.update(&[b]); count += 1; assert_eq!(count, state.count()); } let hash = state.finalize(); assert_eq!(&hash.to_hex(), output, "hash mismatch"); } } #[test] fn test_multiple_finalizes() { let mut state = State::new(); assert_eq!(&state.finalize().to_hex(), EMPTY_HASH, "hash mismatch"); assert_eq!(&state.finalize().to_hex(), EMPTY_HASH, "hash mismatch"); assert_eq!(&state.finalize().to_hex(), EMPTY_HASH, "hash mismatch"); state.update(b"abc"); assert_eq!(&state.finalize().to_hex(), ABC_HASH, "hash mismatch"); assert_eq!(&state.finalize().to_hex(), ABC_HASH, "hash mismatch"); assert_eq!(&state.finalize().to_hex(), ABC_HASH, "hash mismatch"); } #[cfg(feature = "std")] #[test] fn test_write() { use std::io::prelude::*; let mut state = State::new(); state.write_all(&[0; 1000]).unwrap(); let hash = state.finalize(); assert_eq!(&hash.to_hex(), THOUSAND_HASH, "hash mismatch"); } // You can check this case against the equivalent Python: // // import hashlib // hashlib.blake2b( // b'foo', // digest_size=18, // key=b"bar", // salt=b"bazbazbazbazbazb", // person=b"bing bing bing b", // fanout=2, // depth=3, // leaf_size=0x04050607, // node_offset=0x08090a0b0c0d0e0f, // node_depth=16, // inner_size=17, // last_node=True, // ).hexdigest() #[test] fn test_all_parameters() { let mut params = Params::new(); params .hash_length(18) // Make sure a shorter key properly overwrites a longer one. .key(b"not the real key") .key(b"bar") .salt(b"bazbazbazbazbazb") .personal(b"bing bing bing b") .fanout(2) .max_depth(3) .max_leaf_length(0x04050607) .node_offset(0x08090a0b0c0d0e0f) .node_depth(16) .inner_hash_length(17) .last_node(true); // Check the State API. assert_eq!( "ec0f59cb65f92e7fcca1280ba859a6925ded", ¶ms.to_state().update(b"foo").finalize().to_hex() ); // Check the all-at-once API. assert_eq!( "ec0f59cb65f92e7fcca1280ba859a6925ded", ¶ms.hash(b"foo").to_hex() ); } #[test] fn test_all_parameters_blake2bp() { let mut params = blake2bp::Params::new(); params .hash_length(18) // Make sure a shorter key properly overwrites a longer one. .key(b"not the real key") .key(b"bar"); // Check the State API. assert_eq!( "8c54e888a8a01c63da6585c058fe54ea81df", ¶ms.to_state().update(b"foo").finalize().to_hex() ); // Check the all-at-once API. assert_eq!( "8c54e888a8a01c63da6585c058fe54ea81df", ¶ms.hash(b"foo").to_hex() ); } #[test] #[should_panic] fn test_short_hash_length_panics() { Params::new().hash_length(0); } #[test] #[should_panic] fn test_long_hash_length_panics() { Params::new().hash_length(OUTBYTES + 1); } #[test] #[should_panic] fn test_long_key_panics() { Params::new().key(&[0; KEYBYTES + 1]); } #[test] #[should_panic] fn test_long_salt_panics() { Params::new().salt(&[0; SALTBYTES + 1]); } #[test] #[should_panic] fn test_long_personal_panics() { Params::new().personal(&[0; PERSONALBYTES + 1]); } #[test] fn test_zero_max_depth_supported() { Params::new().max_depth(0); } #[test] #[should_panic] fn test_long_inner_hash_length_panics() { Params::new().inner_hash_length(OUTBYTES + 1); } #[test] #[should_panic] fn test_blake2bp_short_hash_length_panics() { blake2bp::Params::new().hash_length(0); } #[test] #[should_panic] fn test_blake2bp_long_hash_length_panics() { blake2bp::Params::new().hash_length(OUTBYTES + 1); } #[test] #[should_panic] fn test_blake2bp_long_key_panics() { blake2bp::Params::new().key(&[0; KEYBYTES + 1]); } #[test] fn test_hash_from() { let h = blake2b(b"foo"); assert_eq!(h, Hash::from(h.as_array())); assert_eq!(h, Hash::from(*h.as_array())); }