digest/0000755000176200001440000000000014660651077011545 5ustar liggesusersdigest/tests/0000755000176200001440000000000014462776614012714 5ustar liggesusersdigest/tests/tinytest.R0000644000176200001440000000122114462776614014716 0ustar liggesusers if (requireNamespace("tinytest", quietly=TRUE)) { ## Set a seed to make the test deterministic set.seed(42) ## there are several more granular ways to test files in a ## tinytest directory, see its package vignette; tests can also ## run once the package is installed using the same command ## we need version 0.9.3 or later if (packageVersion("tinytest") >= "0.9.3") { ## expect_length is in tinytest 1.4.1 if (!"expect_length" %in% getNamespaceExports("tinytest")) { expect_length <- function(x, n) tinytest::expect_equal(length(x), n) } tinytest::test_package("digest") } } digest/MD50000644000176200001440000001025514660651077012060 0ustar liggesusers50ae86a3b12b628b9c4a3f09bc2f3b2d *ChangeLog 35f6ff5aaec0857709bc9092ce9ec752 *DESCRIPTION b50ecf9fbca661ae3c3a6a77e2ad55d2 *NAMESPACE 3af8a593fcd744f27b2f1c96703b4086 *R/AES.R 7fb62b92f6907d6426df47ff8bc55939 *R/digest.R 481f6406e42aeaf0eb9da80ebd1c3601 *R/digest2int.R 3afa681f8cae1c57282057525baf3731 *R/hmac.R db9e769e83f9c108e97ce8eb8dd7e825 *R/init.R 14cddbd8465ff30bec5a355855130888 *R/sha1.R 71b80de880320399feb9f51abd8b8f64 *R/vdigest.R a673761518607ae09b8f6357c6304c60 *R/zzz.R d7d8d3b314a5a95b7baf127d7e893a11 *README.md 3e350a70cd41fa837d56a14f78cdc299 *build/vignette.rds b47a60fc7c78d503aba2c3af8d8f3d9a *cleanup cdd17dda453ee280bdbaac3b405874d6 *demo/00Index 247a7f1a523cb39feb2941b00b82906a *demo/vectorised.R b234ee4d69f5fce4486a80fdaf4a4263 *inst/GPL-2 0473d4bde070176922b60166512ee4e7 *inst/doc/sha1.R 452baf57f531c2ec1056756511635aff *inst/doc/sha1.html 6b37c59ba0b533a23e48589c49be6851 *inst/doc/sha1.md 82a62fb6337a2be9ae069fee3d50b614 *inst/include/pmurhashAPI.h d8752af8c06b44fef31655f207b4106f *inst/tinytest/test_aes.R f930ce5c1f9bbcd3b6fe97fd7b5b27f8 *inst/tinytest/test_blake3.R 052184be0e884920438e1779a03f8759 *inst/tinytest/test_crc32.R 54e8706929993b3ebc546fe6ee3b6322 *inst/tinytest/test_digest.R 6681e580fd409258ed4fbdeb77a14cd4 *inst/tinytest/test_digest2int.R 8e8415ddde36eb03eae83e0ae0e5bb0c *inst/tinytest/test_encoding.R 2b3efdc9712ac7f956f79a80da4d0d00 *inst/tinytest/test_hmac.R 557fbd734cbf96e7946b9dca3eaa9afe *inst/tinytest/test_misc.R 40f6e8975785e79091b3a9dffd430e2b *inst/tinytest/test_new_matrix_behaviour.R 1d71bffa7dc302062c44ed92b57bdbcc *inst/tinytest/test_num2hex.R 612e0a4832b86b69c8c04f2916f3b16a *inst/tinytest/test_raw.R 6e3548a6e9bdf2681f6bdc173ee0a8d2 *inst/tinytest/test_sha1.R 5a7722c7c256daabc3e27b343c4321b6 *man/AES.Rd 9445b1d6e1796c0535e911cd3ee21040 *man/digest.Rd d45f3ac5c54a9b0f08ba342b22b9d332 *man/digest2int.Rd 4e1c2885b0335799671cc924991f5471 *man/hmac.Rd 9d566e9d729f574b922fb174f20f1d3b *man/makeRaw.Rd b00315435b5f3bf83ef66af1911270ec *man/sha1.Rd 990744b0bce3f88d9ec0203bc4812a78 *man/vdigest.Rd 8cd655ef39c18515fe615d956b21863d *src/Makevars 8cd655ef39c18515fe615d956b21863d *src/Makevars.win 1fbc5e27d2c0af2b9946288b299e4c79 *src/SpookyV2.cpp 29f0a9285b34812cdf91dcf65d0594c4 *src/SpookyV2.h a529d657388750546c7e03442eb01c31 *src/aes.c 12398d282dd19894ed5548fc55fff7c4 *src/aes.h 5c0685a5951493e00e070f1a154225ba *src/blake3.c 23350b7351de5b42a6aff1bbeadd4338 *src/blake3.h 5d44f6c6e0e8333b0d343beba8c4f772 *src/blake3_dispatch.c ec01dc4c7f8395d7d48ed355818527bc *src/blake3_impl.h a50e559803122fe80a4434dbf0d4b319 *src/blake3_portable.c 8eecbbc3edfec9fd8b0ee3209c1b924f *src/crc32.c 4d1cfa258e52f3f91b09afdba248fec5 *src/crc32.h dd634d9edc40fba14ace4af5a083ed18 *src/crc32c.cpp 51ca266871968cf71820b241bdb27ecc *src/crc32c.h 9173588aa634f1fca88baf44fe4abf5c *src/crc32c/crc32c_config.h 11ec70bd87cb4b8871625eebea7474e0 *src/crc32c/crc32c_internal.h c54ed3e558bdf85c3fc5b2ad51366983 *src/crc32c/crc32c_prefetch.h eb57fc780cb7319f9450a0b2b2e1d367 *src/crc32c/crc32c_read_le.h e2e7031f1ecb4873d9a278c5558449f5 *src/crc32c/crc32c_round_up.h 533014571c97e2ffcd39319c56eb1941 *src/crc32c_portable.cpp ad7e463db44bd7a8378aea6b74ee095a *src/digest.c dc3762ab113fccd9ae068ae763906c89 *src/digest2int.c 93b9220dbdf27296bdf2824923f0b4f0 *src/init.c 972a4374ab3e726afbab89332562daf2 *src/md5.c 9847470291dc6d9a7d86ae83d225745e *src/md5.h 3c8dc8288322e8acc3e0686d8d6d1808 *src/pmurhash.c 798a590806add96318b6cfab0d2805ce *src/pmurhash.h fbba106feeade17ebda82d30c580e40b *src/raes.c 2501b53d87d8f7e46cdbfe7b4f132643 *src/sha1.c 7cf279b5c8a6743e49a2c37d105733f5 *src/sha1.h c320b4628947353a4b1040458e04642c *src/sha2.c 8cb23ad7160a7a9faf1f3b2b6ed3c6bc *src/sha2.h c8089bc1a0cda904888249da6af7f29c *src/sha256.c 82a758ff49a4a2c5dfc82852d57f9a44 *src/sha256.h a2a536db75a78b080a02ae296a4fb9be *src/spooky_serialize.cpp 9f681b5a2cafa7dff67d7e4ac13d1302 *src/xxhash.c c6a5f8d7d22366a65a25aa87b9e33e9a *src/xxhash.h 9f96783529fb4458a873ed787b5d2630 *src/zconf.h 95fdcdaa6c64cf71726e192d5eb1b608 *src/zlib.h c4564a01a95ad6d04fec9c4ef0561cee *src/zutil.h 06e793bdffcc0b843ec921990ce840b2 *tests/tinytest.R 6b37c59ba0b533a23e48589c49be6851 *vignettes/sha1.md c337c0e0c1e0c0dd67a87649ff442d9f *vignettes/water.css digest/R/0000755000176200001440000000000014636007246011742 5ustar liggesusersdigest/R/digest.R0000644000176200001440000001625514636007246013355 0ustar liggesusers ## digest -- hash digest functions for R ## ## Copyright (C) 2003 - 2024 Dirk Eddelbuettel ## Copyright (C) 2009 - 2019 Henrik Bengtsson ## Copyright (C) 2012 - 2019 Hannes Muehleisen ## Copyright (C) 2014 - 2019 Jim Hester ## Copyright (C) 2019 Kendon Bell ## Copyright (C) 2019 Matthew de Queljoe ## ## This file is part of digest. ## ## digest is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## digest is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with digest. If not, see . digest <- function(object, algo=c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "spookyhash", "blake3", "crc32c", "xxh3_64", "xxh3_128"), serialize=TRUE, file=FALSE, length=Inf, skip="auto", ascii=FALSE, raw=FALSE, seed=0, errormode=c("stop","warn","silent"), serializeVersion=.getSerializeVersion()) { # Explicitly specify choices; this is much faster than having match.arg() # infer them from the function's formals. algo <- match.arg(algo, c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "spookyhash", "blake3", "crc32c", "xxh3_64", "xxh3_128")) errormode <- match.arg(errormode, c("stop", "warn", "silent")) if (is.infinite(length)) { length <- -1 # internally we use -1 for infinite len } if (is.character(file) && missing(object)) { object <- file # nocov file <- TRUE # nocov } is_streaming_algo <- algo == "spookyhash" if (is_streaming_algo && !serialize) { .errorhandler(paste0(algo, " algorithm is not available without serialization."), # #nocov mode=errormode) # #nocov } if (serialize && !file) { if (!is_streaming_algo) { ## support the 'nosharing' option in pqR's serialize() object <- if (.hasNoSharing()) serialize (object, connection=NULL, ascii=ascii, nosharing=TRUE, version=serializeVersion) else serialize (object, connection=NULL, ascii=ascii, version=serializeVersion) } ## we support raw vectors, so no mangling of 'object' is necessary ## regardless of R version ## skip="auto" - skips the serialization header [SU] if (is.character(skip) && skip == "auto") skip <- set_skip(object, ascii) } else if (!is.character(object) && !inherits(object,"raw") && !is_streaming_algo) { return(.errorhandler(paste("Argument object must be of type character", # #nocov "or raw vector if serialize is FALSE"), mode=errormode)) # #nocov } if (file && !is.character(object)) return(.errorhandler("file=TRUE can only be used with a character object", # #nocov mode=errormode)) # #nocov if (file && is_streaming_algo) return(.errorhandler(paste0(algo, " algorithm can not be used with files."), # #nocov mode=errormode)) # #nocov ## HB 14 Mar 2007: null op, only turned to char if alreadt char ##if (!inherits(object,"raw")) ## object <- as.character(object) algoint <- algo_int(algo) if (file) { algoint <- algoint+100 object <- path.expand(object) if (.isWindows()) object <- enc2utf8(object) check_file(object, errormode) } ## if skip is auto (or any other text for that matter), we just turn it ## into 0 because auto should have been converted into a number earlier ## if it was valid [SU] if (is.character(skip)) skip <- 0 if (!is_streaming_algo) { val <- .Call(digest_impl, object, as.integer(algoint), as.integer(length), as.integer(skip), as.integer(raw), as.integer(seed)) } else if (algo == "spookyhash"){ # 0s are the seeds. They are included to enable testing against fastdigest. val <- paste(.Call(spookydigest_impl, object, skip, 0, 0, serializeVersion, NULL), collapse="") } ## crc32 output was not guaranteed to be eight chars long, which we corrected ## this allows to get the old behaviour back for compatibility if ((algoint == 3 || algoint == 103) && .getCRC32PreferOldOutput()) { val <- sub("^0+", "", val) } return(val) } ## utility functions used by digest() and getVDigest() below .errorhandler <- function(txt, obj="", mode="stop") { if (mode == "stop") { # nocov start stop(txt, obj, call.=FALSE) } else if (mode == "warn") { warning(txt, obj, call.=FALSE) return(invisible(NA)) } else { return(invisible(NULL)) # nocov end } } algo_int <- function(algo) switch( algo, md5 = 1, sha1 = 2, crc32 = 3, sha256 = 4, sha512 = 5, xxhash32 = 6, xxhash64 = 7, murmur32 = 8, spookyhash = 9, blake3 = 10, crc32c = 11, xxh3_64 = 12, xxh3_128 = 13 ) ## HB 14 Mar 2007: ## Exclude serialization header (non-data dependent bytes but R ## version specific). In ASCII, the header consists of for rows ## ending with a newline ('\n'). We need to skip these. ## The end of 4th row is *typically* within the first 18 bytes set_skip <- function(object, ascii){ if (!ascii) return(14) ## Was: skip <- if (ascii) 18 else 14 which(object[1:30] == as.raw(10))[4] # nocov } check_file <- function(object, errormode){ if (!file.exists(object)) { return(.errorhandler("The file does not exist: ", object, mode=errormode)) # nocov start } if (!isTRUE(!file.info(object)$isdir)) { return(.errorhandler("The specified pathname is not a file: ", object, mode=errormode)) } if (file.access(object, 4)) { return(.errorhandler("The specified file is not readable: ", object, mode=errormode)) # #nocov end } } digest/R/digest2int.R0000644000176200001440000000013113371554565014142 0ustar liggesusersdigest2int <- function(x, seed = 0L) { .Call(digest2int_impl, x, as.integer(seed)) } digest/R/hmac.R0000644000176200001440000000522613546165263013005 0ustar liggesusers ## hmac -- Hash-based message authentication code for R ## ## Copyright (C) 2011 - 2019 Mario Frasca and Dirk Eddelbuettel ## Copyright (C) 2012 - 2019 Hannes Muehleisen and Dirk Eddelbuettel ## ## This file is part of digest. ## ## digest is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## digest is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with digest. If not, see . makeRaw <- function(object) ## generic function, converts an object to a raw UseMethod('makeRaw') makeRaw.raw <- function(object) object makeRaw.character <- function(object) charToRaw(object) # splits a hex-string into the values it contains. makeRaw.digest <- function(object) { parts <- sapply(seq(1, nchar(object), 2), function(i) { substr(object, i, i + 1) }) as.raw(as.hexmode(parts)) } makeRaw.default <- function(object) as.raw(object) # key shall be padded if its length is smaller than the block size of the # respective algorithm, hashed if longer. block sizes are as follows: # # md5 64 # sha1 64 # crc32 4 # sha256 64 # sha512 128 (doh!) padWithZeros <- function(k,algo) { blocksize <- 64 if (algo == "crc32") blocksize <- 4 # #nocov if (algo == "sha512") blocksize <- 128 k <- makeRaw(k) if(length(k) > blocksize) {# not while() k <-digest(k, algo=algo, serialize=FALSE,raw=TRUE) if (algo == "crc32") { k <- substring(k, seq(1,7,2), seq(2,8,2)) # #nocov k <- makeRaw(strtoi(k,16)) # #nocov } } makeRaw(c(k, rep(0, blocksize - length(k)))) } hmac <- function(key, object, algo=c("md5", "sha1", "crc32", "sha256", "sha512"), serialize=FALSE, raw=FALSE, ...) { padded.key <- padWithZeros(key,algo) i.xored.key <- xor(padded.key, makeRaw(0x36)) character.digest <- digest(c(i.xored.key, makeRaw(object)), algo=algo, serialize=serialize, ...) raw.digest <- makeRaw.digest(character.digest) o.xored.key <- xor(padded.key, makeRaw(0x5c)) result <- digest(c(o.xored.key, raw.digest), algo=algo, serialize=serialize, ...) if (raw) result <- makeRaw.digest(result) return(result) } digest/R/AES.R0000644000176200001440000001310514446063606012476 0ustar liggesusers## AES -- Advanced Encryption Standard (AES) block cipher for R ## ## Copyright (C) 2013 - 2019 Duncan Murdoch and Dirk Eddelbuettel ## Copyright (C) 2019 Ion Suruceanu ## ## This file is part of digest. ## ## digest is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## digest is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with digest. If not, see . ## This is loosely modelled on the AES code from the Python Crypto package. ## Currently only ECB, CBC, CFB and CTR modes are supported... modes <- c("ECB", "CBC", "CFB", "PGP", "OFB", "CTR", "OPENPGP") AES <- function(key, mode=c("ECB", "CBC", "CFB", "CTR"), IV=NULL, padding=FALSE) { mode <- match(match.arg(mode), modes) if (!(mode %in% c(1:3, 6))) stop("Only ECB, CBC, CFB and CTR mode encryption are supported.") # #nocov if (padding && mode != 2) stop("Only CBC mode supports padding") # #nocov key <- as.raw(key) IV <- as.raw(IV) context <- .Call(AESinit, key) block_size <- 16 key_size <- length(key) rm(key) encrypt <- function(text) { if (typeof(text) == "character") text <- charToRaw(text) if (mode == 1) .Call(AESencryptECB, context, text) else if (mode == 2) { if (padding) { bytes <- block_size - (length(text) %% block_size) text <- c(text, rep(as.raw(bytes), times = bytes)) } len <- length(text) if (len %% block_size != 0) stop("Text length must be a multiple of ", block_size, " bytes, or use `padding=TRUE`") result <- raw(length(text)) for (i in seq_len(len/block_size)) { ind <- (i-1)*block_size + seq(block_size) IV <<- .Call(AESencryptECB, context, xor(text[ind], IV)) result[ind] <- IV } result } else if (mode == 3) { if (length(IV) != block_size) { stop("IV length must equal block size") } result <- raw(length(text)) blocks <- split(text, ceiling(seq_along(text)/block_size)) i <- 0 for (b in blocks) { if (block_size == length(IV)) { out <- .Call(AESencryptECB, context, IV) } len <- length(b) ind <- i*length(IV)+1:len i <- i + 1 result[ind] <- xor(b, out[0:len]) IV <<- result[ind] } result } else if (mode == 6) { len <- length(text) blocks <- (len + 15) %/% 16 result <- raw(16*blocks) zero <- as.raw(0) for (i in 1:blocks) { result[16*(i-1)+1:16] <- IV byte <- 16 repeat { IV[byte] <<- as.raw((as.integer(IV[byte]) + 1) %% 256) if (IV[byte] != zero || byte == 1) break byte <- byte - 1 } } result <- .Call(AESencryptECB, context, result) length(result) <- len xor(text, result) } } decrypt <- function(ciphertext, raw = FALSE) { if (mode == 1) result <- .Call(AESdecryptECB, context, ciphertext) else if (mode == 2) { len <- length(ciphertext) if (len %% 16 != 0) stop("Ciphertext length must be a multiple of 16 bytes") # #nocov result <- raw(length(ciphertext)) for (i in seq_len(len/16)) { ind <- (i-1)*16 + 1:16 res <- .Call(AESdecryptECB, context, ciphertext[ind]) result[ind] <- xor(res, IV) IV <<- ciphertext[ind] } if (padding) { bytes_to_remove <- as.integer(utils::tail(result, 1)) result <- utils::head(result, -bytes_to_remove) } } else if (mode == 3) { if (length(IV) != block_size) { stop("IV length must equal block size") } result <- raw(length(ciphertext)) blocks <- split(ciphertext, ceiling(seq_along(ciphertext)/block_size)) i <- 0 for (b in blocks) { if (block_size == length(IV)) { out <- .Call(AESencryptECB, context, IV) } len <- length(b) ind <- i*length(IV)+1:len i <- i + 1 result[ind] <- xor(b, out[0:len]) IV <<- b } } else if (mode == 6) result <- encrypt(ciphertext) if (!raw) result <- rawToChar(result) # #nocov result } structure(list(encrypt=encrypt, decrypt=decrypt, block_size=function() block_size, IV=function() IV, key_size=function() key_size, mode=function() modes[mode]), class = "AES") } print.AES <- function(x, ...) # #nocov cat("AES cipher object; mode", x$mode(), "key size", x$key_size(), "\n") # #nocov digest/R/zzz.R0000644000176200001440000000015013017636737012723 0ustar liggesusers.onUnload <- function (libpath) { # nocov start library.dynam.unload("digest", libpath) } # nocov end digest/R/init.R0000644000176200001440000000517314423472174013036 0ustar liggesusers ## digest -- hash digest functions for R ## ## Copyright (C) 2003 - 2023 Dirk Eddelbuettel ## ## This file is part of digest. ## ## digest is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## digest is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with digest. If not, see . .pkgenv <- new.env(parent=emptyenv()) .onLoad <- function(libname, pkgname) { ## we set a default level of two, with a possible override .pkgenv[["serializeVersion"]] <- getOption("serializeVersion", 2L) # #nocov start ## allow old crc32 behaviour .pkgenv[["crc32Preference"]] <- getOption("digestOldCRC32Format", FALSE) ## allow version specific sha1 behaviour .pkgenv[["sha1PackageVersion"]] <- getOption("sha1PackageVersion", packageVersion("digest")) ## cache if we are on Windows as the call is a little expensive (GH issue #137) .pkgenv[["isWindows"]] <- Sys.info()[["sysname"]] == "Windows" ## cache if serialize() supports 'nosharing' .pkgenv[["hasNoSharing"]] <- "nosharing" %in% names(formals(serialize)) # #nocov end } .getSerializeVersion <- function() { ## return the options() value if set, otherwise the package env value ## doing it as a two-step ensure we can set a different default later getOption("serializeVersion", .pkgenv[["serializeVersion"]]) } .getCRC32PreferOldOutput <- function() { ## return the options() value if set, otherwise the package env value ## doing it as a two-step ensure we can set a different default later getOption("digestOldCRC32Format", .pkgenv[["crc32Preference"]]) } .getsha1PackageVersion <- function() { ## return the options() value if set, otherwise the package env value ## doing it as a two-step ensure we can set a different default later package_version( getOption("sha1PackageVersion", .pkgenv[["sha1PackageVersion"]]) ) } .isWindows <- function() { ## return the cached value of Sys.info()[["sysname"]] == "Windows" .pkgenv[["isWindows"]] } .hasNoSharing <- function() { ## return the cached value of "nosharing" %in% names(formals(serialize)) .pkgenv[["hasNoSharing"]] } digest/R/sha1.R0000644000176200001440000003055014573325431012723 0ustar liggesusers ## sha1 -- SHA1 hash generation for R ## ## Copyright (C) 2015 - 2019 Thierry Onkelinx and Dirk Eddelbuettel ## Copyright (C) 2016 - 2019 Viliam Simko ## ## This file is part of digest. ## ## digest is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## digest is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with digest. If not, see . # functions written by Thierry Onkelinx sha1 <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ UseMethod("sha1") } sha1.default <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { if (is.list(x)) { return( sha1.list(x, digits = digits, zapsmall = zapsmall, ..., algo = algo) ) } stop( # #nocov start "sha1() has no method for the '", paste(class(x), collapse = "', '"), "' class", call. = FALSE ) # #nocov end } sha1.numeric <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ y <- num2hex( x, digits = digits, zapsmall = zapsmall ) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } sha1.matrix <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ # needed to make results comparable between 32-bit and 64-bit if (storage.mode(x) == "double") { y <- matrix( #return a matrix with the same dimensions as x apply( x, 2, num2hex, digits = digits, zapsmall = zapsmall ), ncol = ncol(x) ) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } else { attr(x, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(x, algo = algo) } } sha1.complex <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { # a vector of complex numbers is converted into 2-column matrix (Re,Im) y <- cbind(Re(x), Im(x)) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) sha1(y, digits = digits, zapsmall = zapsmall, ..., algo = algo) } sha1.Date <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { y <- as.numeric(x) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) sha1(y, digits = digits, zapsmall = zapsmall, ..., algo = algo) } sha1.array <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { # Array x encoded as list of two elements: # 1. lengths of all dimensions of x # 2. all cells of x as a single vector y <- list( dimension = dim(x), value = as.numeric(x)) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) sha1(y, digits = digits, zapsmall = zapsmall, ..., algo = algo) } sha1.data.frame <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ if (length(x)) { # needed to make results comparable between 32-bit and 64-bit y <- vapply( x, sha1, digits = digits, zapsmall = zapsmall, ..., algo = algo, FUN.VALUE = NA_character_ ) } else { y <- x } y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } sha1.list <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ if (length(x)) { # needed to make results comparable between 32-bit and 64-bit y <- vapply( x, sha1, digits = digits, zapsmall = zapsmall, ..., algo = algo, FUN.VALUE = NA_character_ ) } else { y <- x } y <- add_attributes(x, y) attr(y, "digest::sha1") <- list( class = class(x), digits = as.integer(digits), zapsmall = as.integer(zapsmall), ... = ... ) digest(y, algo = algo) } sha1.POSIXlt <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { y <- do.call( data.frame, lapply(unclass(as.POSIXlt(x)), unlist) ) y$sec <- num2hex(y$sec, digits = digits, zapsmall = zapsmall) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } sha1.POSIXct <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { y <- sha1( as.POSIXlt(x), digits = digits, zapsmall = zapsmall, ..., algo = algo ) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } sha1.anova <- function(x, digits = 4L, zapsmall = 7L, ..., algo = "sha1"){ if (digits > 4) { warning("Hash on 32 bit might be different from hash on 64 bit with digits > 4") # #nocov } y <- apply( x, 1, num2hex, digits = digits, zapsmall = zapsmall ) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } sha1.pairlist <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { # needed to make results comparable between 32-bit and 64-bit y <- vapply( x, sha1, digits = digits, zapsmall = zapsmall, ..., algo = algo, FUN.VALUE = NA_character_ ) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } sha1.function <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ dots <- list(...) if (is.null(dots$environment)) { dots$environment <- TRUE } if (isTRUE(dots$environment)) { y <- list( formals = formals(x), body = as.character(body(x)), environment = digest(environment(x), algo = algo) ) } else { y <- list( formals = formals(x), body = as.character(body(x)) ) } y <- vapply( y, sha1, digits = digits, zapsmall = zapsmall, environment = dots$environment, ... = dots, algo = algo, FUN.VALUE = NA_character_ ) y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = y, digits = digits, zapsmall = zapsmall, algo = algo, dots ) digest(y, algo = algo) } sha1.formula <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1"){ dots <- list(...) if (is.null(dots$environment)) { dots$environment <- TRUE } y <- vapply( x, sha1, digits = digits, zapsmall = zapsmall, ... = dots, algo = algo, FUN.VALUE = NA_character_ ) if (isTRUE(dots$environment)) { y <- c( y, digest(environment(x), algo = algo) ) } y <- add_attributes(x, y) attr(y, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(y, algo = algo) } "sha1.(" <- function(...) {sha1.formula(...)} add_attributes <- function(x, y) { if (.getsha1PackageVersion() < package_version("0.6.22.2")) { return(y) } extra <- attributes(x) if (package_version("0.6.23.2") <= .getsha1PackageVersion()) { extra <- extra[names(extra) != "srcref"] } attributes(y) <- c(attributes(y), "digest::attributes" = extra) return(y) } # sha1_attr_digest variants #### sha1_attr_digest <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { # attributes can be set on objects calling sha1_attr_digest() attr(x, "digest::sha1") <- attr_sha1( x = x, digits = digits, zapsmall = zapsmall, algo = algo, ... ) digest(x, algo = algo) } sha1.call <- function(...) {sha1_attr_digest(...)} sha1.character <- function(...) {sha1_attr_digest(...)} sha1.factor <- function(...) {sha1_attr_digest(...)} sha1.logical <- function(...) {sha1_attr_digest(...)} sha1.integer <- function(...) {sha1_attr_digest(...)} sha1.raw <- function(...) {sha1_attr_digest(...)} # sha1_digest variants #### sha1_digest <- function(x, digits = 14L, zapsmall = 7L, ..., algo = "sha1") { # attributes cannot be set on objects calling sha1_digest() digest(x, algo = algo) } sha1.name <- function(...) {sha1_digest(...)} sha1.NULL <- function(...) {sha1_digest(...)} # Support Functions #### attr_sha1 <- function(x, digits, zapsmall, algo, ...) { if (algo == "sha1") { return( list( class = class(x), digits = as.integer(digits), zapsmall = as.integer(zapsmall), ... ) ) } list( class = class(x), digits = as.integer(digits), zapsmall = as.integer(zapsmall), algo = algo, ... ) } num2hex <- function(x, digits = 14L, zapsmall = 7L){ if (!is.numeric(x)) { stop("x is not numeric") # #nocov } if (!is.integer(digits)) { if (!all.equal(as.integer(digits), digits)) { stop("digits is not integer") # #nocov } digits <- as.integer(digits) } if (length(digits) != 1) { stop("digits must contain exactly one integer") # #nocov } if (digits < 1) { stop("digits must be positive") # #nocov } if (!is.integer(zapsmall)) { if (!all.equal(as.integer(zapsmall), zapsmall)) { stop("zapsmall is not integer") # #nocov } zapsmall <- as.integer(zapsmall) } if (length(zapsmall) != 1) { stop("zapsmall must contain exactly one integer") # #nocov } if (zapsmall < 1) { stop("zapsmall must be positive") # #nocov } if (length(x) == 0) { return(character(0)) } x.na <- is.na(x) if (all(x.na)) { return(x) } x.inf <- is.infinite(x) output <- rep(NA_character_, length(x)) output[x.inf & x > 0] <- "Inf" output[x.inf & x < 0] <- "-Inf" # detect "small" numbers x.zero <- !x.na & !x.inf & abs(x) <= (2^floor(log2(10 ^ -zapsmall))) output[x.zero] <- "0" # The calculations for non-na, non-inf, non-zero values are computationally # more intense. Don't do them unless necessary. x.finite <- !(x.na | x.inf | x.zero) if (!any(x.finite)) { return(output) } x_abs <- abs(x[x.finite]) exponent <- floor(log2(x_abs)) negative <- c("", "-")[(x[x.finite] < 0) + 1] x.hex <- sprintf("%a", x_abs*2^-exponent) nc_x <- nchar(x.hex) digits.hex <- ceiling(log(10 ^ digits, base = 16)) # select mantissa (starting format is 0x1.[0-9a-f]+p[+-][0-9]+), remove the # beginning through the decimal place, including the fact that exact powers # of two will not have a decimal place. # Remove the beginning through the decimal place (if it exists). mask_decimal <- startsWith(x.hex, "0x1.") start_character <- 4 + mask_decimal # select required precision stop_character <- pmin(nc_x - 3, start_character + digits.hex - 1) mantissa <- substring(x.hex, start_character, stop_character) # Drop trailing zeros mantissa <- gsub(x = mantissa, pattern = "0*$", replacement = "") output[x.finite] <- sprintf("%s%s %d", negative, mantissa, exponent) return(output) } digest/R/vdigest.R0000644000176200001440000001642014573367376013552 0ustar liggesusers ## vdigest -- Vectorist digest functions for R ## ## Copyright (C) 2019 - 2024 Matthew de Queljoe and Dirk Eddelbuettel ## ## This file is part of digest. ## ## digest is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 2 of the License, or ## (at your option) any later version. ## ## digest is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with digest. If not, see . getVDigest <- function(algo = c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "spookyhash", "blake3", "crc32c", "xxh3_64", "xxh3_128"), errormode=c("stop","warn","silent")){ algo <- match.arg(algo, c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "spookyhash", "blake3", "crc32c", "xxh3_64", "xxh3_128")) errormode <- match.arg(errormode, c("stop","warn","silent")) algoint <- algo_int(algo) non_streaming_algos <- c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "blake3", "crc32c", "xxh3_64", "xxh3_128") if (algo %in% non_streaming_algos) return(non_streaming_digest(algo, errormode, algoint)) streaming_digest(algo, errormode, algoint) } non_streaming_digest <- function(algo, errormode, algoint){ function(object, serialize=TRUE, file=FALSE, length=Inf, skip="auto", ascii=FALSE, seed=0, serializeVersion=.getSerializeVersion()){ if (is.infinite(length)) length <- -1 # internally we use -1 for infinite len if (is.character(file) && missing(object)) { object <- file # nocov file <- TRUE # nocov } if (serialize && !file) { ## support the 'nosharing' option in pqR's serialize() object <- if (.hasNoSharing()) serialize_( object, connection = NULL, ascii = ascii, nosharing = TRUE, version = serializeVersion ) else serialize_(object, connection = NULL, ascii = ascii, version = serializeVersion) ## we support raw vectors, so no mangling of 'object' is necessary ## regardless of R version ## skip="auto" - skips the serialization header [SU] if (any(!is.na(pmatch(skip,"auto")))) skip <- set_skip(object, ascii) } else if (!is.character(object) && !inherits(object,"raw")) { return(.errorhandler(paste("Argument object must be of type character", # #nocov "or raw vector if serialize is FALSE"), mode=errormode)) # #nocov } if (file && !is.character(object)) return(.errorhandler("file=TRUE can only be used with a character object", # #nocov mode=errormode)) # #nocov if (file) { algoint <- algoint+100 object <- path.expand(object) if (.isWindows()) object <- enc2utf8(object) check_file(object, errormode) } ## if skip is auto (or any other text for that matter), we just turn it ## into 0 because auto should have been converted into a number earlier ## if it was valid [SU] if (is.character(skip)) skip <- 0 val <- .Call( vdigest_impl, object, as.integer(algoint), as.integer(length), as.integer(skip), 0L, # raw always FALSE as.integer(seed) ) ## crc32 output was not guaranteed to be eight chars long, which we corrected ## this allows to get the old behaviour back for compatibility if ((algoint == 3 || algoint == 103) && .getCRC32PreferOldOutput()) { val <- sub("^0+", "", val) # #nocov } return(val) } } streaming_digest <- function(algo, errormode, algoint){ function(object, serialize=TRUE, file=FALSE, length=Inf, skip="auto", ascii=FALSE, seed=0, serializeVersion=.getSerializeVersion()){ if (is.infinite(length)) length <- -1 # internally we use -1 for infinite len if (is.character(file) && missing(object)) { object <- file # nocov file <- TRUE # nocov } if (!serialize){ .errorhandler(paste0(algo, " algorithm is not available without serialization."), # #nocov mode=errormode) # #nocov } if (serialize && !file) { ## we support raw vectors, so no mangling of 'object' is necessary ## regardless of R version ## skip="auto" - skips the serialization header [SU] if (any(!is.na(pmatch(skip,"auto")))) skip <- set_skip(object, ascii) } if (file) return(.errorhandler(paste0(algo, " algorithm can not be used with files."), # #nocov mode=errormode)) # #nocov ## if skip is auto (or any other text for that matter), we just turn it ## into 0 because auto should have been converted into a number earlier ## if it was valid [SU] if (is.character(skip)) skip <- 0 # #nocov if (algo == "spookyhash"){ # 0s are the seeds. They are included to enable testing against fastdigest. val <- vapply(object, function(o) paste( .Call(spookydigest_impl, o, skip, 0, 0, serializeVersion), collapse = "" ), character(1), USE.NAMES = FALSE) } ## crc32 output was not guaranteed to be eight chars long, which we corrected ## this allows to get the old behaviour back for compatibility if ((algoint == 3 || algoint == 103) && .getCRC32PreferOldOutput()) { val <- sub("^0+", "", val) # #nocov } return(val) } } serialize_ <- function(object, ...){ if (length(object)) return(lapply(object, serialize, ...)) serialize(object, ...) } digest/cleanup0000755000176200001440000000015414660633606013120 0ustar liggesusers#!/bin/sh rm -f config.* src/symbols.rds \ src/*.o src/*.d src/*.a src/*.dll src/*.so src/*.rc */*~ *~ digest/demo/0000755000176200001440000000000013540654254012465 5ustar liggesusersdigest/demo/00Index0000644000176200001440000000011613540654254013615 0ustar liggesusersvectorised Timing comparison of approaches for vectorised digest digest/demo/vectorised.R0000644000176200001440000000162613540654254014764 0ustar liggesuserslibrary(digest) #vectorisation md5 <- getVDigest() digest2 <- base::Vectorize(digest) x <- rep(letters, 1e3) rbenchmark::benchmark( vdigest = md5(x, serialize = FALSE), Vectorize = digest2(x, serialize = FALSE), vapply = vapply(x, digest, character(1), serialize = FALSE), replications = 5 )[,1:4] all(md5(x, serialize=FALSE) == digest2(x, serialize=FALSE)) all(md5(x, serialize=FALSE) == vapply(x, digest, character(1), serialize = FALSE)) #repeated calls stretch_key <- function(d, n) { md5 <- getVDigest() for (i in seq_len(n)) d <- md5(d, serialize = FALSE) d } stretch_key2 <- function(d, n) { for (i in seq_len(n)) d <- digest(d, serialize = FALSE) d } rbenchmark::benchmark( vdigest = stretch_key('abc123', 65e3), plaindigest = stretch_key2('abc123', 65e3), replications = 10 )[,1:4] stretch_key('abc123', 65e3) == stretch_key2('abc123', 65e3) digest/vignettes/0000755000176200001440000000000014660633605013552 5ustar liggesusersdigest/vignettes/water.css0000644000176200001440000000225114146242237015402 0ustar liggesusers/* our addition */ body { max-width: 50rem; margin-left: auto; margin-right: auto; font-family: system-ui; } /* Inline codes */ code { padding: 2px; border-radius: unset; } /* Code blocks */ pre { background-color: unset; border: solid #aaa 1px; padding: 8px; } pre.numberSource { margin: 0; padding-left: 0; } div.sourceCode { overflow: visible; } pre, pre.sourceCode { overflow-x: auto; } pre>code { white-space: pre; overflow: visible; background-color: unset; padding: 0; } pre.sourceCode.numberSource { overflow-x: visible; } pre.sourceCode.numberSource>code { white-space: pre-wrap } pre.sourceCode.numberSource>code>span { left: 8px; text-indent: -4.6em; } /* code folding */ .chunk-summary { text-align: right; } .chunk-summary+pre, .chunk-summary+div.sourceCode { margin-top: 2px; } /* TOC */ nav > ul { border: .0625rem solid #444; border-radius: 4px; margin: 5px; padding: 5px; } nav ul { list-style-type: none; padding-inline-start: 1rem; } nav ul li { padding: 0; } nav ul ul { margin-top: 0; margin-bottom: 0; padding-top: 0; padding-bottom: 0; } nav code { background-color: unset; color: unset; } digest/vignettes/sha1.md0000644000176200001440000001620314146242237014726 0ustar liggesusers --- title: "Calculating SHA1 hashes with digest() and sha1()" author: "Thierry Onkelinx and Dirk Eddelbuettel" date: "Written Jan 2016, updated Jan 2018 and Oct 2020" css: "water.css" --- NB: This vignette is (still) work-in-progress and not yet complete. ## Short intro on hashes TBD ## Difference between `digest()` and `sha1()` R [FAQ 7.31](https://cran.r-project.org/doc/FAQ/R-FAQ.html#Why-doesn_0027t-R-think-these-numbers-are-equal_003f) illustrates potential problems with floating point arithmetic. Mathematically the equality $x = \sqrt{x}^2$ should hold. But the precision of floating points numbers is finite. Hence some rounding is done, leading to numbers which are no longer identical. An illustration: ```{#faq7_31 .R} # FAQ 7.31 a0 <- 2 b <- sqrt(a0) a1 <- b ^ 2 identical(a0, a1) a0 - a1 a <- c(a0, a1) # hexadecimal representation sprintf("%a", a) ``` Although the difference is small, any difference will result in different hash when using the `digest()` function. However, the `sha1()` function tackles this problem by using the hexadecimal representation of the numbers and truncates that representation to a certain number of digits prior to calculating the hash function. ```{#faq7_31digest .R} library(digest) # different hashes with digest sapply(a, digest, algo = "sha1") # same hash with sha1 with default digits (14) sapply(a, sha1) # larger digits can lead to different hashes sapply(a, sha1, digits = 15) # decreasing the number of digits gives a stronger truncation # the hash will change when then truncation gives a different result # case where truncating gives same hexadecimal value sapply(a, sha1, digits = 13) sapply(a, sha1, digits = 10) # case where truncating gives different hexadecimal value c(sha1(pi), sha1(pi, digits = 13), sha1(pi, digits = 10)) ``` The result of floating point arithematic on 32-bit and 64-bit can be slightly different. E.g. `print(pi ^ 11, 22)` returns `294204.01797389047` on 32-bit and `294204.01797389053` on 64-bit. Note that only the last 2 digits are different. | command | 32-bit | 64-bit| | - | - | - | | `print(pi ^ 11, 22)` | `294204.01797389047` | `294204.01797389053` | | `sprintf("%a", pi ^ 11)`| `"0x1.1f4f01267bf5fp+18"` | `"0x1.1f4f01267bf6p+18"` | | `digest(pi ^ 11, algo = "sha1")` | `"c5efc7f167df1bb402b27cf9b405d7cebfba339a"` | `"b61f6fea5e2a7952692cefe8bba86a00af3de713"`| | `sha1(pi ^ 11, digits = 14)` | `"5c7740500b8f78ec2354ea6af58ea69634d9b7b1"` | `"4f3e296b9922a7ddece2183b1478d0685609a359"` | | `sha1(pi ^ 11, digits = 13)` | `"372289f87396b0877ccb4790cf40bcb5e658cad7"` | `"372289f87396b0877ccb4790cf40bcb5e658cad7"` | | `sha1(pi ^ 11, digits = 10)` | `"c05965af43f9566bfb5622f335817f674abfc9e4"` | `"c05965af43f9566bfb5622f335817f674abfc9e4"` | ## Choosing `digest()` or `sha1()` TBD ## Creating a sha1 method for other classes ### How to 1. Identify the relevant components for the hash. 1. Determine the class of each relevant component and check if they are handled by `sha1()`. - Write a method for each component class not yet handled by `sha1`. 1. Extract the relevant components. 1. Combine the relevant components into a list. Not required in case of a single component. 1. Apply `sha1()` on the (list of) relevant component(s). 1. Turn this into a function with name sha1._classname_. 1. sha1._classname_ needs exactly the same arguments as `sha1()` 1. Choose sensible defaults for the arguments - `zapsmall = 7` is recommended. - `digits = 14` is recommended in case all numerics are data. - `digits = 4` is recommended in case some numerics stem from floating point arithmetic. ### summary.lm Let's illustrate this using the summary of a simple linear regression. Suppose that we want a hash that takes into account the coefficients, their standard error and sigma. ```{#sha1_lm_sum .R} # taken from the help file of lm.influence lm_SR <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LifeCycleSavings) lm_sum <- summary(lm_SR) class(lm_sum) # str() gives the structure of the lm object str(lm_sum) # extract the coefficients and their standard error coef_sum <- coef(lm_sum)[, c("Estimate", "Std. Error")] # extract sigma sigma <- lm_sum$sigma # check the class of each component class(coef_sum) class(sigma) # sha1() has methods for both matrix and numeric # because the values originate from floating point arithmetic it is better to use a low number of digits sha1(coef_sum, digits = 4) sha1(sigma, digits = 4) # we want a single hash # combining the components in a list is a solution that works sha1(list(coef_sum, sigma), digits = 4) # now turn everything into an S3 method # - a function with name "sha1.classname" # - must have the same arguments as sha1() sha1.summary.lm <- function(x, digits = 4, zapsmall = 7){ coef_sum <- coef(x)[, c("Estimate", "Std. Error")] sigma <- x$sigma combined <- list(coef_sum, sigma) sha1(combined, digits = digits, zapsmall = zapsmall) } sha1(lm_sum) # try an altered dataset LCS2 <- LifeCycleSavings[rownames(LifeCycleSavings) != "Zambia", ] lm_SR2 <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LCS2) sha1(summary(lm_SR2)) ``` ### lm Let's illustrate this using the summary of a simple linear regression. Suppose that we want a hash that takes into account the coefficients, their standard error and sigma. ```{#sha1_lm .R} class(lm_SR) # str() gives the structure of the lm object str(lm_SR) # extract the model and the terms lm_model <- lm_SR$model lm_terms <- lm_SR$terms # check their class class(lm_model) # handled by sha1() class(lm_terms) # not handled by sha1() # define a method for formula sha1.formula <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){ sha1(as.character(x), digits = digits, zapsmall = zapsmall, algo = algo) } sha1(lm_terms) sha1(lm_model) # define a method for lm sha1.lm <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){ lm_model <- x$model lm_terms <- x$terms combined <- list(lm_model, lm_terms) sha1(combined, digits = digits, zapsmall = zapsmall, ..., algo = algo) } sha1(lm_SR) sha1(lm_SR2) ``` ## Using hashes to track changes in analysis Use case - automated analysis - update frequency of the data might be lower than the frequency of automated analysis - similar analyses on many datasets (e.g. many species in ecology) - analyses that require a lot of computing time - not rerunning an analysis because nothing has changed saves enough resources to compensate the overhead of tracking changes - Bundle all relevant information on an analysis in a class - data - method - formula - other metadata - resulting model - calculate `sha1()` file fingerprint ~ `sha1()` on the stable parts status fingerprint ~ `sha1()` on the parts that result for the model 1. Prepare analysis objects 1. Store each analysis object in a rda file which uses the file fingerprint as filename - File will already exist when no change in analysis - Don't overwrite existing files 1. Loop over all rda files - Do nothing if the analysis was run - Otherwise run the analysis and update the status and status fingerprint digest/src/0000755000176200001440000000000014660633605012331 5ustar liggesusersdigest/src/digest.c0000644000176200001440000005266114636027615013767 0ustar liggesusers/* digest -- hash digest functions for R Copyright (C) 2003 - 2024 Dirk Eddelbuettel This file is part of digest. digest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. digest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with digest. If not, see . */ #include // for uint32_t #include #include #include #include #include #include #include #include #include "sha1.h" #include "sha2.h" #include "sha256.h" #include "md5.h" #include "zlib.h" #include "xxhash.h" #include "pmurhash.h" #include "blake3.h" #include "crc32c.h" #ifdef _WIN32 #include #endif unsigned long ZEXPORT digest_crc32(unsigned long crc, const unsigned char FAR *buf, unsigned len); static const char *sha2_hex_digits = "0123456789abcdef"; FILE* open_with_widechar_on_windows(const char* txt) { FILE* out; #ifdef _WIN32 wchar_t* buf; size_t len = MultiByteToWideChar(CP_UTF8, 0, txt, -1, NULL, 0); if (len <= 0) { error("Cannot convert file to Unicode: %s", txt); } buf = (wchar_t*) R_alloc(len, sizeof(wchar_t)); if (buf == NULL) { error("Could not allocate buffer of size: %llu", len); } MultiByteToWideChar(CP_UTF8, 0, txt, -1, buf, len); out = _wfopen(buf, L"rb"); #else out = fopen(txt, "rb"); #endif return out; } SEXP digest(SEXP Txt, SEXP Algo, SEXP Length, SEXP Skip, SEXP Leave_raw, SEXP Seed) { size_t BUF_SIZE = 1024; FILE *fp=0; char *txt; int algo = INTEGER_VALUE(Algo); int length = INTEGER_VALUE(Length); int skip = INTEGER_VALUE(Skip); int seed = INTEGER_VALUE(Seed); int leaveRaw = INTEGER_VALUE(Leave_raw); SEXP result = R_NilValue; char output[128+1], *outputp = output; /* 33 for md5, 41 for sha1, 65 for sha256, 128 for sha512; plus trailing NULL */ R_xlen_t nChar; int output_length = -1; if (IS_RAW(Txt)) { /* Txt is either RAW */ txt = (char*) RAW(Txt); #if defined(R_VERSION) && R_VERSION >= R_Version(3,0,0) nChar = XLENGTH(Txt); #else nChar = LENGTH(Txt); #endif } else { /* or a string */ txt = (char*) STRING_VALUE(Txt); nChar = strlen(txt); if (algo >= 100) { fp = open_with_widechar_on_windows(txt); if (!fp) { error("Cannot open input file: %s", txt); /* #nocov */ } } } if (skip > 0 && algo < 100) { if (skip>=nChar) { nChar=0; /* #nocov */ } else { nChar -= skip; txt += skip; } } if (length>=0 && length> 4]; *outputp++ = sha2_hex_digits[*d & 0x0f]; d++; } *outputp = (char)0; } break; } case 6: { /* xxhash32 case */ XXH32_hash_t val = XXH32(txt, nChar, seed); snprintf(output, 128, "%08x", val); break; } case 7: { /* xxhash64 case */ XXH64_hash_t val = XXH64(txt, nChar, seed); snprintf(output, 128, "%016" PRIx64, val); break; } case 8: { /* MurmurHash3 32 */ unsigned int val = PMurHash32(seed, txt, nChar); snprintf(output, 128, "%08x", val); break; } case 10: { /* blake3 */ output_length = BLAKE3_OUT_LEN; blake3_hasher hasher; blake3_hasher_init(&hasher); blake3_hasher_update(&hasher, txt, nChar); uint8_t val[BLAKE3_OUT_LEN]; blake3_hasher_finalize(&hasher, val, BLAKE3_OUT_LEN); if (leaveRaw) { memcpy(output, val, BLAKE3_OUT_LEN); } else { for (size_t i = 0; i < BLAKE3_OUT_LEN; i++) { snprintf(output + i * 2, 3, "%02x", val[i]); } } break; } case 11: { /* crc32c */ uint32_t crc = 0; /* initial value, can be zero */ crc = crc32c_extend(crc, (const uint8_t*) txt, (size_t) nChar); snprintf(output, 128, "%08x", crc); break; } case 12: { /* xxh3_64bits */ XXH64_hash_t val = XXH3_64bits_withSeed(txt, nChar, seed); snprintf(output, 128, "%016" PRIx64, val); break; } case 13: { /* xxh3_128bits */ XXH128_hash_t val = XXH3_128bits_withSeed(txt, nChar, seed); snprintf(output, 128, "%016" PRIx64 "%016" PRIx64, val.high64, val.low64); break; } case 101: { /* md5 file case */ int j; md5_context ctx; output_length = 16; unsigned char buf[BUF_SIZE]; unsigned char md5sum[16]; if (skip > 0) fseek(fp, skip, SEEK_SET); md5_starts( &ctx ); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; md5_update( &ctx, buf, nChar ); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) md5_update( &ctx, buf, nChar ); } md5_finish( &ctx, md5sum ); memcpy(output, md5sum, 16); if (!leaveRaw) for (j = 0; j < 16; j++) snprintf(output + j * 2, 3, "%02x", md5sum[j]); break; } case 102: { /* sha1 file case */ int j; sha1_context ctx; output_length = 20; unsigned char buf[BUF_SIZE]; unsigned char sha1sum[20]; if (skip > 0) fseek(fp, skip, SEEK_SET); sha1_starts ( &ctx ); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; sha1_update( &ctx, buf, nChar ); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) sha1_update( &ctx, buf, nChar ); } sha1_finish ( &ctx, sha1sum ); memcpy(output, sha1sum, 20); if (!leaveRaw) for ( j = 0; j < 20; j++ ) snprintf( output + j * 2, 3, "%02x", sha1sum[j] ); break; } case 103: { /* crc32 file case */ unsigned char buf[BUF_SIZE]; unsigned long val; if (skip > 0) fseek(fp, skip, SEEK_SET); val = digest_crc32(0L, 0, 0); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; val = digest_crc32(val , buf, (unsigned) nChar); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) val = digest_crc32(val , buf, (unsigned) nChar); } snprintf(output, 128, "%08x", (unsigned int) val); break; } case 104: { /* sha256 file case */ int j; sha256_context ctx; output_length = 32; unsigned char buf[BUF_SIZE]; unsigned char sha256sum[32]; if (skip > 0) fseek(fp, skip, SEEK_SET); sha256_starts ( &ctx ); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; sha256_update( &ctx, buf, nChar ); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) sha256_update( &ctx, buf, nChar ); } sha256_finish ( &ctx, sha256sum ); memcpy(output, sha256sum, 32); if (!leaveRaw) for ( j = 0; j < 32; j++ ) snprintf( output + j * 2, 3, "%02x", sha256sum[j] ); break; } case 105: { /* sha2-512 file case */ int j; SHA512_CTX ctx; output_length = SHA512_DIGEST_LENGTH; uint8_t sha512sum[output_length], *d = sha512sum; unsigned char buf[BUF_SIZE]; if (skip > 0) fseek(fp, skip, SEEK_SET); SHA512_Init(&ctx); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; SHA512_Update( &ctx, buf, nChar ); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) SHA512_Update( &ctx, buf, nChar ); } /* Calling SHA512_Final, because SHA512_End will already convert the hash to a string, and we also want RAW */ SHA512_Final(sha512sum, &ctx); memcpy(output, sha512sum, output_length); /* adapted from SHA512_End */ if (!leaveRaw) { for (j = 0; j < output_length; j++) { *outputp++ = sha2_hex_digits[(*d & 0xf0) >> 4]; *outputp++ = sha2_hex_digits[*d & 0x0f]; d++; } *outputp = (char)0; } break; } case 106: { /* xxhash32 */ unsigned char buf[BUF_SIZE]; XXH32_state_t* const state = XXH32_createState(); if (skip > 0) fseek(fp, skip, SEEK_SET); XXH_errorcode const resetResult = XXH32_reset(state, seed); if (resetResult == XXH_ERROR) { error("Error in `XXH32_reset()`"); /* #nocov */ } if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; XXH_errorcode const updateResult = XXH32_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH32_update()`"); /* #nocov */ } length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) { XXH_errorcode const updateResult = XXH32_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH32_update()`"); /* #nocov */ } } } XXH32_hash_t val = XXH32_digest(state); XXH32_freeState(state); snprintf(output, 128, "%08x", val); break; } case 107: { /* xxhash64 */ unsigned char buf[BUF_SIZE]; XXH64_state_t* const state = XXH64_createState(); if (skip > 0) fseek(fp, skip, SEEK_SET); XXH_errorcode const resetResult = XXH64_reset(state, seed); if (resetResult == XXH_ERROR) { error("Error in `XXH64_reset()`"); /* #nocov */ } if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; XXH_errorcode const updateResult = XXH64_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH64_update()`"); /* #nocov */ } length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) { XXH_errorcode const updateResult = XXH64_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH64_update()`"); /* #nocov */ } } } XXH64_hash_t val = XXH64_digest(state); XXH64_freeState(state); snprintf(output, 128, "%016" PRIx64, val); break; } case 108: { /* murmur32 */ unsigned int h1=seed, carry=0; unsigned char buf[BUF_SIZE]; size_t total_length = 0; if (skip > 0) fseek(fp, skip, SEEK_SET); if (length>=0) { while( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; PMurHash32_Process(&h1, &carry, buf, nChar); length -= nChar; total_length += nChar; } } else { while( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) { PMurHash32_Process(&h1, &carry, buf, nChar); total_length += nChar; } } unsigned int val = PMurHash32_Result(h1, carry, total_length); snprintf(output, 128, "%08x", val); break; } case 110: { /* blake3 file case */ output_length = BLAKE3_OUT_LEN; unsigned char buf[BUF_SIZE]; uint8_t val[BLAKE3_OUT_LEN]; blake3_hasher hasher; blake3_hasher_init(&hasher); if (skip > 0) fseek(fp, skip, SEEK_SET); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; blake3_hasher_update( &hasher, buf, nChar ); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) blake3_hasher_update( &hasher, buf, nChar ); } blake3_hasher_finalize(&hasher, val, BLAKE3_OUT_LEN); if (leaveRaw) { memcpy(output, val, BLAKE3_OUT_LEN); } else { for (size_t i = 0; i < BLAKE3_OUT_LEN; i++) { snprintf(output + i * 2, 3, "%02x", val[i]); } } break; } case 111: { /* crc32c */ unsigned char buf[BUF_SIZE]; uint32_t crc = 0; if (skip > 0) fseek(fp, skip, SEEK_SET); if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; crc = crc32c_extend(crc, (const uint8_t*) buf, (size_t) nChar); length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) crc = crc32c_extend(crc, (const uint8_t*) buf, (size_t) nChar); } snprintf(output, 128, "%08x", (unsigned int) crc); break; } case 112: { /* xxh3_64 */ unsigned char buf[BUF_SIZE]; XXH3_state_t* const state = XXH3_createState(); if (skip > 0) fseek(fp, skip, SEEK_SET); XXH_errorcode const resetResult = XXH3_64bits_reset(state); if (resetResult == XXH_ERROR) { error("Error in `XXH3_reset()`"); /* #nocov */ } if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; XXH_errorcode const updateResult = XXH3_64bits_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH3_64bits_update()`"); /* #nocov */ } length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) { XXH_errorcode const updateResult = XXH3_64bits_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH3_64bit_update()`"); /* #nocov */ } } } XXH64_hash_t val = XXH3_64bits_digest(state); XXH3_freeState(state); snprintf(output, 128, "%016" PRIx64, val); break; } case 113: { /* xxh3_128 */ unsigned char buf[BUF_SIZE]; XXH3_state_t* const state = XXH3_createState(); if (skip > 0) fseek(fp, skip, SEEK_SET); XXH_errorcode const resetResult = XXH3_128bits_reset(state); if (resetResult == XXH_ERROR) { error("Error in `XXH3_reset()`"); /* #nocov */ } if (length>=0) { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0 && length>0) { if (nChar>length) nChar=length; XXH_errorcode const updateResult = XXH3_128bits_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH3_128bits_update()`"); /* #nocov */ } length -= nChar; } } else { while ( ( nChar = fread( buf, 1, sizeof( buf ), fp ) ) > 0) { XXH_errorcode const updateResult = XXH3_128bits_update(state, buf, nChar); if (updateResult == XXH_ERROR) { error("Error in `XXH3_128bit_update()`"); /* #nocov */ } } } XXH128_hash_t val = XXH3_128bits_digest(state); XXH3_freeState(state); snprintf(output, 128, "%016" PRIx64 "%016" PRIx64, val.high64, val.low64); break; } default: { error("Unsupported algorithm code"); /* should not be reached due to test in R */ /* #nocov */ } } /* end switch */ if (algo >= 100 && fp) { fclose(fp); } if (leaveRaw && output_length > 0) { PROTECT(result=allocVector(RAWSXP, output_length)); memcpy(RAW(result), output, output_length); } else { PROTECT(result=allocVector(STRSXP, 1)); SET_STRING_ELT(result, 0, mkChar(output)); } UNPROTECT(1); return result; } SEXP vdigest(SEXP Txt, SEXP Algo, SEXP Length, SEXP Skip, SEXP Leave_raw, SEXP Seed){ R_xlen_t n = length(Txt); if (TYPEOF(Txt) == RAWSXP || n == 0) return(digest(Txt, Algo, Length, Skip, Leave_raw, Seed)); SEXP ans = PROTECT(allocVector(STRSXP, n)); SEXP d = R_NilValue; if (TYPEOF(Txt) == VECSXP){ for (R_xlen_t i = 0; i < n; i++){ d = digest(VECTOR_ELT(Txt, i), Algo, Length, Skip, Leave_raw, Seed); SET_STRING_ELT(ans, i, STRING_ELT(d, 0)); } } else { for (R_xlen_t i = 0; i < n; i++){ d = digest(STRING_ELT(Txt, i), Algo, Length, Skip, Leave_raw, Seed); SET_STRING_ELT(ans, i, STRING_ELT(d, 0)); } } UNPROTECT(1); return ans; } // Also already used in sha2.h // // We can rely on WORDS_BIGENDIAN only be defined on big endian systems thanks to Rconfig. // // A number of other #define based tests are in other source files here for different hash // algorithm implementations notably crc32c, pmurhash, sha2 and xxhash // // A small and elegant test is also in package qs based on https://stackoverflow.com/a/1001373 // edd 02 Dec 2013 use Rconfig.h to define BYTE_ORDER, unless already defined #ifndef BYTE_ORDER // see sha2.c comments, and on the internet at large #define LITTLE_ENDIAN 1234 #define BIG_ENDIAN 4321 #ifdef WORDS_BIGENDIAN #define BYTE_ORDER BIG_ENDIAN #else #define BYTE_ORDER LITTLE_ENDIAN #endif #endif SEXP is_big_endian(void) { return Rf_ScalarLogical(BYTE_ORDER == BIG_ENDIAN); } SEXP is_little_endian(void) { return Rf_ScalarLogical(BYTE_ORDER == LITTLE_ENDIAN); } digest/src/digest2int.c0000644000176200001440000000174013371554565014560 0ustar liggesusers#include #include #include // https://en.wikipedia.org/wiki/Jenkins_hash_function#one_at_a_time uint32_t jenkins_one_at_a_time_hash(const char *key, uint32_t seed) { uint32_t hash = seed; for(; *key; ++key) { hash += *key; hash += (hash << 10); hash ^= (hash >> 6); } hash += (hash << 3); hash ^= (hash >> 11); hash += (hash << 15); return hash; } SEXP digest2int(SEXP input, SEXP Seed) { uint32_t seed = INTEGER_VALUE(Seed); if (TYPEOF(input) != STRSXP) error("invalid input - should be character vector"); R_xlen_t n = xlength(input); SEXP result = PROTECT(allocVector(INTSXP, n )); memset(INTEGER(result), 0, n * sizeof(int)); int *res_ptr = INTEGER(result); for(R_xlen_t i = 0; i < n; i++) { const char* element_ptr = CHAR(STRING_ELT(input, i)); res_ptr[i] = jenkins_one_at_a_time_hash(element_ptr, seed); } UNPROTECT(1); return(result); } digest/src/md5.c0000644000176200001440000002060613216563247013166 0ustar liggesusers/* * RFC 1321 compliant MD5 implementation, * by Christophe Devine ; * this program is licensed under the GPL. */ #include #include "md5.h" #define GET_UINT32(n,b,i) \ { \ (n) = ( (uint32) (b)[(i) ] ) \ | ( (uint32) (b)[(i) + 1] << 8 ) \ | ( (uint32) (b)[(i) + 2] << 16 ) \ | ( (uint32) (b)[(i) + 3] << 24 ); \ } #define PUT_UINT32(n,b,i) \ { \ (b)[(i) ] = (uint8) ( (n) ); \ (b)[(i) + 1] = (uint8) ( (n) >> 8 ); \ (b)[(i) + 2] = (uint8) ( (n) >> 16 ); \ (b)[(i) + 3] = (uint8) ( (n) >> 24 ); \ } void md5_starts( md5_context *ctx ) { ctx->total[0] = 0; ctx->total[1] = 0; ctx->state[0] = 0x67452301; ctx->state[1] = 0xEFCDAB89; ctx->state[2] = 0x98BADCFE; ctx->state[3] = 0x10325476; } void md5_process( md5_context *ctx, uint8 data[64] ) { uint32 X[16], A, B, C, D; GET_UINT32( X[0], data, 0 ); GET_UINT32( X[1], data, 4 ); GET_UINT32( X[2], data, 8 ); GET_UINT32( X[3], data, 12 ); GET_UINT32( X[4], data, 16 ); GET_UINT32( X[5], data, 20 ); GET_UINT32( X[6], data, 24 ); GET_UINT32( X[7], data, 28 ); GET_UINT32( X[8], data, 32 ); GET_UINT32( X[9], data, 36 ); GET_UINT32( X[10], data, 40 ); GET_UINT32( X[11], data, 44 ); GET_UINT32( X[12], data, 48 ); GET_UINT32( X[13], data, 52 ); GET_UINT32( X[14], data, 56 ); GET_UINT32( X[15], data, 60 ); #define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) #define P(a,b,c,d,k,s,t) \ { \ a += F(b,c,d) + X[k] + t; a = S(a,s) + b; \ } A = ctx->state[0]; B = ctx->state[1]; C = ctx->state[2]; D = ctx->state[3]; #define F(x,y,z) (z ^ (x & (y ^ z))) P( A, B, C, D, 0, 7, 0xD76AA478 ); P( D, A, B, C, 1, 12, 0xE8C7B756 ); P( C, D, A, B, 2, 17, 0x242070DB ); P( B, C, D, A, 3, 22, 0xC1BDCEEE ); P( A, B, C, D, 4, 7, 0xF57C0FAF ); P( D, A, B, C, 5, 12, 0x4787C62A ); P( C, D, A, B, 6, 17, 0xA8304613 ); P( B, C, D, A, 7, 22, 0xFD469501 ); P( A, B, C, D, 8, 7, 0x698098D8 ); P( D, A, B, C, 9, 12, 0x8B44F7AF ); P( C, D, A, B, 10, 17, 0xFFFF5BB1 ); P( B, C, D, A, 11, 22, 0x895CD7BE ); P( A, B, C, D, 12, 7, 0x6B901122 ); P( D, A, B, C, 13, 12, 0xFD987193 ); P( C, D, A, B, 14, 17, 0xA679438E ); P( B, C, D, A, 15, 22, 0x49B40821 ); #undef F #define F(x,y,z) (y ^ (z & (x ^ y))) P( A, B, C, D, 1, 5, 0xF61E2562 ); P( D, A, B, C, 6, 9, 0xC040B340 ); P( C, D, A, B, 11, 14, 0x265E5A51 ); P( B, C, D, A, 0, 20, 0xE9B6C7AA ); P( A, B, C, D, 5, 5, 0xD62F105D ); P( D, A, B, C, 10, 9, 0x02441453 ); P( C, D, A, B, 15, 14, 0xD8A1E681 ); P( B, C, D, A, 4, 20, 0xE7D3FBC8 ); P( A, B, C, D, 9, 5, 0x21E1CDE6 ); P( D, A, B, C, 14, 9, 0xC33707D6 ); P( C, D, A, B, 3, 14, 0xF4D50D87 ); P( B, C, D, A, 8, 20, 0x455A14ED ); P( A, B, C, D, 13, 5, 0xA9E3E905 ); P( D, A, B, C, 2, 9, 0xFCEFA3F8 ); P( C, D, A, B, 7, 14, 0x676F02D9 ); P( B, C, D, A, 12, 20, 0x8D2A4C8A ); #undef F #define F(x,y,z) (x ^ y ^ z) P( A, B, C, D, 5, 4, 0xFFFA3942 ); P( D, A, B, C, 8, 11, 0x8771F681 ); P( C, D, A, B, 11, 16, 0x6D9D6122 ); P( B, C, D, A, 14, 23, 0xFDE5380C ); P( A, B, C, D, 1, 4, 0xA4BEEA44 ); P( D, A, B, C, 4, 11, 0x4BDECFA9 ); P( C, D, A, B, 7, 16, 0xF6BB4B60 ); P( B, C, D, A, 10, 23, 0xBEBFBC70 ); P( A, B, C, D, 13, 4, 0x289B7EC6 ); P( D, A, B, C, 0, 11, 0xEAA127FA ); P( C, D, A, B, 3, 16, 0xD4EF3085 ); P( B, C, D, A, 6, 23, 0x04881D05 ); P( A, B, C, D, 9, 4, 0xD9D4D039 ); P( D, A, B, C, 12, 11, 0xE6DB99E5 ); P( C, D, A, B, 15, 16, 0x1FA27CF8 ); P( B, C, D, A, 2, 23, 0xC4AC5665 ); #undef F #define F(x,y,z) (y ^ (x | ~z)) P( A, B, C, D, 0, 6, 0xF4292244 ); P( D, A, B, C, 7, 10, 0x432AFF97 ); P( C, D, A, B, 14, 15, 0xAB9423A7 ); P( B, C, D, A, 5, 21, 0xFC93A039 ); P( A, B, C, D, 12, 6, 0x655B59C3 ); P( D, A, B, C, 3, 10, 0x8F0CCC92 ); P( C, D, A, B, 10, 15, 0xFFEFF47D ); P( B, C, D, A, 1, 21, 0x85845DD1 ); P( A, B, C, D, 8, 6, 0x6FA87E4F ); P( D, A, B, C, 15, 10, 0xFE2CE6E0 ); P( C, D, A, B, 6, 15, 0xA3014314 ); P( B, C, D, A, 13, 21, 0x4E0811A1 ); P( A, B, C, D, 4, 6, 0xF7537E82 ); P( D, A, B, C, 11, 10, 0xBD3AF235 ); P( C, D, A, B, 2, 15, 0x2AD7D2BB ); P( B, C, D, A, 9, 21, 0xEB86D391 ); #undef F ctx->state[0] += A; ctx->state[1] += B; ctx->state[2] += C; ctx->state[3] += D; } void md5_update( md5_context *ctx, uint8 *input, uint32 length ) { uint32 left, fill; if( ! length ) return; left = ctx->total[0] & 0x3F; fill = 64 - left; ctx->total[0] += length; ctx->total[0] &= 0xFFFFFFFF; if( ctx->total[0] < length ) ctx->total[1]++; /* #nocov */ if( left && length >= fill ) { memcpy( (void *) (ctx->buffer + left), (void *) input, fill ); md5_process( ctx, ctx->buffer ); length -= fill; input += fill; left = 0; } while( length >= 64 ) { md5_process( ctx, input ); length -= 64; input += 64; } if( length ) { memcpy( (void *) (ctx->buffer + left), (void *) input, length ); } } static uint8 md5_padding[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; void md5_finish( md5_context *ctx, uint8 digest[16] ) { uint32 last, padn; uint32 high, low; uint8 msglen[8]; high = ( ctx->total[0] >> 29 ) | ( ctx->total[1] << 3 ); low = ( ctx->total[0] << 3 ); PUT_UINT32( low, msglen, 0 ); PUT_UINT32( high, msglen, 4 ); last = ctx->total[0] & 0x3F; padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); md5_update( ctx, md5_padding, padn ); md5_update( ctx, msglen, 8 ); PUT_UINT32( ctx->state[0], digest, 0 ); PUT_UINT32( ctx->state[1], digest, 4 ); PUT_UINT32( ctx->state[2], digest, 8 ); PUT_UINT32( ctx->state[3], digest, 12 ); } #ifdef TEST #include #include /* * those are the standard RFC 1321 test vectors */ static char *msg[] = { "", "a", "abc", "message digest", "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "12345678901234567890123456789012345678901234567890123456789012" \ "345678901234567890" }; static char *val[] = { "d41d8cd98f00b204e9800998ecf8427e", "0cc175b9c0f1b6a831c399e269772661", "900150983cd24fb0d6963f7d28e17f72", "f96b697d7cb7938d525a2f31aaf161d0", "c3fcd3d76192e4007dfb496cca67e13b", "d174ab98d277d9f5a5611c2c9f419d9f", "57edf4a22be3c955ac49da2e2107b67a" }; int main( int argc, char *argv[] ) { FILE *f; int i, j; char output[33]; md5_context ctx; unsigned char buf[1000]; unsigned char md5sum[16]; if( argc < 2 ) { printf( "\n MD5 Validation Tests:\n\n" ); for( i = 0; i < 7; i++ ) { printf( " Test %d ", i + 1 ); md5_starts( &ctx ); md5_update( &ctx, (uint8 *) msg[i], strlen( msg[i] ) ); md5_finish( &ctx, md5sum ); for( j = 0; j < 16; j++ ) { sprintf( output + j * 2, "%02x", md5sum[j] ); } if( memcmp( output, val[i], 32 ) ) { printf( "failed!\n" ); return( 1 ); } printf( "passed.\n" ); } printf( "\n" ); } else { if( ! ( f = fopen( argv[1], "rb" ) ) ) { perror( "fopen" ); return( 1 ); } md5_starts( &ctx ); while( ( i = fread( buf, 1, sizeof( buf ), f ) ) > 0 ) { md5_update( &ctx, buf, i ); } md5_finish( &ctx, md5sum ); for( j = 0; j < 16; j++ ) { printf( "%02x", md5sum[j] ); } printf( " %s\n", argv[1] ); } return( 0 ); } #endif digest/src/Makevars.win0000644000176200001440000000002414423472174014614 0ustar liggesusers PKG_CPPFLAGS = -I. digest/src/sha2.c0000644000176200001440000010026614323255156013334 0ustar liggesusers/* * FILE: sha2.c * AUTHOR: Aaron D. Gifford - http://www.aarongifford.com/ * * Copyright (c) 2000-2001, Aaron D. Gifford * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Modified by R. Neal, 2018, to avoid mis-alignment faults. * */ #include /* memcpy()/memset() or bcopy()/bzero() */ #include /* assert() */ #include "sha2.h" /* * ASSERT NOTE: * Some sanity checking code is included using assert(). On my FreeBSD * system, this additional code can be removed by compiling with NDEBUG * defined. Check your own systems manpage on assert() to see how to * compile WITHOUT the sanity checking code on your system. * * UNROLLED TRANSFORM LOOP NOTE: * You can define SHA2_UNROLL_TRANSFORM to use the unrolled transform * loop version for the hash transform rounds (defined using macros * later in this file). Either define on the command line, for example: * * cc -DSHA2_UNROLL_TRANSFORM -o sha2 sha2.c sha2prog.c * * or define below: * * #define SHA2_UNROLL_TRANSFORM * */ /*** SHA-256/384/512 Machine Architecture Definitions *****************/ /* * BYTE_ORDER NOTE: * * Please make sure that your system defines BYTE_ORDER. If your * architecture is little-endian, make sure it also defines * LITTLE_ENDIAN and that the two (BYTE_ORDER and LITTLE_ENDIAN) are * equivilent. * * If your system does not define the above, then you can do so by * hand like this: * * #define LITTLE_ENDIAN 1234 * #define BIG_ENDIAN 4321 * * And for little-endian machines, add: * * #define BYTE_ORDER LITTLE_ENDIAN * * Or for big-endian machines: * * #define BYTE_ORDER BIG_ENDIAN * * The FreeBSD machine this was written on defines BYTE_ORDER * appropriately by including (which in turn includes * where the appropriate definitions are actually * made). */ #if !defined(BYTE_ORDER) || (BYTE_ORDER != LITTLE_ENDIAN && BYTE_ORDER != BIG_ENDIAN) #error Define BYTE_ORDER to be equal to either LITTLE_ENDIAN or BIG_ENDIAN #endif /* * Define the followingsha2_* types to types of the correct length on * the native archtecture. Most BSD systems and Linux define u_intXX_t * types. Machines with very recent ANSI C headers, can use the * uintXX_t definintions from inttypes.h by defining SHA2_USE_INTTYPES_H * during compile or in the sha.h header file. * * Machines that support neither u_intXX_t nor inttypes.h's uintXX_t * will need to define these three typedefs below (and the appropriate * ones in sha.h too) by hand according to their system architecture. * * Thank you, Jun-ichiro itojun Hagino, for suggesting using u_intXX_t * types and pointing out recent ANSI C support for uintXX_t in inttypes.h. */ #ifdef SHA2_USE_INTTYPES_H typedef uint8_t sha2_byte; /* Exactly 1 byte */ typedef uint32_t sha2_word32; /* Exactly 4 bytes */ typedef uint64_t sha2_word64; /* Exactly 8 bytes */ #else /* SHA2_USE_INTTYPES_H */ typedef u_int8_t sha2_byte; /* Exactly 1 byte */ typedef u_int32_t sha2_word32; /* Exactly 4 bytes */ typedef u_int64_t sha2_word64; /* Exactly 8 bytes */ #endif /* SHA2_USE_INTTYPES_H */ /*** SHA-256/384/512 Various Length Definitions ***********************/ /* NOTE: Most of these are in sha2.h */ #define SHA256_SHORT_BLOCK_LENGTH (SHA256_BLOCK_LENGTH - 8) #define SHA384_SHORT_BLOCK_LENGTH (SHA384_BLOCK_LENGTH - 16) #define SHA512_SHORT_BLOCK_LENGTH (SHA512_BLOCK_LENGTH - 16) /*** ENDIAN REVERSAL MACROS *******************************************/ #if BYTE_ORDER == LITTLE_ENDIAN #define REVERSE32(w,x) { \ sha2_word32 tmp = (w); \ tmp = (tmp >> 16) | (tmp << 16); \ (x) = ((tmp & 0xff00ff00UL) >> 8) | ((tmp & 0x00ff00ffUL) << 8); \ } #define REVERSE64(w,x) { \ sha2_word64 tmp = (w); \ tmp = (tmp >> 32) | (tmp << 32); \ tmp = ((tmp & 0xff00ff00ff00ff00ULL) >> 8) | \ ((tmp & 0x00ff00ff00ff00ffULL) << 8); \ (x) = ((tmp & 0xffff0000ffff0000ULL) >> 16) | \ ((tmp & 0x0000ffff0000ffffULL) << 16); \ } #endif /* BYTE_ORDER == LITTLE_ENDIAN */ /* * Macro for incrementally adding the unsigned 64-bit integer n to the * unsigned 128-bit integer (represented using a two-element array of * 64-bit words): */ #define ADDINC128(w,n) { \ (w)[0] += (sha2_word64)(n); \ if ((w)[0] < (n)) { \ (w)[1]++; \ } \ } /* * Macros for copying blocks of memory and for zeroing out ranges * of memory. Using these macros makes it easy to switch from * using memset()/memcpy() and using bzero()/bcopy(). * * Please define either SHA2_USE_MEMSET_MEMCPY or define * SHA2_USE_BZERO_BCOPY depending on which function set you * choose to use: */ #if !defined(SHA2_USE_MEMSET_MEMCPY) && !defined(SHA2_USE_BZERO_BCOPY) /* Default to memset()/memcpy() if no option is specified */ #define SHA2_USE_MEMSET_MEMCPY 1 #endif #if defined(SHA2_USE_MEMSET_MEMCPY) && defined(SHA2_USE_BZERO_BCOPY) /* Abort with an error if BOTH options are defined */ #error Define either SHA2_USE_MEMSET_MEMCPY or SHA2_USE_BZERO_BCOPY, not both! #endif #ifdef SHA2_USE_MEMSET_MEMCPY #define MEMSET_BZERO(p,l) memset((p), 0, (l)) #define MEMCPY_BCOPY(d,s,l) memcpy((d), (s), (l)) #endif #ifdef SHA2_USE_BZERO_BCOPY #define MEMSET_BZERO(p,l) bzero((p), (l)) #define MEMCPY_BCOPY(d,s,l) bcopy((s), (d), (l)) #endif /*** THE SIX LOGICAL FUNCTIONS ****************************************/ /* * Bit shifting and rotation (used by the six SHA-XYZ logical functions: * * NOTE: The naming of R and S appears backwards here (R is a SHIFT and * S is a ROTATION) because the SHA-256/384/512 description document * (see http://csrc.nist.gov/cryptval/shs/sha256-384-512.pdf) uses this * same "backwards" definition. */ /* Shift-right (used in SHA-256, SHA-384, and SHA-512): */ #define R(b,x) ((x) >> (b)) /* 32-bit Rotate-right (used in SHA-256): */ #define S32(b,x) (((x) >> (b)) | ((x) << (32 - (b)))) /* 64-bit Rotate-right (used in SHA-384 and SHA-512): */ #define S64(b,x) (((x) >> (b)) | ((x) << (64 - (b)))) /* Two of six logical functions used in SHA-256, SHA-384, and SHA-512: */ #define Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) #define Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) /* Four of six logical functions used in SHA-256: */ #define Sigma0_256(x) (S32(2, (x)) ^ S32(13, (x)) ^ S32(22, (x))) #define Sigma1_256(x) (S32(6, (x)) ^ S32(11, (x)) ^ S32(25, (x))) #define sigma0_256(x) (S32(7, (x)) ^ S32(18, (x)) ^ R(3 , (x))) #define sigma1_256(x) (S32(17, (x)) ^ S32(19, (x)) ^ R(10, (x))) /* Four of six logical functions used in SHA-384 and SHA-512: */ #define Sigma0_512(x) (S64(28, (x)) ^ S64(34, (x)) ^ S64(39, (x))) #define Sigma1_512(x) (S64(14, (x)) ^ S64(18, (x)) ^ S64(41, (x))) #define sigma0_512(x) (S64( 1, (x)) ^ S64( 8, (x)) ^ R( 7, (x))) #define sigma1_512(x) (S64(19, (x)) ^ S64(61, (x)) ^ R( 6, (x))) /*** INTERNAL FUNCTION PROTOTYPES *************************************/ /* NOTE: These should not be accessed directly from outside this * library -- they are intended for private internal visibility/use * only. */ static void SHA512_Last(SHA512_CTX*); /*static void SHA256_Transform(SHA256_CTX*);*/ static void SHA512_Transform(SHA512_CTX*); /* /\*** SHA-XYZ INITIAL HASH VALUES AND CONSTANTS ************************\/ */ /* /\* Hash constant words K for SHA-256: *\/ */ /* const static sha2_word32 K256[64] = { */ /* 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, */ /* 0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, */ /* 0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL, */ /* 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, 0xc19bf174UL, */ /* 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, */ /* 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, */ /* 0x983e5152UL, 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, */ /* 0xc6e00bf3UL, 0xd5a79147UL, 0x06ca6351UL, 0x14292967UL, */ /* 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, 0x53380d13UL, */ /* 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, */ /* 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, */ /* 0xd192e819UL, 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, */ /* 0x19a4c116UL, 0x1e376c08UL, 0x2748774cUL, 0x34b0bcb5UL, */ /* 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL, */ /* 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, */ /* 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL */ /* }; */ /* /\* Initial hash value H for SHA-256: *\/ */ /* const static sha2_word32 sha256_initial_hash_value[8] = { */ /* 0x6a09e667UL, */ /* 0xbb67ae85UL, */ /* 0x3c6ef372UL, */ /* 0xa54ff53aUL, */ /* 0x510e527fUL, */ /* 0x9b05688cUL, */ /* 0x1f83d9abUL, */ /* 0x5be0cd19UL */ /* }; */ /* Hash constant words K for SHA-384 and SHA-512: */ const static sha2_word64 K512[80] = { 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL }; /* /\* Initial hash value H for SHA-384 *\/ */ /* const static sha2_word64 sha384_initial_hash_value[8] = { */ /* 0xcbbb9d5dc1059ed8ULL, */ /* 0x629a292a367cd507ULL, */ /* 0x9159015a3070dd17ULL, */ /* 0x152fecd8f70e5939ULL, */ /* 0x67332667ffc00b31ULL, */ /* 0x8eb44a8768581511ULL, */ /* 0xdb0c2e0d64f98fa7ULL, */ /* 0x47b5481dbefa4fa4ULL */ /* }; */ /* Initial hash value H for SHA-512 */ const static sha2_word64 sha512_initial_hash_value[8] = { 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL }; /* * Constant used by SHA256/384/512_End() functions for converting the * digest to a readable hexadecimal character string: */ static const char *sha2_hex_digits = "0123456789abcdef"; /*** SHA-256: *********************************************************/ #if 0 void SHA256_Init(SHA256_CTX* context) { /* #nocov start */ if (context == (SHA256_CTX*)0) { return; } MEMCPY_BCOPY(context->state, sha256_initial_hash_value, SHA256_DIGEST_LENGTH); MEMSET_BZERO(context->buffer, SHA256_BLOCK_LENGTH); context->bitcount = 0; } /* #nocov end */ #endif #ifdef SHA2_UNROLL_TRANSFORM /* Unrolled SHA-256 round macros: */ #if BYTE_ORDER == LITTLE_ENDIAN #define ROUND256_0_TO_15(a,b,c,d,e,f,g,h) \ REVERSE32(W256[j], W256[j]); \ T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \ K256[j] + W256[j]; \ (d) += T1; \ (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ j++ #else /* BYTE_ORDER == LITTLE_ENDIAN */ #define ROUND256_0_TO_15(a,b,c,d,e,f,g,h) \ T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \ K256[j] + W256[j]; \ (d) += T1; \ (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ j++ #endif /* BYTE_ORDER == LITTLE_ENDIAN */ #define ROUND256(a,b,c,d,e,f,g,h) \ s0 = W256[(j+1)&0x0f]; \ s0 = sigma0_256(s0); \ s1 = W256[(j+14)&0x0f]; \ s1 = sigma1_256(s1); \ T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + \ (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); \ (d) += T1; \ (h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \ j++ #if 0 static void SHA256_Transform(SHA256_CTX* context) { sha2_word32 a, b, c, d, e, f, g, h, s0, s1; sha2_word32 T1, *W256; int j; W256 = (sha2_word32*)context->buffer; /* Initialize registers with the prev. intermediate value */ a = context->state[0]; b = context->state[1]; c = context->state[2]; d = context->state[3]; e = context->state[4]; f = context->state[5]; g = context->state[6]; h = context->state[7]; j = 0; do { /* Rounds 0 to 15 (unrolled): */ ROUND256_0_TO_15(a,b,c,d,e,f,g,h); ROUND256_0_TO_15(h,a,b,c,d,e,f,g); ROUND256_0_TO_15(g,h,a,b,c,d,e,f); ROUND256_0_TO_15(f,g,h,a,b,c,d,e); ROUND256_0_TO_15(e,f,g,h,a,b,c,d); ROUND256_0_TO_15(d,e,f,g,h,a,b,c); ROUND256_0_TO_15(c,d,e,f,g,h,a,b); ROUND256_0_TO_15(b,c,d,e,f,g,h,a); } while (j < 16); /* Now for the remaining rounds to 64: */ do { ROUND256(a,b,c,d,e,f,g,h); ROUND256(h,a,b,c,d,e,f,g); ROUND256(g,h,a,b,c,d,e,f); ROUND256(f,g,h,a,b,c,d,e); ROUND256(e,f,g,h,a,b,c,d); ROUND256(d,e,f,g,h,a,b,c); ROUND256(c,d,e,f,g,h,a,b); ROUND256(b,c,d,e,f,g,h,a); } while (j < 64); /* Compute the current intermediate hash value */ context->state[0] += a; context->state[1] += b; context->state[2] += c; context->state[3] += d; context->state[4] += e; context->state[5] += f; context->state[6] += g; context->state[7] += h; /* Clean up */ a = b = c = d = e = f = g = h = T1 = 0; } #endif #else /* SHA2_UNROLL_TRANSFORM */ #if 0 static void SHA256_Transform(SHA256_CTX* context) { // #nocov sha2_word32 a, b, c, d, e, f, g, h, s0, s1; sha2_word32 T1, T2, *W256; int j; W256 = (sha2_word32*)context->buffer; /* Initialize registers with the prev. intermediate value */ a = context->state[0]; b = context->state[1]; c = context->state[2]; d = context->state[3]; e = context->state[4]; f = context->state[5]; g = context->state[6]; h = context->state[7]; j = 0; do { #if BYTE_ORDER == LITTLE_ENDIAN /* convert to host byte order */ REVERSE32(W256[j],W256[j]); /* Apply the SHA-256 compression function to update a..h */ T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + W256[j]; #else /* BYTE_ORDER == LITTLE_ENDIAN */ /* Apply the SHA-256 compression function to update a..h with copy */ T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + W256[j]; #endif /* BYTE_ORDER == LITTLE_ENDIAN */ T2 = Sigma0_256(a) + Maj(a, b, c); h = g; g = f; f = e; e = d + T1; d = c; c = b; b = a; a = T1 + T2; j++; } while (j < 16); do { /* Part of the message block expansion: */ s0 = W256[(j+1)&0x0f]; s0 = sigma0_256(s0); s1 = W256[(j+14)&0x0f]; s1 = sigma1_256(s1); /* Apply the SHA-256 compression function to update a..h */ T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); T2 = Sigma0_256(a) + Maj(a, b, c); h = g; g = f; f = e; e = d + T1; d = c; c = b; b = a; a = T1 + T2; j++; } while (j < 64); /* Compute the current intermediate hash value */ context->state[0] += a; context->state[1] += b; context->state[2] += c; context->state[3] += d; context->state[4] += e; context->state[5] += f; context->state[6] += g; context->state[7] += h; /* Clean up */ a = b = c = d = e = f = g = h = T1 = T2 = 0; } // #nocov #endif /* around SHA256_Transform */ #endif /* SHA2_UNROLL_TRANSFORM */ #if 0 void SHA256_Update(SHA256_CTX* context, const sha2_byte *data, size_t len) { // #nocov unsigned int freespace, usedspace; if (len == 0) { /* Calling with no data is valid - we do nothing */ return; } /* Sanity check: */ assert(context != (SHA256_CTX*)0 && data != (sha2_byte*)0); usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH; if (usedspace > 0) { /* Calculate how much free space is available in the buffer */ freespace = SHA256_BLOCK_LENGTH - usedspace; if (len >= freespace) { /* Fill the buffer completely and process it */ MEMCPY_BCOPY(&context->buffer[usedspace], data, freespace); context->bitcount += freespace << 3; len -= freespace; data += freespace; SHA256_Transform(context); } else { /* The buffer is not yet full */ MEMCPY_BCOPY(&context->buffer[usedspace], data, len); context->bitcount += len << 3; /* Clean up: */ usedspace = freespace = 0; return; } } while (len >= SHA256_BLOCK_LENGTH) { /* Process as many complete blocks as we can */ MEMCPY_BCOPY(context->buffer, data, SHA256_BLOCK_LENGTH); SHA256_Transform(context); context->bitcount += SHA256_BLOCK_LENGTH << 3; len -= SHA256_BLOCK_LENGTH; data += SHA256_BLOCK_LENGTH; } if (len > 0) { /* There's left-overs, so save 'em */ MEMCPY_BCOPY(context->buffer, data, len); context->bitcount += len << 3; } /* Clean up: */ usedspace = freespace = 0; } // #nocov void SHA256_Final(sha2_byte digest[], SHA256_CTX* context) { // #nocov sha2_word32 *d = (sha2_word32*)digest; unsigned int usedspace; /* Sanity check: */ assert(context != (SHA256_CTX*)0); /* If no digest buffer is passed, we don't bother doing this: */ if (digest != (sha2_byte*)0) { usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH; #if BYTE_ORDER == LITTLE_ENDIAN /* Convert FROM host byte order */ REVERSE64(context->bitcount,context->bitcount); #endif if (usedspace > 0) { /* Begin padding with a 1 bit: */ context->buffer[usedspace++] = 0x80; if (usedspace <= SHA256_SHORT_BLOCK_LENGTH) { /* Set-up for the last transform: */ MEMSET_BZERO(&context->buffer[usedspace], SHA256_SHORT_BLOCK_LENGTH - usedspace); } else { if (usedspace < SHA256_BLOCK_LENGTH) { MEMSET_BZERO(&context->buffer[usedspace], SHA256_BLOCK_LENGTH - usedspace); } /* Do second-to-last transform: */ SHA256_Transform(context); /* And set-up for the last transform: */ MEMSET_BZERO(context->buffer, SHA256_SHORT_BLOCK_LENGTH); } } else { /* Set-up for the last transform: */ MEMSET_BZERO(context->buffer, SHA256_SHORT_BLOCK_LENGTH); /* Begin padding with a 1 bit: */ *context->buffer = 0x80; } /* Set the bit count: */ /* *(sha2_word64*)&context->buffer[SHA256_SHORT_BLOCK_LENGTH] = context->bitcount; */ MEMCPY_BCOPY(&context->buffer[SHA256_SHORT_BLOCK_LENGTH], &context->bitcount, sizeof(sha2_word64)); /* Final transform: */ SHA256_Transform(context); #if BYTE_ORDER == LITTLE_ENDIAN { /* Convert TO host byte order */ int j; for (j = 0; j < 8; j++) { REVERSE32(context->state[j],context->state[j]); *d++ = context->state[j]; } } #else MEMCPY_BCOPY(d, context->state, SHA256_DIGEST_LENGTH); #endif } /* Clean up state data: */ MEMSET_BZERO(context, sizeof(SHA256_CTX)); usedspace = 0; } // #nocov char *SHA256_End(SHA256_CTX* context, char buffer[]) { /* #nocov start */ sha2_byte digest[SHA256_DIGEST_LENGTH], *d = digest; int i; /* Sanity check: */ assert(context != (SHA256_CTX*)0); if (buffer != (char*)0) { SHA256_Final(digest, context); for (i = 0; i < SHA256_DIGEST_LENGTH; i++) { *buffer++ = sha2_hex_digits[(*d & 0xf0) >> 4]; *buffer++ = sha2_hex_digits[*d & 0x0f]; d++; } *buffer = (char)0; } else { MEMSET_BZERO(context, sizeof(SHA256_CTX)); } MEMSET_BZERO(digest, SHA256_DIGEST_LENGTH); return buffer; } char* SHA256_Data(const sha2_byte* data, size_t len, char digest[SHA256_DIGEST_STRING_LENGTH]) { SHA256_CTX context; SHA256_Init(&context); SHA256_Update(&context, data, len); return SHA256_End(&context, digest); } /* #nocov end */ #endif /* SHA256_ functions */ /*** SHA-512: *********************************************************/ void SHA512_Init(SHA512_CTX* context) { if (context == (SHA512_CTX*)0) { return; // #nocov } MEMCPY_BCOPY(context->state, sha512_initial_hash_value, SHA512_DIGEST_LENGTH); MEMSET_BZERO(context->buffer, SHA512_BLOCK_LENGTH); context->bitcount[0] = context->bitcount[1] = 0; } #ifdef SHA2_UNROLL_TRANSFORM /* Unrolled SHA-512 round macros: */ #if BYTE_ORDER == LITTLE_ENDIAN #define ROUND512_0_TO_15(a,b,c,d,e,f,g,h) \ REVERSE64(W512[j], W512[j]); \ T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + \ K512[j] + W512[j]; \ (d) += T1, \ (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)), \ j++ #else /* BYTE_ORDER == LITTLE_ENDIAN */ #define ROUND512_0_TO_15(a,b,c,d,e,f,g,h) \ T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + \ K512[j] + W512[j]; \ (d) += T1; \ (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)); \ j++ #endif /* BYTE_ORDER == LITTLE_ENDIAN */ #define ROUND512(a,b,c,d,e,f,g,h) \ s0 = W512[(j+1)&0x0f]; \ s0 = sigma0_512(s0); \ s1 = W512[(j+14)&0x0f]; \ s1 = sigma1_512(s1); \ T1 = (h) + Sigma1_512(e) + Ch((e), (f), (g)) + K512[j] + \ (W512[j&0x0f] += s1 + W512[(j+9)&0x0f] + s0); \ (d) += T1; \ (h) = T1 + Sigma0_512(a) + Maj((a), (b), (c)); \ j++ static void SHA512_Transform(SHA512_CTX* context) { sha2_word64 a, b, c, d, e, f, g, h, s0, s1; sha2_word64 T1, *W512 = (sha2_word64*)context->buffer; int j; /* Initialize registers with the prev. intermediate value */ a = context->state[0]; b = context->state[1]; c = context->state[2]; d = context->state[3]; e = context->state[4]; f = context->state[5]; g = context->state[6]; h = context->state[7]; j = 0; do { ROUND512_0_TO_15(a,b,c,d,e,f,g,h); ROUND512_0_TO_15(h,a,b,c,d,e,f,g); ROUND512_0_TO_15(g,h,a,b,c,d,e,f); ROUND512_0_TO_15(f,g,h,a,b,c,d,e); ROUND512_0_TO_15(e,f,g,h,a,b,c,d); ROUND512_0_TO_15(d,e,f,g,h,a,b,c); ROUND512_0_TO_15(c,d,e,f,g,h,a,b); ROUND512_0_TO_15(b,c,d,e,f,g,h,a); } while (j < 16); /* Now for the remaining rounds up to 79: */ do { ROUND512(a,b,c,d,e,f,g,h); ROUND512(h,a,b,c,d,e,f,g); ROUND512(g,h,a,b,c,d,e,f); ROUND512(f,g,h,a,b,c,d,e); ROUND512(e,f,g,h,a,b,c,d); ROUND512(d,e,f,g,h,a,b,c); ROUND512(c,d,e,f,g,h,a,b); ROUND512(b,c,d,e,f,g,h,a); } while (j < 80); /* Compute the current intermediate hash value */ context->state[0] += a; context->state[1] += b; context->state[2] += c; context->state[3] += d; context->state[4] += e; context->state[5] += f; context->state[6] += g; context->state[7] += h; /* Clean up */ a = b = c = d = e = f = g = h = T1 = 0; } #else /* SHA2_UNROLL_TRANSFORM */ static void SHA512_Transform(SHA512_CTX* context) { sha2_word64 a, b, c, d, e, f, g, h, s0, s1; sha2_word64 T1, T2, *W512 = (sha2_word64*)context->buffer; int j; /* Initialize registers with the prev. intermediate value */ a = context->state[0]; b = context->state[1]; c = context->state[2]; d = context->state[3]; e = context->state[4]; f = context->state[5]; g = context->state[6]; h = context->state[7]; j = 0; do { #if BYTE_ORDER == LITTLE_ENDIAN /* Convert TO host byte order */ REVERSE64(W512[j], W512[j]); /* Apply the SHA-512 compression function to update a..h */ T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + W512[j]; #else /* BYTE_ORDER == LITTLE_ENDIAN */ /* Apply the SHA-512 compression function to update a..h with copy */ T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + W512[j]; #endif /* BYTE_ORDER == LITTLE_ENDIAN */ T2 = Sigma0_512(a) + Maj(a, b, c); h = g; g = f; f = e; e = d + T1; d = c; c = b; b = a; a = T1 + T2; j++; } while (j < 16); do { /* Part of the message block expansion: */ s0 = W512[(j+1)&0x0f]; s0 = sigma0_512(s0); s1 = W512[(j+14)&0x0f]; s1 = sigma1_512(s1); /* Apply the SHA-512 compression function to update a..h */ T1 = h + Sigma1_512(e) + Ch(e, f, g) + K512[j] + (W512[j&0x0f] += s1 + W512[(j+9)&0x0f] + s0); T2 = Sigma0_512(a) + Maj(a, b, c); h = g; g = f; f = e; e = d + T1; d = c; c = b; b = a; a = T1 + T2; j++; } while (j < 80); /* Compute the current intermediate hash value */ context->state[0] += a; context->state[1] += b; context->state[2] += c; context->state[3] += d; context->state[4] += e; context->state[5] += f; context->state[6] += g; context->state[7] += h; /* Clean up */ a = b = c = d = e = f = g = h = T1 = T2 = 0; } #endif /* SHA2_UNROLL_TRANSFORM */ void SHA512_Update(SHA512_CTX* context, const sha2_byte *data, size_t len) { unsigned int freespace, usedspace; if (len == 0) { /* Calling with no data is valid - we do nothing */ return; } /* Sanity check: */ assert(context != (SHA512_CTX*)0 && data != (sha2_byte*)0); usedspace = (context->bitcount[0] >> 3) % SHA512_BLOCK_LENGTH; if (usedspace > 0) { /* #nocov start */ /* Calculate how much free space is available in the buffer */ freespace = SHA512_BLOCK_LENGTH - usedspace; if (len >= freespace) { /* Fill the buffer completely and process it */ MEMCPY_BCOPY(&context->buffer[usedspace], data, freespace); ADDINC128(context->bitcount, freespace << 3); len -= freespace; data += freespace; SHA512_Transform(context); } else { /* The buffer is not yet full */ MEMCPY_BCOPY(&context->buffer[usedspace], data, len); ADDINC128(context->bitcount, len << 3); /* Clean up: */ usedspace = freespace = 0; return; } } /* #nocov end */ while (len >= SHA512_BLOCK_LENGTH) { /* Process as many complete blocks as we can */ MEMCPY_BCOPY(context->buffer, data, SHA512_BLOCK_LENGTH); SHA512_Transform(context); ADDINC128(context->bitcount, SHA512_BLOCK_LENGTH << 3); len -= SHA512_BLOCK_LENGTH; data += SHA512_BLOCK_LENGTH; } if (len > 0) { /* There's left-overs, so save 'em */ MEMCPY_BCOPY(context->buffer, data, len); ADDINC128(context->bitcount, len << 3); } /* Clean up: */ usedspace = freespace = 0; } static void SHA512_Last(SHA512_CTX* context) { unsigned int usedspace; usedspace = (context->bitcount[0] >> 3) % SHA512_BLOCK_LENGTH; #if BYTE_ORDER == LITTLE_ENDIAN /* Convert FROM host byte order */ REVERSE64(context->bitcount[0],context->bitcount[0]); REVERSE64(context->bitcount[1],context->bitcount[1]); #endif if (usedspace > 0) { /* Begin padding with a 1 bit: */ context->buffer[usedspace++] = 0x80; if (usedspace <= SHA512_SHORT_BLOCK_LENGTH) { /* Set-up for the last transform: */ MEMSET_BZERO(&context->buffer[usedspace], SHA512_SHORT_BLOCK_LENGTH - usedspace); } else { /* #nocov start */ if (usedspace < SHA512_BLOCK_LENGTH) { MEMSET_BZERO(&context->buffer[usedspace], SHA512_BLOCK_LENGTH - usedspace); } /* Do second-to-last transform: */ SHA512_Transform(context); /* And set-up for the last transform: */ MEMSET_BZERO(context->buffer, SHA512_BLOCK_LENGTH - 2); } /* #nocov end */ } else { /* Prepare for final transform: */ MEMSET_BZERO(context->buffer, SHA512_SHORT_BLOCK_LENGTH); /* Begin padding with a 1 bit: */ *context->buffer = 0x80; } /* Store the length of input data (in bits): */ /* *(sha2_word64*)&context->buffer[SHA512_SHORT_BLOCK_LENGTH] = context->bitcount[1]; */ /* *(sha2_word64*)&context->buffer[SHA512_SHORT_BLOCK_LENGTH+8] = context->bitcount[0]; */ MEMCPY_BCOPY(&context->buffer[SHA512_SHORT_BLOCK_LENGTH], &context->bitcount[1], sizeof(sha2_word64)); MEMCPY_BCOPY(&context->buffer[SHA512_SHORT_BLOCK_LENGTH+8], &context->bitcount[0], sizeof(sha2_word64)); /* Final transform: */ SHA512_Transform(context); } void SHA512_Final(sha2_byte digest[SHA512_DIGEST_LENGTH], SHA512_CTX* context) { sha2_word64 *d = (sha2_word64*)digest; /* Sanity check: */ assert(context != (SHA512_CTX*)0); /* If no digest buffer is passed, we don't bother doing this: */ if (digest != (sha2_byte*)0) { SHA512_Last(context); /* Save the hash data for output: */ #if BYTE_ORDER == LITTLE_ENDIAN { /* Convert TO host byte order */ int j; for (j = 0; j < 8; j++) { REVERSE64(context->state[j],context->state[j]); *d++ = context->state[j]; } } #else MEMCPY_BCOPY(d, context->state, SHA512_DIGEST_LENGTH); #endif } /* Zero out state data */ MEMSET_BZERO(context, sizeof(SHA512_CTX)); } char *SHA512_End(SHA512_CTX* context, char buffer[SHA512_DIGEST_STRING_LENGTH]) { /* #nocov start */ sha2_byte digest[SHA512_DIGEST_LENGTH], *d = digest; int i; /* Sanity check: */ assert(context != (SHA512_CTX*)0); if (buffer != (char*)0) { SHA512_Final(digest, context); for (i = 0; i < SHA512_DIGEST_LENGTH; i++) { *buffer++ = sha2_hex_digits[(*d & 0xf0) >> 4]; *buffer++ = sha2_hex_digits[*d & 0x0f]; d++; } *buffer = (char)0; } else { MEMSET_BZERO(context, sizeof(SHA512_CTX)); } MEMSET_BZERO(digest, SHA512_DIGEST_LENGTH); return buffer; } char* SHA512_Data(const sha2_byte* data, size_t len, char digest[SHA512_DIGEST_STRING_LENGTH]) { SHA512_CTX context; SHA512_Init(&context); SHA512_Update(&context, data, len); return SHA512_End(&context, digest); } /*** SHA-384: *********************************************************/ #if 0 void SHA384_Init(SHA384_CTX* context) { if (context == (SHA384_CTX*)0) { return; } MEMCPY_BCOPY(context->state, sha384_initial_hash_value, SHA512_DIGEST_LENGTH); MEMSET_BZERO(context->buffer, SHA384_BLOCK_LENGTH); context->bitcount[0] = context->bitcount[1] = 0; } void SHA384_Update(SHA384_CTX* context, const sha2_byte* data, size_t len) { SHA512_Update((SHA512_CTX*)context, data, len); } void SHA384_Final(sha2_byte digest[], SHA384_CTX* context) { sha2_word64 *d = (sha2_word64*)digest; /* Sanity check: */ assert(context != (SHA384_CTX*)0); /* If no digest buffer is passed, we don't bother doing this: */ if (digest != (sha2_byte*)0) { SHA512_Last((SHA512_CTX*)context); /* Save the hash data for output: */ #if BYTE_ORDER == LITTLE_ENDIAN { /* Convert TO host byte order */ int j; for (j = 0; j < 6; j++) { REVERSE64(context->state[j],context->state[j]); *d++ = context->state[j]; } } #else MEMCPY_BCOPY(d, context->state, SHA384_DIGEST_LENGTH); #endif } /* Zero out state data */ MEMSET_BZERO(context, sizeof(SHA384_CTX)); } char *SHA384_End(SHA384_CTX* context, char buffer[]) { sha2_byte digest[SHA384_DIGEST_LENGTH], *d = digest; int i; /* Sanity check: */ assert(context != (SHA384_CTX*)0); if (buffer != (char*)0) { SHA384_Final(digest, context); for (i = 0; i < SHA384_DIGEST_LENGTH; i++) { *buffer++ = sha2_hex_digits[(*d & 0xf0) >> 4]; *buffer++ = sha2_hex_digits[*d & 0x0f]; d++; } *buffer = (char)0; } else { MEMSET_BZERO(context, sizeof(SHA384_CTX)); } MEMSET_BZERO(digest, SHA384_DIGEST_LENGTH); return buffer; } char* SHA384_Data(const sha2_byte* data, size_t len, char digest[SHA384_DIGEST_STRING_LENGTH]) { SHA384_CTX context; SHA384_Init(&context); SHA384_Update(&context, data, len); return SHA384_End(&context, digest); } /* #nocov end */ #endif digest/src/crc32.h0000644000176200001440000007355012247504516013425 0ustar liggesusers/* crc32.h -- tables for rapid CRC calculation * Generated automatically by crc32.c */ local const unsigned long FAR crc_table[TBLS][256] = { { 0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL, 0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL, 0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL, 0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL, 0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL, 0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL, 0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL, 0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL, 0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL, 0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL, 0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL, 0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL, 0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL, 0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL, 0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL, 0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL, 0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL, 0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL, 0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL, 0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL, 0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL, 0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL, 0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL, 0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL, 0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL, 0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL, 0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL, 0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL, 0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL, 0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL, 0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL, 0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL, 0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL, 0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL, 0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL, 0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL, 0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL, 0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL, 0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL, 0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL, 0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL, 0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL, 0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL, 0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL, 0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL, 0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL, 0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL, 0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL, 0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL, 0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL, 0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL, 0x2d02ef8dUL #ifdef BYFOUR }, { 0x00000000UL, 0x191b3141UL, 0x32366282UL, 0x2b2d53c3UL, 0x646cc504UL, 0x7d77f445UL, 0x565aa786UL, 0x4f4196c7UL, 0xc8d98a08UL, 0xd1c2bb49UL, 0xfaefe88aUL, 0xe3f4d9cbUL, 0xacb54f0cUL, 0xb5ae7e4dUL, 0x9e832d8eUL, 0x87981ccfUL, 0x4ac21251UL, 0x53d92310UL, 0x78f470d3UL, 0x61ef4192UL, 0x2eaed755UL, 0x37b5e614UL, 0x1c98b5d7UL, 0x05838496UL, 0x821b9859UL, 0x9b00a918UL, 0xb02dfadbUL, 0xa936cb9aUL, 0xe6775d5dUL, 0xff6c6c1cUL, 0xd4413fdfUL, 0xcd5a0e9eUL, 0x958424a2UL, 0x8c9f15e3UL, 0xa7b24620UL, 0xbea97761UL, 0xf1e8e1a6UL, 0xe8f3d0e7UL, 0xc3de8324UL, 0xdac5b265UL, 0x5d5daeaaUL, 0x44469febUL, 0x6f6bcc28UL, 0x7670fd69UL, 0x39316baeUL, 0x202a5aefUL, 0x0b07092cUL, 0x121c386dUL, 0xdf4636f3UL, 0xc65d07b2UL, 0xed705471UL, 0xf46b6530UL, 0xbb2af3f7UL, 0xa231c2b6UL, 0x891c9175UL, 0x9007a034UL, 0x179fbcfbUL, 0x0e848dbaUL, 0x25a9de79UL, 0x3cb2ef38UL, 0x73f379ffUL, 0x6ae848beUL, 0x41c51b7dUL, 0x58de2a3cUL, 0xf0794f05UL, 0xe9627e44UL, 0xc24f2d87UL, 0xdb541cc6UL, 0x94158a01UL, 0x8d0ebb40UL, 0xa623e883UL, 0xbf38d9c2UL, 0x38a0c50dUL, 0x21bbf44cUL, 0x0a96a78fUL, 0x138d96ceUL, 0x5ccc0009UL, 0x45d73148UL, 0x6efa628bUL, 0x77e153caUL, 0xbabb5d54UL, 0xa3a06c15UL, 0x888d3fd6UL, 0x91960e97UL, 0xded79850UL, 0xc7cca911UL, 0xece1fad2UL, 0xf5facb93UL, 0x7262d75cUL, 0x6b79e61dUL, 0x4054b5deUL, 0x594f849fUL, 0x160e1258UL, 0x0f152319UL, 0x243870daUL, 0x3d23419bUL, 0x65fd6ba7UL, 0x7ce65ae6UL, 0x57cb0925UL, 0x4ed03864UL, 0x0191aea3UL, 0x188a9fe2UL, 0x33a7cc21UL, 0x2abcfd60UL, 0xad24e1afUL, 0xb43fd0eeUL, 0x9f12832dUL, 0x8609b26cUL, 0xc94824abUL, 0xd05315eaUL, 0xfb7e4629UL, 0xe2657768UL, 0x2f3f79f6UL, 0x362448b7UL, 0x1d091b74UL, 0x04122a35UL, 0x4b53bcf2UL, 0x52488db3UL, 0x7965de70UL, 0x607eef31UL, 0xe7e6f3feUL, 0xfefdc2bfUL, 0xd5d0917cUL, 0xcccba03dUL, 0x838a36faUL, 0x9a9107bbUL, 0xb1bc5478UL, 0xa8a76539UL, 0x3b83984bUL, 0x2298a90aUL, 0x09b5fac9UL, 0x10aecb88UL, 0x5fef5d4fUL, 0x46f46c0eUL, 0x6dd93fcdUL, 0x74c20e8cUL, 0xf35a1243UL, 0xea412302UL, 0xc16c70c1UL, 0xd8774180UL, 0x9736d747UL, 0x8e2de606UL, 0xa500b5c5UL, 0xbc1b8484UL, 0x71418a1aUL, 0x685abb5bUL, 0x4377e898UL, 0x5a6cd9d9UL, 0x152d4f1eUL, 0x0c367e5fUL, 0x271b2d9cUL, 0x3e001cddUL, 0xb9980012UL, 0xa0833153UL, 0x8bae6290UL, 0x92b553d1UL, 0xddf4c516UL, 0xc4eff457UL, 0xefc2a794UL, 0xf6d996d5UL, 0xae07bce9UL, 0xb71c8da8UL, 0x9c31de6bUL, 0x852aef2aUL, 0xca6b79edUL, 0xd37048acUL, 0xf85d1b6fUL, 0xe1462a2eUL, 0x66de36e1UL, 0x7fc507a0UL, 0x54e85463UL, 0x4df36522UL, 0x02b2f3e5UL, 0x1ba9c2a4UL, 0x30849167UL, 0x299fa026UL, 0xe4c5aeb8UL, 0xfdde9ff9UL, 0xd6f3cc3aUL, 0xcfe8fd7bUL, 0x80a96bbcUL, 0x99b25afdUL, 0xb29f093eUL, 0xab84387fUL, 0x2c1c24b0UL, 0x350715f1UL, 0x1e2a4632UL, 0x07317773UL, 0x4870e1b4UL, 0x516bd0f5UL, 0x7a468336UL, 0x635db277UL, 0xcbfad74eUL, 0xd2e1e60fUL, 0xf9ccb5ccUL, 0xe0d7848dUL, 0xaf96124aUL, 0xb68d230bUL, 0x9da070c8UL, 0x84bb4189UL, 0x03235d46UL, 0x1a386c07UL, 0x31153fc4UL, 0x280e0e85UL, 0x674f9842UL, 0x7e54a903UL, 0x5579fac0UL, 0x4c62cb81UL, 0x8138c51fUL, 0x9823f45eUL, 0xb30ea79dUL, 0xaa1596dcUL, 0xe554001bUL, 0xfc4f315aUL, 0xd7626299UL, 0xce7953d8UL, 0x49e14f17UL, 0x50fa7e56UL, 0x7bd72d95UL, 0x62cc1cd4UL, 0x2d8d8a13UL, 0x3496bb52UL, 0x1fbbe891UL, 0x06a0d9d0UL, 0x5e7ef3ecUL, 0x4765c2adUL, 0x6c48916eUL, 0x7553a02fUL, 0x3a1236e8UL, 0x230907a9UL, 0x0824546aUL, 0x113f652bUL, 0x96a779e4UL, 0x8fbc48a5UL, 0xa4911b66UL, 0xbd8a2a27UL, 0xf2cbbce0UL, 0xebd08da1UL, 0xc0fdde62UL, 0xd9e6ef23UL, 0x14bce1bdUL, 0x0da7d0fcUL, 0x268a833fUL, 0x3f91b27eUL, 0x70d024b9UL, 0x69cb15f8UL, 0x42e6463bUL, 0x5bfd777aUL, 0xdc656bb5UL, 0xc57e5af4UL, 0xee530937UL, 0xf7483876UL, 0xb809aeb1UL, 0xa1129ff0UL, 0x8a3fcc33UL, 0x9324fd72UL }, { 0x00000000UL, 0x01c26a37UL, 0x0384d46eUL, 0x0246be59UL, 0x0709a8dcUL, 0x06cbc2ebUL, 0x048d7cb2UL, 0x054f1685UL, 0x0e1351b8UL, 0x0fd13b8fUL, 0x0d9785d6UL, 0x0c55efe1UL, 0x091af964UL, 0x08d89353UL, 0x0a9e2d0aUL, 0x0b5c473dUL, 0x1c26a370UL, 0x1de4c947UL, 0x1fa2771eUL, 0x1e601d29UL, 0x1b2f0bacUL, 0x1aed619bUL, 0x18abdfc2UL, 0x1969b5f5UL, 0x1235f2c8UL, 0x13f798ffUL, 0x11b126a6UL, 0x10734c91UL, 0x153c5a14UL, 0x14fe3023UL, 0x16b88e7aUL, 0x177ae44dUL, 0x384d46e0UL, 0x398f2cd7UL, 0x3bc9928eUL, 0x3a0bf8b9UL, 0x3f44ee3cUL, 0x3e86840bUL, 0x3cc03a52UL, 0x3d025065UL, 0x365e1758UL, 0x379c7d6fUL, 0x35dac336UL, 0x3418a901UL, 0x3157bf84UL, 0x3095d5b3UL, 0x32d36beaUL, 0x331101ddUL, 0x246be590UL, 0x25a98fa7UL, 0x27ef31feUL, 0x262d5bc9UL, 0x23624d4cUL, 0x22a0277bUL, 0x20e69922UL, 0x2124f315UL, 0x2a78b428UL, 0x2bbade1fUL, 0x29fc6046UL, 0x283e0a71UL, 0x2d711cf4UL, 0x2cb376c3UL, 0x2ef5c89aUL, 0x2f37a2adUL, 0x709a8dc0UL, 0x7158e7f7UL, 0x731e59aeUL, 0x72dc3399UL, 0x7793251cUL, 0x76514f2bUL, 0x7417f172UL, 0x75d59b45UL, 0x7e89dc78UL, 0x7f4bb64fUL, 0x7d0d0816UL, 0x7ccf6221UL, 0x798074a4UL, 0x78421e93UL, 0x7a04a0caUL, 0x7bc6cafdUL, 0x6cbc2eb0UL, 0x6d7e4487UL, 0x6f38fadeUL, 0x6efa90e9UL, 0x6bb5866cUL, 0x6a77ec5bUL, 0x68315202UL, 0x69f33835UL, 0x62af7f08UL, 0x636d153fUL, 0x612bab66UL, 0x60e9c151UL, 0x65a6d7d4UL, 0x6464bde3UL, 0x662203baUL, 0x67e0698dUL, 0x48d7cb20UL, 0x4915a117UL, 0x4b531f4eUL, 0x4a917579UL, 0x4fde63fcUL, 0x4e1c09cbUL, 0x4c5ab792UL, 0x4d98dda5UL, 0x46c49a98UL, 0x4706f0afUL, 0x45404ef6UL, 0x448224c1UL, 0x41cd3244UL, 0x400f5873UL, 0x4249e62aUL, 0x438b8c1dUL, 0x54f16850UL, 0x55330267UL, 0x5775bc3eUL, 0x56b7d609UL, 0x53f8c08cUL, 0x523aaabbUL, 0x507c14e2UL, 0x51be7ed5UL, 0x5ae239e8UL, 0x5b2053dfUL, 0x5966ed86UL, 0x58a487b1UL, 0x5deb9134UL, 0x5c29fb03UL, 0x5e6f455aUL, 0x5fad2f6dUL, 0xe1351b80UL, 0xe0f771b7UL, 0xe2b1cfeeUL, 0xe373a5d9UL, 0xe63cb35cUL, 0xe7fed96bUL, 0xe5b86732UL, 0xe47a0d05UL, 0xef264a38UL, 0xeee4200fUL, 0xeca29e56UL, 0xed60f461UL, 0xe82fe2e4UL, 0xe9ed88d3UL, 0xebab368aUL, 0xea695cbdUL, 0xfd13b8f0UL, 0xfcd1d2c7UL, 0xfe976c9eUL, 0xff5506a9UL, 0xfa1a102cUL, 0xfbd87a1bUL, 0xf99ec442UL, 0xf85cae75UL, 0xf300e948UL, 0xf2c2837fUL, 0xf0843d26UL, 0xf1465711UL, 0xf4094194UL, 0xf5cb2ba3UL, 0xf78d95faUL, 0xf64fffcdUL, 0xd9785d60UL, 0xd8ba3757UL, 0xdafc890eUL, 0xdb3ee339UL, 0xde71f5bcUL, 0xdfb39f8bUL, 0xddf521d2UL, 0xdc374be5UL, 0xd76b0cd8UL, 0xd6a966efUL, 0xd4efd8b6UL, 0xd52db281UL, 0xd062a404UL, 0xd1a0ce33UL, 0xd3e6706aUL, 0xd2241a5dUL, 0xc55efe10UL, 0xc49c9427UL, 0xc6da2a7eUL, 0xc7184049UL, 0xc25756ccUL, 0xc3953cfbUL, 0xc1d382a2UL, 0xc011e895UL, 0xcb4dafa8UL, 0xca8fc59fUL, 0xc8c97bc6UL, 0xc90b11f1UL, 0xcc440774UL, 0xcd866d43UL, 0xcfc0d31aUL, 0xce02b92dUL, 0x91af9640UL, 0x906dfc77UL, 0x922b422eUL, 0x93e92819UL, 0x96a63e9cUL, 0x976454abUL, 0x9522eaf2UL, 0x94e080c5UL, 0x9fbcc7f8UL, 0x9e7eadcfUL, 0x9c381396UL, 0x9dfa79a1UL, 0x98b56f24UL, 0x99770513UL, 0x9b31bb4aUL, 0x9af3d17dUL, 0x8d893530UL, 0x8c4b5f07UL, 0x8e0de15eUL, 0x8fcf8b69UL, 0x8a809decUL, 0x8b42f7dbUL, 0x89044982UL, 0x88c623b5UL, 0x839a6488UL, 0x82580ebfUL, 0x801eb0e6UL, 0x81dcdad1UL, 0x8493cc54UL, 0x8551a663UL, 0x8717183aUL, 0x86d5720dUL, 0xa9e2d0a0UL, 0xa820ba97UL, 0xaa6604ceUL, 0xaba46ef9UL, 0xaeeb787cUL, 0xaf29124bUL, 0xad6fac12UL, 0xacadc625UL, 0xa7f18118UL, 0xa633eb2fUL, 0xa4755576UL, 0xa5b73f41UL, 0xa0f829c4UL, 0xa13a43f3UL, 0xa37cfdaaUL, 0xa2be979dUL, 0xb5c473d0UL, 0xb40619e7UL, 0xb640a7beUL, 0xb782cd89UL, 0xb2cddb0cUL, 0xb30fb13bUL, 0xb1490f62UL, 0xb08b6555UL, 0xbbd72268UL, 0xba15485fUL, 0xb853f606UL, 0xb9919c31UL, 0xbcde8ab4UL, 0xbd1ce083UL, 0xbf5a5edaUL, 0xbe9834edUL }, { 0x00000000UL, 0xb8bc6765UL, 0xaa09c88bUL, 0x12b5afeeUL, 0x8f629757UL, 0x37def032UL, 0x256b5fdcUL, 0x9dd738b9UL, 0xc5b428efUL, 0x7d084f8aUL, 0x6fbde064UL, 0xd7018701UL, 0x4ad6bfb8UL, 0xf26ad8ddUL, 0xe0df7733UL, 0x58631056UL, 0x5019579fUL, 0xe8a530faUL, 0xfa109f14UL, 0x42acf871UL, 0xdf7bc0c8UL, 0x67c7a7adUL, 0x75720843UL, 0xcdce6f26UL, 0x95ad7f70UL, 0x2d111815UL, 0x3fa4b7fbUL, 0x8718d09eUL, 0x1acfe827UL, 0xa2738f42UL, 0xb0c620acUL, 0x087a47c9UL, 0xa032af3eUL, 0x188ec85bUL, 0x0a3b67b5UL, 0xb28700d0UL, 0x2f503869UL, 0x97ec5f0cUL, 0x8559f0e2UL, 0x3de59787UL, 0x658687d1UL, 0xdd3ae0b4UL, 0xcf8f4f5aUL, 0x7733283fUL, 0xeae41086UL, 0x525877e3UL, 0x40edd80dUL, 0xf851bf68UL, 0xf02bf8a1UL, 0x48979fc4UL, 0x5a22302aUL, 0xe29e574fUL, 0x7f496ff6UL, 0xc7f50893UL, 0xd540a77dUL, 0x6dfcc018UL, 0x359fd04eUL, 0x8d23b72bUL, 0x9f9618c5UL, 0x272a7fa0UL, 0xbafd4719UL, 0x0241207cUL, 0x10f48f92UL, 0xa848e8f7UL, 0x9b14583dUL, 0x23a83f58UL, 0x311d90b6UL, 0x89a1f7d3UL, 0x1476cf6aUL, 0xaccaa80fUL, 0xbe7f07e1UL, 0x06c36084UL, 0x5ea070d2UL, 0xe61c17b7UL, 0xf4a9b859UL, 0x4c15df3cUL, 0xd1c2e785UL, 0x697e80e0UL, 0x7bcb2f0eUL, 0xc377486bUL, 0xcb0d0fa2UL, 0x73b168c7UL, 0x6104c729UL, 0xd9b8a04cUL, 0x446f98f5UL, 0xfcd3ff90UL, 0xee66507eUL, 0x56da371bUL, 0x0eb9274dUL, 0xb6054028UL, 0xa4b0efc6UL, 0x1c0c88a3UL, 0x81dbb01aUL, 0x3967d77fUL, 0x2bd27891UL, 0x936e1ff4UL, 0x3b26f703UL, 0x839a9066UL, 0x912f3f88UL, 0x299358edUL, 0xb4446054UL, 0x0cf80731UL, 0x1e4da8dfUL, 0xa6f1cfbaUL, 0xfe92dfecUL, 0x462eb889UL, 0x549b1767UL, 0xec277002UL, 0x71f048bbUL, 0xc94c2fdeUL, 0xdbf98030UL, 0x6345e755UL, 0x6b3fa09cUL, 0xd383c7f9UL, 0xc1366817UL, 0x798a0f72UL, 0xe45d37cbUL, 0x5ce150aeUL, 0x4e54ff40UL, 0xf6e89825UL, 0xae8b8873UL, 0x1637ef16UL, 0x048240f8UL, 0xbc3e279dUL, 0x21e91f24UL, 0x99557841UL, 0x8be0d7afUL, 0x335cb0caUL, 0xed59b63bUL, 0x55e5d15eUL, 0x47507eb0UL, 0xffec19d5UL, 0x623b216cUL, 0xda874609UL, 0xc832e9e7UL, 0x708e8e82UL, 0x28ed9ed4UL, 0x9051f9b1UL, 0x82e4565fUL, 0x3a58313aUL, 0xa78f0983UL, 0x1f336ee6UL, 0x0d86c108UL, 0xb53aa66dUL, 0xbd40e1a4UL, 0x05fc86c1UL, 0x1749292fUL, 0xaff54e4aUL, 0x322276f3UL, 0x8a9e1196UL, 0x982bbe78UL, 0x2097d91dUL, 0x78f4c94bUL, 0xc048ae2eUL, 0xd2fd01c0UL, 0x6a4166a5UL, 0xf7965e1cUL, 0x4f2a3979UL, 0x5d9f9697UL, 0xe523f1f2UL, 0x4d6b1905UL, 0xf5d77e60UL, 0xe762d18eUL, 0x5fdeb6ebUL, 0xc2098e52UL, 0x7ab5e937UL, 0x680046d9UL, 0xd0bc21bcUL, 0x88df31eaUL, 0x3063568fUL, 0x22d6f961UL, 0x9a6a9e04UL, 0x07bda6bdUL, 0xbf01c1d8UL, 0xadb46e36UL, 0x15080953UL, 0x1d724e9aUL, 0xa5ce29ffUL, 0xb77b8611UL, 0x0fc7e174UL, 0x9210d9cdUL, 0x2aacbea8UL, 0x38191146UL, 0x80a57623UL, 0xd8c66675UL, 0x607a0110UL, 0x72cfaefeUL, 0xca73c99bUL, 0x57a4f122UL, 0xef189647UL, 0xfdad39a9UL, 0x45115eccUL, 0x764dee06UL, 0xcef18963UL, 0xdc44268dUL, 0x64f841e8UL, 0xf92f7951UL, 0x41931e34UL, 0x5326b1daUL, 0xeb9ad6bfUL, 0xb3f9c6e9UL, 0x0b45a18cUL, 0x19f00e62UL, 0xa14c6907UL, 0x3c9b51beUL, 0x842736dbUL, 0x96929935UL, 0x2e2efe50UL, 0x2654b999UL, 0x9ee8defcUL, 0x8c5d7112UL, 0x34e11677UL, 0xa9362eceUL, 0x118a49abUL, 0x033fe645UL, 0xbb838120UL, 0xe3e09176UL, 0x5b5cf613UL, 0x49e959fdUL, 0xf1553e98UL, 0x6c820621UL, 0xd43e6144UL, 0xc68bceaaUL, 0x7e37a9cfUL, 0xd67f4138UL, 0x6ec3265dUL, 0x7c7689b3UL, 0xc4caeed6UL, 0x591dd66fUL, 0xe1a1b10aUL, 0xf3141ee4UL, 0x4ba87981UL, 0x13cb69d7UL, 0xab770eb2UL, 0xb9c2a15cUL, 0x017ec639UL, 0x9ca9fe80UL, 0x241599e5UL, 0x36a0360bUL, 0x8e1c516eUL, 0x866616a7UL, 0x3eda71c2UL, 0x2c6fde2cUL, 0x94d3b949UL, 0x090481f0UL, 0xb1b8e695UL, 0xa30d497bUL, 0x1bb12e1eUL, 0x43d23e48UL, 0xfb6e592dUL, 0xe9dbf6c3UL, 0x516791a6UL, 0xccb0a91fUL, 0x740cce7aUL, 0x66b96194UL, 0xde0506f1UL }, { 0x00000000UL, 0x96300777UL, 0x2c610eeeUL, 0xba510999UL, 0x19c46d07UL, 0x8ff46a70UL, 0x35a563e9UL, 0xa395649eUL, 0x3288db0eUL, 0xa4b8dc79UL, 0x1ee9d5e0UL, 0x88d9d297UL, 0x2b4cb609UL, 0xbd7cb17eUL, 0x072db8e7UL, 0x911dbf90UL, 0x6410b71dUL, 0xf220b06aUL, 0x4871b9f3UL, 0xde41be84UL, 0x7dd4da1aUL, 0xebe4dd6dUL, 0x51b5d4f4UL, 0xc785d383UL, 0x56986c13UL, 0xc0a86b64UL, 0x7af962fdUL, 0xecc9658aUL, 0x4f5c0114UL, 0xd96c0663UL, 0x633d0ffaUL, 0xf50d088dUL, 0xc8206e3bUL, 0x5e10694cUL, 0xe44160d5UL, 0x727167a2UL, 0xd1e4033cUL, 0x47d4044bUL, 0xfd850dd2UL, 0x6bb50aa5UL, 0xfaa8b535UL, 0x6c98b242UL, 0xd6c9bbdbUL, 0x40f9bcacUL, 0xe36cd832UL, 0x755cdf45UL, 0xcf0dd6dcUL, 0x593dd1abUL, 0xac30d926UL, 0x3a00de51UL, 0x8051d7c8UL, 0x1661d0bfUL, 0xb5f4b421UL, 0x23c4b356UL, 0x9995bacfUL, 0x0fa5bdb8UL, 0x9eb80228UL, 0x0888055fUL, 0xb2d90cc6UL, 0x24e90bb1UL, 0x877c6f2fUL, 0x114c6858UL, 0xab1d61c1UL, 0x3d2d66b6UL, 0x9041dc76UL, 0x0671db01UL, 0xbc20d298UL, 0x2a10d5efUL, 0x8985b171UL, 0x1fb5b606UL, 0xa5e4bf9fUL, 0x33d4b8e8UL, 0xa2c90778UL, 0x34f9000fUL, 0x8ea80996UL, 0x18980ee1UL, 0xbb0d6a7fUL, 0x2d3d6d08UL, 0x976c6491UL, 0x015c63e6UL, 0xf4516b6bUL, 0x62616c1cUL, 0xd8306585UL, 0x4e0062f2UL, 0xed95066cUL, 0x7ba5011bUL, 0xc1f40882UL, 0x57c40ff5UL, 0xc6d9b065UL, 0x50e9b712UL, 0xeab8be8bUL, 0x7c88b9fcUL, 0xdf1ddd62UL, 0x492dda15UL, 0xf37cd38cUL, 0x654cd4fbUL, 0x5861b24dUL, 0xce51b53aUL, 0x7400bca3UL, 0xe230bbd4UL, 0x41a5df4aUL, 0xd795d83dUL, 0x6dc4d1a4UL, 0xfbf4d6d3UL, 0x6ae96943UL, 0xfcd96e34UL, 0x468867adUL, 0xd0b860daUL, 0x732d0444UL, 0xe51d0333UL, 0x5f4c0aaaUL, 0xc97c0dddUL, 0x3c710550UL, 0xaa410227UL, 0x10100bbeUL, 0x86200cc9UL, 0x25b56857UL, 0xb3856f20UL, 0x09d466b9UL, 0x9fe461ceUL, 0x0ef9de5eUL, 0x98c9d929UL, 0x2298d0b0UL, 0xb4a8d7c7UL, 0x173db359UL, 0x810db42eUL, 0x3b5cbdb7UL, 0xad6cbac0UL, 0x2083b8edUL, 0xb6b3bf9aUL, 0x0ce2b603UL, 0x9ad2b174UL, 0x3947d5eaUL, 0xaf77d29dUL, 0x1526db04UL, 0x8316dc73UL, 0x120b63e3UL, 0x843b6494UL, 0x3e6a6d0dUL, 0xa85a6a7aUL, 0x0bcf0ee4UL, 0x9dff0993UL, 0x27ae000aUL, 0xb19e077dUL, 0x44930ff0UL, 0xd2a30887UL, 0x68f2011eUL, 0xfec20669UL, 0x5d5762f7UL, 0xcb676580UL, 0x71366c19UL, 0xe7066b6eUL, 0x761bd4feUL, 0xe02bd389UL, 0x5a7ada10UL, 0xcc4add67UL, 0x6fdfb9f9UL, 0xf9efbe8eUL, 0x43beb717UL, 0xd58eb060UL, 0xe8a3d6d6UL, 0x7e93d1a1UL, 0xc4c2d838UL, 0x52f2df4fUL, 0xf167bbd1UL, 0x6757bca6UL, 0xdd06b53fUL, 0x4b36b248UL, 0xda2b0dd8UL, 0x4c1b0aafUL, 0xf64a0336UL, 0x607a0441UL, 0xc3ef60dfUL, 0x55df67a8UL, 0xef8e6e31UL, 0x79be6946UL, 0x8cb361cbUL, 0x1a8366bcUL, 0xa0d26f25UL, 0x36e26852UL, 0x95770cccUL, 0x03470bbbUL, 0xb9160222UL, 0x2f260555UL, 0xbe3bbac5UL, 0x280bbdb2UL, 0x925ab42bUL, 0x046ab35cUL, 0xa7ffd7c2UL, 0x31cfd0b5UL, 0x8b9ed92cUL, 0x1daede5bUL, 0xb0c2649bUL, 0x26f263ecUL, 0x9ca36a75UL, 0x0a936d02UL, 0xa906099cUL, 0x3f360eebUL, 0x85670772UL, 0x13570005UL, 0x824abf95UL, 0x147ab8e2UL, 0xae2bb17bUL, 0x381bb60cUL, 0x9b8ed292UL, 0x0dbed5e5UL, 0xb7efdc7cUL, 0x21dfdb0bUL, 0xd4d2d386UL, 0x42e2d4f1UL, 0xf8b3dd68UL, 0x6e83da1fUL, 0xcd16be81UL, 0x5b26b9f6UL, 0xe177b06fUL, 0x7747b718UL, 0xe65a0888UL, 0x706a0fffUL, 0xca3b0666UL, 0x5c0b0111UL, 0xff9e658fUL, 0x69ae62f8UL, 0xd3ff6b61UL, 0x45cf6c16UL, 0x78e20aa0UL, 0xeed20dd7UL, 0x5483044eUL, 0xc2b30339UL, 0x612667a7UL, 0xf71660d0UL, 0x4d476949UL, 0xdb776e3eUL, 0x4a6ad1aeUL, 0xdc5ad6d9UL, 0x660bdf40UL, 0xf03bd837UL, 0x53aebca9UL, 0xc59ebbdeUL, 0x7fcfb247UL, 0xe9ffb530UL, 0x1cf2bdbdUL, 0x8ac2bacaUL, 0x3093b353UL, 0xa6a3b424UL, 0x0536d0baUL, 0x9306d7cdUL, 0x2957de54UL, 0xbf67d923UL, 0x2e7a66b3UL, 0xb84a61c4UL, 0x021b685dUL, 0x942b6f2aUL, 0x37be0bb4UL, 0xa18e0cc3UL, 0x1bdf055aUL, 0x8def022dUL }, { 0x00000000UL, 0x41311b19UL, 0x82623632UL, 0xc3532d2bUL, 0x04c56c64UL, 0x45f4777dUL, 0x86a75a56UL, 0xc796414fUL, 0x088ad9c8UL, 0x49bbc2d1UL, 0x8ae8effaUL, 0xcbd9f4e3UL, 0x0c4fb5acUL, 0x4d7eaeb5UL, 0x8e2d839eUL, 0xcf1c9887UL, 0x5112c24aUL, 0x1023d953UL, 0xd370f478UL, 0x9241ef61UL, 0x55d7ae2eUL, 0x14e6b537UL, 0xd7b5981cUL, 0x96848305UL, 0x59981b82UL, 0x18a9009bUL, 0xdbfa2db0UL, 0x9acb36a9UL, 0x5d5d77e6UL, 0x1c6c6cffUL, 0xdf3f41d4UL, 0x9e0e5acdUL, 0xa2248495UL, 0xe3159f8cUL, 0x2046b2a7UL, 0x6177a9beUL, 0xa6e1e8f1UL, 0xe7d0f3e8UL, 0x2483dec3UL, 0x65b2c5daUL, 0xaaae5d5dUL, 0xeb9f4644UL, 0x28cc6b6fUL, 0x69fd7076UL, 0xae6b3139UL, 0xef5a2a20UL, 0x2c09070bUL, 0x6d381c12UL, 0xf33646dfUL, 0xb2075dc6UL, 0x715470edUL, 0x30656bf4UL, 0xf7f32abbUL, 0xb6c231a2UL, 0x75911c89UL, 0x34a00790UL, 0xfbbc9f17UL, 0xba8d840eUL, 0x79dea925UL, 0x38efb23cUL, 0xff79f373UL, 0xbe48e86aUL, 0x7d1bc541UL, 0x3c2ade58UL, 0x054f79f0UL, 0x447e62e9UL, 0x872d4fc2UL, 0xc61c54dbUL, 0x018a1594UL, 0x40bb0e8dUL, 0x83e823a6UL, 0xc2d938bfUL, 0x0dc5a038UL, 0x4cf4bb21UL, 0x8fa7960aUL, 0xce968d13UL, 0x0900cc5cUL, 0x4831d745UL, 0x8b62fa6eUL, 0xca53e177UL, 0x545dbbbaUL, 0x156ca0a3UL, 0xd63f8d88UL, 0x970e9691UL, 0x5098d7deUL, 0x11a9ccc7UL, 0xd2fae1ecUL, 0x93cbfaf5UL, 0x5cd76272UL, 0x1de6796bUL, 0xdeb55440UL, 0x9f844f59UL, 0x58120e16UL, 0x1923150fUL, 0xda703824UL, 0x9b41233dUL, 0xa76bfd65UL, 0xe65ae67cUL, 0x2509cb57UL, 0x6438d04eUL, 0xa3ae9101UL, 0xe29f8a18UL, 0x21cca733UL, 0x60fdbc2aUL, 0xafe124adUL, 0xeed03fb4UL, 0x2d83129fUL, 0x6cb20986UL, 0xab2448c9UL, 0xea1553d0UL, 0x29467efbUL, 0x687765e2UL, 0xf6793f2fUL, 0xb7482436UL, 0x741b091dUL, 0x352a1204UL, 0xf2bc534bUL, 0xb38d4852UL, 0x70de6579UL, 0x31ef7e60UL, 0xfef3e6e7UL, 0xbfc2fdfeUL, 0x7c91d0d5UL, 0x3da0cbccUL, 0xfa368a83UL, 0xbb07919aUL, 0x7854bcb1UL, 0x3965a7a8UL, 0x4b98833bUL, 0x0aa99822UL, 0xc9fab509UL, 0x88cbae10UL, 0x4f5def5fUL, 0x0e6cf446UL, 0xcd3fd96dUL, 0x8c0ec274UL, 0x43125af3UL, 0x022341eaUL, 0xc1706cc1UL, 0x804177d8UL, 0x47d73697UL, 0x06e62d8eUL, 0xc5b500a5UL, 0x84841bbcUL, 0x1a8a4171UL, 0x5bbb5a68UL, 0x98e87743UL, 0xd9d96c5aUL, 0x1e4f2d15UL, 0x5f7e360cUL, 0x9c2d1b27UL, 0xdd1c003eUL, 0x120098b9UL, 0x533183a0UL, 0x9062ae8bUL, 0xd153b592UL, 0x16c5f4ddUL, 0x57f4efc4UL, 0x94a7c2efUL, 0xd596d9f6UL, 0xe9bc07aeUL, 0xa88d1cb7UL, 0x6bde319cUL, 0x2aef2a85UL, 0xed796bcaUL, 0xac4870d3UL, 0x6f1b5df8UL, 0x2e2a46e1UL, 0xe136de66UL, 0xa007c57fUL, 0x6354e854UL, 0x2265f34dUL, 0xe5f3b202UL, 0xa4c2a91bUL, 0x67918430UL, 0x26a09f29UL, 0xb8aec5e4UL, 0xf99fdefdUL, 0x3accf3d6UL, 0x7bfde8cfUL, 0xbc6ba980UL, 0xfd5ab299UL, 0x3e099fb2UL, 0x7f3884abUL, 0xb0241c2cUL, 0xf1150735UL, 0x32462a1eUL, 0x73773107UL, 0xb4e17048UL, 0xf5d06b51UL, 0x3683467aUL, 0x77b25d63UL, 0x4ed7facbUL, 0x0fe6e1d2UL, 0xccb5ccf9UL, 0x8d84d7e0UL, 0x4a1296afUL, 0x0b238db6UL, 0xc870a09dUL, 0x8941bb84UL, 0x465d2303UL, 0x076c381aUL, 0xc43f1531UL, 0x850e0e28UL, 0x42984f67UL, 0x03a9547eUL, 0xc0fa7955UL, 0x81cb624cUL, 0x1fc53881UL, 0x5ef42398UL, 0x9da70eb3UL, 0xdc9615aaUL, 0x1b0054e5UL, 0x5a314ffcUL, 0x996262d7UL, 0xd85379ceUL, 0x174fe149UL, 0x567efa50UL, 0x952dd77bUL, 0xd41ccc62UL, 0x138a8d2dUL, 0x52bb9634UL, 0x91e8bb1fUL, 0xd0d9a006UL, 0xecf37e5eUL, 0xadc26547UL, 0x6e91486cUL, 0x2fa05375UL, 0xe836123aUL, 0xa9070923UL, 0x6a542408UL, 0x2b653f11UL, 0xe479a796UL, 0xa548bc8fUL, 0x661b91a4UL, 0x272a8abdUL, 0xe0bccbf2UL, 0xa18dd0ebUL, 0x62defdc0UL, 0x23efe6d9UL, 0xbde1bc14UL, 0xfcd0a70dUL, 0x3f838a26UL, 0x7eb2913fUL, 0xb924d070UL, 0xf815cb69UL, 0x3b46e642UL, 0x7a77fd5bUL, 0xb56b65dcUL, 0xf45a7ec5UL, 0x370953eeUL, 0x763848f7UL, 0xb1ae09b8UL, 0xf09f12a1UL, 0x33cc3f8aUL, 0x72fd2493UL }, { 0x00000000UL, 0x376ac201UL, 0x6ed48403UL, 0x59be4602UL, 0xdca80907UL, 0xebc2cb06UL, 0xb27c8d04UL, 0x85164f05UL, 0xb851130eUL, 0x8f3bd10fUL, 0xd685970dUL, 0xe1ef550cUL, 0x64f91a09UL, 0x5393d808UL, 0x0a2d9e0aUL, 0x3d475c0bUL, 0x70a3261cUL, 0x47c9e41dUL, 0x1e77a21fUL, 0x291d601eUL, 0xac0b2f1bUL, 0x9b61ed1aUL, 0xc2dfab18UL, 0xf5b56919UL, 0xc8f23512UL, 0xff98f713UL, 0xa626b111UL, 0x914c7310UL, 0x145a3c15UL, 0x2330fe14UL, 0x7a8eb816UL, 0x4de47a17UL, 0xe0464d38UL, 0xd72c8f39UL, 0x8e92c93bUL, 0xb9f80b3aUL, 0x3cee443fUL, 0x0b84863eUL, 0x523ac03cUL, 0x6550023dUL, 0x58175e36UL, 0x6f7d9c37UL, 0x36c3da35UL, 0x01a91834UL, 0x84bf5731UL, 0xb3d59530UL, 0xea6bd332UL, 0xdd011133UL, 0x90e56b24UL, 0xa78fa925UL, 0xfe31ef27UL, 0xc95b2d26UL, 0x4c4d6223UL, 0x7b27a022UL, 0x2299e620UL, 0x15f32421UL, 0x28b4782aUL, 0x1fdeba2bUL, 0x4660fc29UL, 0x710a3e28UL, 0xf41c712dUL, 0xc376b32cUL, 0x9ac8f52eUL, 0xada2372fUL, 0xc08d9a70UL, 0xf7e75871UL, 0xae591e73UL, 0x9933dc72UL, 0x1c259377UL, 0x2b4f5176UL, 0x72f11774UL, 0x459bd575UL, 0x78dc897eUL, 0x4fb64b7fUL, 0x16080d7dUL, 0x2162cf7cUL, 0xa4748079UL, 0x931e4278UL, 0xcaa0047aUL, 0xfdcac67bUL, 0xb02ebc6cUL, 0x87447e6dUL, 0xdefa386fUL, 0xe990fa6eUL, 0x6c86b56bUL, 0x5bec776aUL, 0x02523168UL, 0x3538f369UL, 0x087faf62UL, 0x3f156d63UL, 0x66ab2b61UL, 0x51c1e960UL, 0xd4d7a665UL, 0xe3bd6464UL, 0xba032266UL, 0x8d69e067UL, 0x20cbd748UL, 0x17a11549UL, 0x4e1f534bUL, 0x7975914aUL, 0xfc63de4fUL, 0xcb091c4eUL, 0x92b75a4cUL, 0xa5dd984dUL, 0x989ac446UL, 0xaff00647UL, 0xf64e4045UL, 0xc1248244UL, 0x4432cd41UL, 0x73580f40UL, 0x2ae64942UL, 0x1d8c8b43UL, 0x5068f154UL, 0x67023355UL, 0x3ebc7557UL, 0x09d6b756UL, 0x8cc0f853UL, 0xbbaa3a52UL, 0xe2147c50UL, 0xd57ebe51UL, 0xe839e25aUL, 0xdf53205bUL, 0x86ed6659UL, 0xb187a458UL, 0x3491eb5dUL, 0x03fb295cUL, 0x5a456f5eUL, 0x6d2fad5fUL, 0x801b35e1UL, 0xb771f7e0UL, 0xeecfb1e2UL, 0xd9a573e3UL, 0x5cb33ce6UL, 0x6bd9fee7UL, 0x3267b8e5UL, 0x050d7ae4UL, 0x384a26efUL, 0x0f20e4eeUL, 0x569ea2ecUL, 0x61f460edUL, 0xe4e22fe8UL, 0xd388ede9UL, 0x8a36abebUL, 0xbd5c69eaUL, 0xf0b813fdUL, 0xc7d2d1fcUL, 0x9e6c97feUL, 0xa90655ffUL, 0x2c101afaUL, 0x1b7ad8fbUL, 0x42c49ef9UL, 0x75ae5cf8UL, 0x48e900f3UL, 0x7f83c2f2UL, 0x263d84f0UL, 0x115746f1UL, 0x944109f4UL, 0xa32bcbf5UL, 0xfa958df7UL, 0xcdff4ff6UL, 0x605d78d9UL, 0x5737bad8UL, 0x0e89fcdaUL, 0x39e33edbUL, 0xbcf571deUL, 0x8b9fb3dfUL, 0xd221f5ddUL, 0xe54b37dcUL, 0xd80c6bd7UL, 0xef66a9d6UL, 0xb6d8efd4UL, 0x81b22dd5UL, 0x04a462d0UL, 0x33cea0d1UL, 0x6a70e6d3UL, 0x5d1a24d2UL, 0x10fe5ec5UL, 0x27949cc4UL, 0x7e2adac6UL, 0x494018c7UL, 0xcc5657c2UL, 0xfb3c95c3UL, 0xa282d3c1UL, 0x95e811c0UL, 0xa8af4dcbUL, 0x9fc58fcaUL, 0xc67bc9c8UL, 0xf1110bc9UL, 0x740744ccUL, 0x436d86cdUL, 0x1ad3c0cfUL, 0x2db902ceUL, 0x4096af91UL, 0x77fc6d90UL, 0x2e422b92UL, 0x1928e993UL, 0x9c3ea696UL, 0xab546497UL, 0xf2ea2295UL, 0xc580e094UL, 0xf8c7bc9fUL, 0xcfad7e9eUL, 0x9613389cUL, 0xa179fa9dUL, 0x246fb598UL, 0x13057799UL, 0x4abb319bUL, 0x7dd1f39aUL, 0x3035898dUL, 0x075f4b8cUL, 0x5ee10d8eUL, 0x698bcf8fUL, 0xec9d808aUL, 0xdbf7428bUL, 0x82490489UL, 0xb523c688UL, 0x88649a83UL, 0xbf0e5882UL, 0xe6b01e80UL, 0xd1dadc81UL, 0x54cc9384UL, 0x63a65185UL, 0x3a181787UL, 0x0d72d586UL, 0xa0d0e2a9UL, 0x97ba20a8UL, 0xce0466aaUL, 0xf96ea4abUL, 0x7c78ebaeUL, 0x4b1229afUL, 0x12ac6fadUL, 0x25c6adacUL, 0x1881f1a7UL, 0x2feb33a6UL, 0x765575a4UL, 0x413fb7a5UL, 0xc429f8a0UL, 0xf3433aa1UL, 0xaafd7ca3UL, 0x9d97bea2UL, 0xd073c4b5UL, 0xe71906b4UL, 0xbea740b6UL, 0x89cd82b7UL, 0x0cdbcdb2UL, 0x3bb10fb3UL, 0x620f49b1UL, 0x55658bb0UL, 0x6822d7bbUL, 0x5f4815baUL, 0x06f653b8UL, 0x319c91b9UL, 0xb48adebcUL, 0x83e01cbdUL, 0xda5e5abfUL, 0xed3498beUL }, { 0x00000000UL, 0x6567bcb8UL, 0x8bc809aaUL, 0xeeafb512UL, 0x5797628fUL, 0x32f0de37UL, 0xdc5f6b25UL, 0xb938d79dUL, 0xef28b4c5UL, 0x8a4f087dUL, 0x64e0bd6fUL, 0x018701d7UL, 0xb8bfd64aUL, 0xddd86af2UL, 0x3377dfe0UL, 0x56106358UL, 0x9f571950UL, 0xfa30a5e8UL, 0x149f10faUL, 0x71f8ac42UL, 0xc8c07bdfUL, 0xada7c767UL, 0x43087275UL, 0x266fcecdUL, 0x707fad95UL, 0x1518112dUL, 0xfbb7a43fUL, 0x9ed01887UL, 0x27e8cf1aUL, 0x428f73a2UL, 0xac20c6b0UL, 0xc9477a08UL, 0x3eaf32a0UL, 0x5bc88e18UL, 0xb5673b0aUL, 0xd00087b2UL, 0x6938502fUL, 0x0c5fec97UL, 0xe2f05985UL, 0x8797e53dUL, 0xd1878665UL, 0xb4e03addUL, 0x5a4f8fcfUL, 0x3f283377UL, 0x8610e4eaUL, 0xe3775852UL, 0x0dd8ed40UL, 0x68bf51f8UL, 0xa1f82bf0UL, 0xc49f9748UL, 0x2a30225aUL, 0x4f579ee2UL, 0xf66f497fUL, 0x9308f5c7UL, 0x7da740d5UL, 0x18c0fc6dUL, 0x4ed09f35UL, 0x2bb7238dUL, 0xc518969fUL, 0xa07f2a27UL, 0x1947fdbaUL, 0x7c204102UL, 0x928ff410UL, 0xf7e848a8UL, 0x3d58149bUL, 0x583fa823UL, 0xb6901d31UL, 0xd3f7a189UL, 0x6acf7614UL, 0x0fa8caacUL, 0xe1077fbeUL, 0x8460c306UL, 0xd270a05eUL, 0xb7171ce6UL, 0x59b8a9f4UL, 0x3cdf154cUL, 0x85e7c2d1UL, 0xe0807e69UL, 0x0e2fcb7bUL, 0x6b4877c3UL, 0xa20f0dcbUL, 0xc768b173UL, 0x29c70461UL, 0x4ca0b8d9UL, 0xf5986f44UL, 0x90ffd3fcUL, 0x7e5066eeUL, 0x1b37da56UL, 0x4d27b90eUL, 0x284005b6UL, 0xc6efb0a4UL, 0xa3880c1cUL, 0x1ab0db81UL, 0x7fd76739UL, 0x9178d22bUL, 0xf41f6e93UL, 0x03f7263bUL, 0x66909a83UL, 0x883f2f91UL, 0xed589329UL, 0x546044b4UL, 0x3107f80cUL, 0xdfa84d1eUL, 0xbacff1a6UL, 0xecdf92feUL, 0x89b82e46UL, 0x67179b54UL, 0x027027ecUL, 0xbb48f071UL, 0xde2f4cc9UL, 0x3080f9dbUL, 0x55e74563UL, 0x9ca03f6bUL, 0xf9c783d3UL, 0x176836c1UL, 0x720f8a79UL, 0xcb375de4UL, 0xae50e15cUL, 0x40ff544eUL, 0x2598e8f6UL, 0x73888baeUL, 0x16ef3716UL, 0xf8408204UL, 0x9d273ebcUL, 0x241fe921UL, 0x41785599UL, 0xafd7e08bUL, 0xcab05c33UL, 0x3bb659edUL, 0x5ed1e555UL, 0xb07e5047UL, 0xd519ecffUL, 0x6c213b62UL, 0x094687daUL, 0xe7e932c8UL, 0x828e8e70UL, 0xd49eed28UL, 0xb1f95190UL, 0x5f56e482UL, 0x3a31583aUL, 0x83098fa7UL, 0xe66e331fUL, 0x08c1860dUL, 0x6da63ab5UL, 0xa4e140bdUL, 0xc186fc05UL, 0x2f294917UL, 0x4a4ef5afUL, 0xf3762232UL, 0x96119e8aUL, 0x78be2b98UL, 0x1dd99720UL, 0x4bc9f478UL, 0x2eae48c0UL, 0xc001fdd2UL, 0xa566416aUL, 0x1c5e96f7UL, 0x79392a4fUL, 0x97969f5dUL, 0xf2f123e5UL, 0x05196b4dUL, 0x607ed7f5UL, 0x8ed162e7UL, 0xebb6de5fUL, 0x528e09c2UL, 0x37e9b57aUL, 0xd9460068UL, 0xbc21bcd0UL, 0xea31df88UL, 0x8f566330UL, 0x61f9d622UL, 0x049e6a9aUL, 0xbda6bd07UL, 0xd8c101bfUL, 0x366eb4adUL, 0x53090815UL, 0x9a4e721dUL, 0xff29cea5UL, 0x11867bb7UL, 0x74e1c70fUL, 0xcdd91092UL, 0xa8beac2aUL, 0x46111938UL, 0x2376a580UL, 0x7566c6d8UL, 0x10017a60UL, 0xfeaecf72UL, 0x9bc973caUL, 0x22f1a457UL, 0x479618efUL, 0xa939adfdUL, 0xcc5e1145UL, 0x06ee4d76UL, 0x6389f1ceUL, 0x8d2644dcUL, 0xe841f864UL, 0x51792ff9UL, 0x341e9341UL, 0xdab12653UL, 0xbfd69aebUL, 0xe9c6f9b3UL, 0x8ca1450bUL, 0x620ef019UL, 0x07694ca1UL, 0xbe519b3cUL, 0xdb362784UL, 0x35999296UL, 0x50fe2e2eUL, 0x99b95426UL, 0xfcdee89eUL, 0x12715d8cUL, 0x7716e134UL, 0xce2e36a9UL, 0xab498a11UL, 0x45e63f03UL, 0x208183bbUL, 0x7691e0e3UL, 0x13f65c5bUL, 0xfd59e949UL, 0x983e55f1UL, 0x2106826cUL, 0x44613ed4UL, 0xaace8bc6UL, 0xcfa9377eUL, 0x38417fd6UL, 0x5d26c36eUL, 0xb389767cUL, 0xd6eecac4UL, 0x6fd61d59UL, 0x0ab1a1e1UL, 0xe41e14f3UL, 0x8179a84bUL, 0xd769cb13UL, 0xb20e77abUL, 0x5ca1c2b9UL, 0x39c67e01UL, 0x80fea99cUL, 0xe5991524UL, 0x0b36a036UL, 0x6e511c8eUL, 0xa7166686UL, 0xc271da3eUL, 0x2cde6f2cUL, 0x49b9d394UL, 0xf0810409UL, 0x95e6b8b1UL, 0x7b490da3UL, 0x1e2eb11bUL, 0x483ed243UL, 0x2d596efbUL, 0xc3f6dbe9UL, 0xa6916751UL, 0x1fa9b0ccUL, 0x7ace0c74UL, 0x9461b966UL, 0xf10605deUL #endif } }; digest/src/pmurhash.c0000644000176200001440000002544413346570641014335 0ustar liggesusers/*----------------------------------------------------------------------------- * MurmurHash3 was written by Austin Appleby, and is placed in the public * domain. * * This implementation was written by Shane Day, and is also public domain. * * This is a portable ANSI C implementation of MurmurHash3_x86_32 (Murmur3A) * with support for progressive processing. */ /*----------------------------------------------------------------------------- If you want to understand the MurmurHash algorithm you would be much better off reading the original source. Just point your browser at: http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp What this version provides? 1. Progressive data feeding. Useful when the entire payload to be hashed does not fit in memory or when the data is streamed through the application. Also useful when hashing a number of strings with a common prefix. A partial hash of a prefix string can be generated and reused for each suffix string. 2. Portability. Plain old C so that it should compile on any old compiler. Both CPU endian and access-alignment neutral, but avoiding inefficient code when possible depending on CPU capabilities. 3. Drop in. I personally like nice self contained public domain code, making it easy to pilfer without loads of refactoring to work properly in the existing application code & makefile structure and mucking around with licence files. Just copy PMurHash.h and PMurHash.c and you're ready to go. How does it work? We can only process entire 32 bit chunks of input, except for the very end that may be shorter. So along with the partial hash we need to give back to the caller a carry containing up to 3 bytes that we were unable to process. This carry also needs to record the number of bytes the carry holds. I use the low 2 bits as a count (0..3) and the carry bytes are shifted into the high byte in stream order. To handle endianess I simply use a macro that reads a uint32_t and define that macro to be a direct read on little endian machines, a read and swap on big endian machines, or a byte-by-byte read if the endianess is unknown. -----------------------------------------------------------------------------*/ #include "pmurhash.h" /* renamed for consistency in digest's src/ */ /* I used ugly type names in the header to avoid potential conflicts with * application or system typedefs & defines. Since I'm not including any more * headers below here I can rename these so that the code reads like C99 */ #undef uint32_t #define uint32_t MH_UINT32 #undef uint8_t #define uint8_t MH_UINT8 /* MSVC warnings we choose to ignore */ #if defined(_MSC_VER) #pragma warning(disable: 4127) /* conditional expression is constant */ #endif /*----------------------------------------------------------------------------- * Endianess, misalignment capabilities and util macros * * The following 3 macros are defined in this section. The other macros defined * are only needed to help derive these 3. * * READ_UINT32(x) Read a little endian unsigned 32-bit int * UNALIGNED_SAFE Defined if READ_UINT32 works on non-word boundaries * ROTL32(x,r) Rotate x left by r bits */ /* Convention is to define __BYTE_ORDER == to one of these values */ #if !defined(__BIG_ENDIAN) #define __BIG_ENDIAN 4321 #endif #if !defined(__LITTLE_ENDIAN) #define __LITTLE_ENDIAN 1234 #endif /* I386 */ #if defined(_M_IX86) || defined(__i386__) || defined(__i386) || defined(i386) #define __BYTE_ORDER __LITTLE_ENDIAN #define UNALIGNED_SAFE #endif /* gcc 'may' define __LITTLE_ENDIAN__ or __BIG_ENDIAN__ to 1 (Note the trailing __), * or even _LITTLE_ENDIAN or _BIG_ENDIAN (Note the single _ prefix) */ #if !defined(__BYTE_ORDER) /* #if defined(__LITTLE_ENDIAN__) && __LITTLE_ENDIAN__==1 || defined(_LITTLE_ENDIAN) && _LITTLE_ENDIAN==1 #define __BYTE_ORDER __LITTLE_ENDIAN #elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 || defined(_BIG_ENDIAN) && _BIG_ENDIAN==1 #define __BYTE_ORDER __BIG_ENDIAN #endif */ #if defined(__LITTLE_ENDIAN__) #if __LITTLE_ENDIAN__==1 #define __BYTE_ORDER __LITTLE_ENDIAN #elif defined(_LITTLE_ENDIAN) #if x_LITTLE_ENDIAN != x #if _LITTLE_ENDIAN==1 #define __BYTE_ORDER __LITTLE_ENDIAN #endif #endif #endif #elif defined(__BIG_ENDIAN__) #if __BIG_ENDIAN__==1 #define __BYTE_ORDER __BIG_ENDIAN #endif #elif defined(_BIG_ENDIAN) #if x_BIG_ENDIAN != x #if _BIG_ENDIAN==1 #define __BYTE_ORDER __BIG_ENDIAN #endif #endif #endif #endif /* gcc (usually) defines xEL/EB macros for ARM and MIPS endianess */ #if !defined(__BYTE_ORDER) #if defined(__ARMEL__) || defined(__MIPSEL__) #define __BYTE_ORDER __LITTLE_ENDIAN #endif #if defined(__ARMEB__) || defined(__MIPSEB__) #define __BYTE_ORDER __BIG_ENDIAN #endif #endif /* Now find best way we can to READ_UINT32 */ #if __BYTE_ORDER==__LITTLE_ENDIAN /* CPU endian matches murmurhash algorithm, so read 32-bit word directly */ #define READ_UINT32(ptr) (*((uint32_t*)(ptr))) #elif __BYTE_ORDER==__BIG_ENDIAN /* TODO: Add additional cases below where a compiler provided bswap32 is available */ #if defined(__GNUC__) && (__GNUC__>4 || (__GNUC__==4 && __GNUC_MINOR__>=3)) #define READ_UINT32(ptr) (__builtin_bswap32(*((uint32_t*)(ptr)))) #else /* Without a known fast bswap32 we're just as well off doing this */ #define READ_UINT32(ptr) (ptr[0]|ptr[1]<<8|ptr[2]<<16|ptr[3]<<24) #define UNALIGNED_SAFE #endif #else /* Unknown endianess so last resort is to read individual bytes */ #define READ_UINT32(ptr) (ptr[0]|ptr[1]<<8|ptr[2]<<16|ptr[3]<<24) /* Since we're not doing word-reads we can skip the messing about with realignment */ #define UNALIGNED_SAFE #endif /* Find best way to ROTL32 */ #if defined(_MSC_VER) #include /* Microsoft put _rotl declaration in here */ #define ROTL32(x,r) _rotl(x,r) #else /* gcc recognises this code and generates a rotate instruction for CPUs with one */ #define ROTL32(x,r) (((uint32_t)x << r) | ((uint32_t)x >> (32 - r))) #endif /*----------------------------------------------------------------------------- * Core murmurhash algorithm macros */ #define C1 (0xcc9e2d51) #define C2 (0x1b873593) /* This is the main processing body of the algorithm. It operates * on each full 32-bits of input. */ #define DOBLOCK(h1, k1) do{ \ k1 *= C1; \ k1 = ROTL32(k1,15); \ k1 *= C2; \ \ h1 ^= k1; \ h1 = ROTL32(h1,13); \ h1 = h1*5+0xe6546b64; \ }while(0) /* Append unaligned bytes to carry, forcing hash churn if we have 4 bytes */ /* cnt=bytes to process, h1=name of h1 var, c=carry, n=bytes in c, ptr/len=payload */ #define DOBYTES(cnt, h1, c, n, ptr, len) do{ \ int _i = cnt; \ while(_i--) { \ c = c>>8 | (uint32_t)*ptr++<<24; \ n++; len--; \ if(n==4) { \ DOBLOCK(h1, c); \ n = 0; \ } \ } }while(0) /*---------------------------------------------------------------------------*/ /* Main hashing function. Initialise carry to 0 and h1 to 0 or an initial seed * if wanted. Both ph1 and pcarry are required arguments. */ void PMurHash32_Process(uint32_t *ph1, uint32_t *pcarry, const void *key, int len) { uint32_t h1 = *ph1; uint32_t c = *pcarry; const uint8_t *ptr = (uint8_t*)key; const uint8_t *end; /* Extract carry count from low 2 bits of c value */ int n = c & 3; #if defined(UNALIGNED_SAFE) /* This CPU handles unaligned word access */ /* Consume any carry bytes */ int i = (4-n) & 3; if(i && i <= len) { DOBYTES(i, h1, c, n, ptr, len); /* #nocov */ } /* Process 32-bit chunks */ end = ptr + len/4*4; for( ; ptr < end ; ptr+=4) { uint32_t k1 = READ_UINT32(ptr); DOBLOCK(h1, k1); } #else /*UNALIGNED_SAFE*/ /* This CPU does not handle unaligned word access */ /* Consume enough so that the next data byte is word aligned */ int i = -(long)ptr & 3; if(i && i <= len) { DOBYTES(i, h1, c, n, ptr, len); } /* We're now aligned. Process in aligned blocks. Specialise for each possible carry count */ end = ptr + len/4*4; switch(n) { /* how many bytes in c */ case 0: /* c=[----] w=[3210] b=[3210]=w c'=[----] */ for( ; ptr < end ; ptr+=4) { uint32_t k1 = READ_UINT32(ptr); DOBLOCK(h1, k1); } break; case 1: /* c=[0---] w=[4321] b=[3210]=c>>24|w<<8 c'=[4---] */ for( ; ptr < end ; ptr+=4) { uint32_t k1 = c>>24; c = READ_UINT32(ptr); k1 |= c<<8; DOBLOCK(h1, k1); } break; case 2: /* c=[10--] w=[5432] b=[3210]=c>>16|w<<16 c'=[54--] */ for( ; ptr < end ; ptr+=4) { uint32_t k1 = c>>16; c = READ_UINT32(ptr); k1 |= c<<16; DOBLOCK(h1, k1); } break; case 3: /* c=[210-] w=[6543] b=[3210]=c>>8|w<<24 c'=[654-] */ for( ; ptr < end ; ptr+=4) { uint32_t k1 = c>>8; c = READ_UINT32(ptr); k1 |= c<<24; DOBLOCK(h1, k1); } } #endif /*UNALIGNED_SAFE*/ /* Advance over whole 32-bit chunks, possibly leaving 1..3 bytes */ len -= len/4*4; /* Append any remaining bytes into carry */ DOBYTES(len, h1, c, n, ptr, len); /* Copy out new running hash and carry */ *ph1 = h1; *pcarry = (c & ~0xff) | n; } /*---------------------------------------------------------------------------*/ /* Finalize a hash. To match the original Murmur3A the total_length must be provided */ uint32_t PMurHash32_Result(uint32_t h, uint32_t carry, uint32_t total_length) { uint32_t k1; int n = carry & 3; if(n) { k1 = carry >> (4-n)*8; k1 *= C1; k1 = ROTL32(k1,15); k1 *= C2; h ^= k1; } h ^= total_length; /* fmix */ h ^= h >> 16; h *= 0x85ebca6b; h ^= h >> 13; h *= 0xc2b2ae35; h ^= h >> 16; return h; } /*---------------------------------------------------------------------------*/ /* Murmur3A compatable all-at-once */ uint32_t PMurHash32(uint32_t seed, const void *key, int len) { uint32_t h1=seed, carry=0; PMurHash32_Process(&h1, &carry, key, len); return PMurHash32_Result(h1, carry, len); } /*---------------------------------------------------------------------------*/ /* Provide an API suitable for smhasher */ void PMurHash32_test(const void *key, int len, uint32_t seed, void *out) /* #nocov start */ { uint32_t h1=seed, carry=0; const uint8_t *ptr = (uint8_t*)key; const uint8_t *end = ptr + len; #if 0 /* Exercise the progressive processing */ while(ptr < end) { //const uint8_t *mid = ptr + rand()%(end-ptr)+1; const uint8_t *mid = ptr + (rand()&0xF); mid = mid #include #include #include "blake3.h" #include "blake3_impl.h" INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], uint8_t flags) { memcpy(self->cv, key, BLAKE3_KEY_LEN); self->chunk_counter = 0; memset(self->buf, 0, BLAKE3_BLOCK_LEN); self->buf_len = 0; self->blocks_compressed = 0; self->flags = flags; } INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], uint64_t chunk_counter) { memcpy(self->cv, key, BLAKE3_KEY_LEN); self->chunk_counter = chunk_counter; self->blocks_compressed = 0; memset(self->buf, 0, BLAKE3_BLOCK_LEN); self->buf_len = 0; } INLINE size_t chunk_state_len(const blake3_chunk_state *self) { return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + ((size_t)self->buf_len); } INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, const uint8_t *input, size_t input_len) { size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); if (take > input_len) { take = input_len; } uint8_t *dest = self->buf + ((size_t)self->buf_len); memcpy(dest, input, take); self->buf_len += (uint8_t)take; return take; } INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { if (self->blocks_compressed == 0) { return CHUNK_START; } else { return 0; } } typedef struct { uint32_t input_cv[8]; uint64_t counter; uint8_t block[BLAKE3_BLOCK_LEN]; uint8_t block_len; uint8_t flags; } output_t; INLINE output_t make_output(const uint32_t input_cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags) { output_t ret; memcpy(ret.input_cv, input_cv, 32); memcpy(ret.block, block, BLAKE3_BLOCK_LEN); ret.block_len = block_len; ret.counter = counter; ret.flags = flags; return ret; } // Chaining values within a given chunk (specifically the compress_in_place // interface) are represented as words. This avoids unnecessary bytes<->words // conversion overhead in the portable implementation. However, the hash_many // interface handles both user input and parent node blocks, so it accepts // bytes. For that reason, chaining values in the CV stack are represented as // bytes. INLINE void output_chaining_value(const output_t *self, uint8_t cv[32]) { uint32_t cv_words[8]; memcpy(cv_words, self->input_cv, 32); blake3_compress_in_place(cv_words, self->block, self->block_len, self->counter, self->flags); store_cv_words(cv, cv_words); } INLINE void output_root_bytes(const output_t *self, uint64_t seek, uint8_t *out, size_t out_len) { uint64_t output_block_counter = seek / 64; size_t offset_within_block = seek % 64; uint8_t wide_buf[64]; while (out_len > 0) { blake3_compress_xof(self->input_cv, self->block, self->block_len, output_block_counter, self->flags | ROOT, wide_buf); size_t available_bytes = 64 - offset_within_block; size_t memcpy_len; if (out_len > available_bytes) { memcpy_len = available_bytes; } else { memcpy_len = out_len; } memcpy(out, wide_buf + offset_within_block, memcpy_len); out += memcpy_len; out_len -= memcpy_len; output_block_counter += 1; offset_within_block = 0; } } INLINE void chunk_state_update(blake3_chunk_state *self, const uint8_t *input, size_t input_len) { if (self->buf_len > 0) { size_t take = chunk_state_fill_buf(self, input, input_len); input += take; input_len -= take; if (input_len > 0) { blake3_compress_in_place( self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, self->flags | chunk_state_maybe_start_flag(self)); self->blocks_compressed += 1; self->buf_len = 0; memset(self->buf, 0, BLAKE3_BLOCK_LEN); } } while (input_len > BLAKE3_BLOCK_LEN) { blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, self->chunk_counter, self->flags | chunk_state_maybe_start_flag(self)); self->blocks_compressed += 1; input += BLAKE3_BLOCK_LEN; input_len -= BLAKE3_BLOCK_LEN; } size_t take = chunk_state_fill_buf(self, input, input_len); input += take; input_len -= take; } INLINE output_t chunk_state_output(const blake3_chunk_state *self) { uint8_t block_flags = self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, block_flags); } INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], const uint32_t key[8], uint8_t flags) { return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); } // Given some input larger than one chunk, return the number of bytes that // should go in the left subtree. This is the largest power-of-2 number of // chunks that leaves at least 1 byte for the right subtree. INLINE size_t left_len(size_t content_len) { // Subtract 1 to reserve at least one byte for the right side. content_len // should always be greater than BLAKE3_CHUNK_LEN. size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; } // Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time // on a single thread. Write out the chunk chaining values and return the // number of chunks hashed. These chunks are never the root and never empty; // those cases use a different codepath. INLINE size_t compress_chunks_parallel(const uint8_t *input, size_t input_len, const uint32_t key[8], uint64_t chunk_counter, uint8_t flags, uint8_t *out) { #if defined(BLAKE3_TESTING) assert(0 < input_len); assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); #endif const uint8_t *chunks_array[MAX_SIMD_DEGREE]; size_t input_position = 0; size_t chunks_array_len = 0; while (input_len - input_position >= BLAKE3_CHUNK_LEN) { chunks_array[chunks_array_len] = &input[input_position]; input_position += BLAKE3_CHUNK_LEN; chunks_array_len += 1; } blake3_hash_many(chunks_array, chunks_array_len, BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, true, flags, CHUNK_START, CHUNK_END, out); // Hash the remaining partial chunk, if there is one. Note that the empty // chunk (meaning the empty message) is a different codepath. if (input_len > input_position) { uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; blake3_chunk_state chunk_state; chunk_state_init(&chunk_state, key, flags); chunk_state.chunk_counter = counter; chunk_state_update(&chunk_state, &input[input_position], input_len - input_position); output_t output = chunk_state_output(&chunk_state); output_chaining_value(&output, &out[chunks_array_len * BLAKE3_OUT_LEN]); return chunks_array_len + 1; } else { return chunks_array_len; } } // Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time // on a single thread. Write out the parent chaining values and return the // number of parents hashed. (If there's an odd input chaining value left over, // return it as an additional output.) These parents are never the root and // never empty; those cases use a different codepath. INLINE size_t compress_parents_parallel(const uint8_t *child_chaining_values, size_t num_chaining_values, const uint32_t key[8], uint8_t flags, uint8_t *out) { #if defined(BLAKE3_TESTING) assert(2 <= num_chaining_values); assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); #endif const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; size_t parents_array_len = 0; while (num_chaining_values - (2 * parents_array_len) >= 2) { parents_array[parents_array_len] = &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; parents_array_len += 1; } blake3_hash_many(parents_array, parents_array_len, 1, key, 0, // Parents always use counter 0. false, flags | PARENT, 0, // Parents have no start flags. 0, // Parents have no end flags. out); // If there's an odd child left over, it becomes an output. if (num_chaining_values > 2 * parents_array_len) { memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], BLAKE3_OUT_LEN); return parents_array_len + 1; } else { return parents_array_len; } } // The wide helper function returns (writes out) an array of chaining values // and returns the length of that array. The number of chaining values returned // is the dyanmically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, // if the input is shorter than that many chunks. The reason for maintaining a // wide array of chaining values going back up the tree, is to allow the // implementation to hash as many parents in parallel as possible. // // As a special case when the SIMD degree is 1, this function will still return // at least 2 outputs. This guarantees that this function doesn't perform the // root compression. (If it did, it would use the wrong flags, and also we // wouldn't be able to implement exendable ouput.) Note that this function is // not used when the whole input is only 1 chunk long; that's a different // codepath. // // Why not just have the caller split the input on the first update(), instead // of implementing this special rule? Because we don't want to limit SIMD or // multi-threading parallelism for that update(). static size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len, const uint32_t key[8], uint64_t chunk_counter, uint8_t flags, uint8_t *out) { // Note that the single chunk case does *not* bump the SIMD degree up to 2 // when it is 1. If this implementation adds multi-threading in the future, // this gives us the option of multi-threading even the 2-chunk case, which // can help performance on smaller platforms. if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { return compress_chunks_parallel(input, input_len, key, chunk_counter, flags, out); } // With more than simd_degree chunks, we need to recurse. Start by dividing // the input into left and right subtrees. (Note that this is only optimal // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree // of 3 or something, we'll need a more complicated strategy.) size_t left_input_len = left_len(input_len); size_t right_input_len = input_len - left_input_len; const uint8_t *right_input = &input[left_input_len]; uint64_t right_chunk_counter = chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to // account for the special case of returning 2 outputs when the SIMD degree // is 1. uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t degree = blake3_simd_degree(); if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { // The special case: We always use a degree of at least two, to make // sure there are two outputs. Except, as noted above, at the chunk // level, where we allow degree=1. (Note that the 1-chunk-input case is // a different codepath.) degree = 2; } uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; // Recurse! If this implementation adds multi-threading support in the // future, this is where it will go. size_t left_n = blake3_compress_subtree_wide(input, left_input_len, key, chunk_counter, flags, cv_array); size_t right_n = blake3_compress_subtree_wide( right_input, right_input_len, key, right_chunk_counter, flags, right_cvs); // The special case again. If simd_degree=1, then we'll have left_n=1 and // right_n=1. Rather than compressing them into a single output, return // them directly, to make sure we always have at least two outputs. if (left_n == 1) { memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); return 2; } // Otherwise, do one layer of parent node compression. size_t num_chaining_values = left_n + right_n; return compress_parents_parallel(cv_array, num_chaining_values, key, flags, out); } // Hash a subtree with compress_subtree_wide(), and then condense the resulting // list of chaining values down to a single parent node. Don't compress that // last parent node, however. Instead, return its message bytes (the // concatenated chaining values of its children). This is necessary when the // first call to update() supplies a complete subtree, because the topmost // parent node of that subtree could end up being the root. It's also necessary // for extended output in the general case. // // As with compress_subtree_wide(), this function is not used on inputs of 1 // chunk or less. That's a different codepath. INLINE void compress_subtree_to_parent_node( const uint8_t *input, size_t input_len, const uint32_t key[8], uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { #if defined(BLAKE3_TESTING) assert(input_len > BLAKE3_CHUNK_LEN); #endif uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; size_t num_cvs = blake3_compress_subtree_wide(input, input_len, key, chunk_counter, flags, cv_array); // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, // compress_subtree_wide() returns more than 2 chaining values. Condense // them into 2 by forming parent nodes repeatedly. uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; while (num_cvs > 2) { num_cvs = compress_parents_parallel(cv_array, num_cvs, key, flags, out_array); memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); } memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); } INLINE void hasher_init_base(blake3_hasher *self, const uint32_t key[8], uint8_t flags) { memcpy(self->key, key, BLAKE3_KEY_LEN); chunk_state_init(&self->chunk, key, flags); self->cv_stack_len = 0; } void blake3_hasher_init(blake3_hasher *self) { hasher_init_base(self, IV, 0); } void blake3_hasher_init_keyed(blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]) { uint32_t key_words[8]; load_key_words(key, key_words); hasher_init_base(self, key_words, KEYED_HASH); } void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context) { blake3_hasher context_hasher; hasher_init_base(&context_hasher, IV, DERIVE_KEY_CONTEXT); blake3_hasher_update(&context_hasher, context, strlen(context)); uint8_t context_key[BLAKE3_KEY_LEN]; blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); uint32_t context_key_words[8]; load_key_words(context_key, context_key_words); hasher_init_base(self, context_key_words, DERIVE_KEY_MATERIAL); } // As described in hasher_push_cv() below, we do "lazy merging", delaying // merges until right before the next CV is about to be added. This is // different from the reference implementation. Another difference is that we // aren't always merging 1 chunk at a time. Instead, each CV might represent // any power-of-two number of chunks, as long as the smaller-above-larger stack // order is maintained. Instead of the "count the trailing 0-bits" algorithm // described in the spec, we use a "count the total number of 1-bits" variant // that doesn't require us to retain the subtree size of the CV on top of the // stack. The principle is the same: each CV that should remain in the stack is // represented by a 1-bit in the total number of chunks (or bytes) so far. INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { size_t post_merge_stack_len = (size_t)popcnt(total_len); while (self->cv_stack_len > post_merge_stack_len) { uint8_t *parent_node = &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; output_t output = parent_output(parent_node, self->key, self->chunk.flags); output_chaining_value(&output, parent_node); self->cv_stack_len -= 1; } } // In reference_impl.rs, we merge the new CV with existing CVs from the stack // before pushing it. We can do that because we know more input is coming, so // we know none of the merges are root. // // This setting is different. We want to feed as much input as possible to // compress_subtree_wide(), without setting aside anything for the chunk_state. // If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once // as a single subtree, if at all possible. // // This leads to two problems: // 1) This 64 KiB input might be the only call that ever gets made to update. // In this case, the root node of the 64 KiB subtree would be the root node // of the whole tree, and it would need to be ROOT finalized. We can't // compress it until we know. // 2) This 64 KiB input might complete a larger tree, whose root node is // similarly going to be the the root of the whole tree. For example, maybe // we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the // node at the root of the 256 KiB subtree until we know how to finalize it. // // The second problem is solved with "lazy merging". That is, when we're about // to add a CV to the stack, we don't merge it with anything first, as the // reference impl does. Instead we do merges using the *previous* CV that was // added, which is sitting on top of the stack, and we put the new CV // (unmerged) on top of the stack afterwards. This guarantees that we never // merge the root node until finalize(). // // Solving the first problem requires an additional tool, // compress_subtree_to_parent_node(). That function always returns the top // *two* chaining values of the subtree it's compressing. We then do lazy // merging with each of them separately, so that the second CV will always // remain unmerged. (That also helps us support extendable output when we're // hashing an input all-at-once.) INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], uint64_t chunk_counter) { hasher_merge_cv_stack(self, chunk_counter); memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, BLAKE3_OUT_LEN); self->cv_stack_len += 1; } void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len) { // Explicitly checking for zero avoids causing UB by passing a null pointer // to memcpy. This comes up in practice with things like: // std::vector v; // blake3_hasher_update(&hasher, v.data(), v.size()); if (input_len == 0) { return; } const uint8_t *input_bytes = (const uint8_t *)input; // If we have some partial chunk bytes in the internal chunk_state, we need // to finish that chunk first. if (chunk_state_len(&self->chunk) > 0) { size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); if (take > input_len) { take = input_len; } chunk_state_update(&self->chunk, input_bytes, take); input_bytes += take; input_len -= take; // If we've filled the current chunk and there's more coming, finalize this // chunk and proceed. In this case we know it's not the root. if (input_len > 0) { output_t output = chunk_state_output(&self->chunk); uint8_t chunk_cv[32]; output_chaining_value(&output, chunk_cv); hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); } else { return; } } // Now the chunk_state is clear, and we have more input. If there's more than // a single chunk (so, definitely not the root chunk), hash the largest whole // subtree we can, with the full benefits of SIMD (and maybe in the future, // multi-threading) parallelism. Two restrictions: // - The subtree has to be a power-of-2 number of chunks. Only subtrees along // the right edge can be incomplete, and we don't know where the right edge // is going to be until we get to finalize(). // - The subtree must evenly divide the total number of chunks up until this // point (if total is not 0). If the current incomplete subtree is only // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have // to complete the current subtree first. // Because we might need to break up the input to form powers of 2, or to // evenly divide what we already have, this part runs in a loop. while (input_len > BLAKE3_CHUNK_LEN) { size_t subtree_len = round_down_to_power_of_2(input_len); uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; // Shrink the subtree_len until it evenly divides the count so far. We know // that subtree_len itself is a power of 2, so we can use a bitmasking // trick instead of an actual remainder operation. (Note that if the caller // consistently passes power-of-2 inputs of the same size, as is hopefully // typical, this loop condition will always fail, and subtree_len will // always be the full length of the input.) // // An aside: We don't have to shrink subtree_len quite this much. For // example, if count_so_far is 1, we could pass 2 chunks to // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still // get the right answer in the end, and we might get to use 2-way SIMD // parallelism. The problem with this optimization, is that it gets us // stuck always hashing 2 chunks. The total number of chunks will remain // odd, and we'll never graduate to higher degrees of parallelism. See // https://github.com/BLAKE3-team/BLAKE3/issues/69. while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { subtree_len /= 2; } // The shrunken subtree_len might now be 1 chunk long. If so, hash that one // chunk by itself. Otherwise, compress the subtree into a pair of CVs. uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; if (subtree_len <= BLAKE3_CHUNK_LEN) { blake3_chunk_state chunk_state; chunk_state_init(&chunk_state, self->key, self->chunk.flags); chunk_state.chunk_counter = self->chunk.chunk_counter; chunk_state_update(&chunk_state, input_bytes, subtree_len); output_t output = chunk_state_output(&chunk_state); uint8_t cv[BLAKE3_OUT_LEN]; output_chaining_value(&output, cv); hasher_push_cv(self, cv, chunk_state.chunk_counter); } else { // This is the high-performance happy path, though getting here depends // on the caller giving us a long enough input. uint8_t cv_pair[2 * BLAKE3_OUT_LEN]; compress_subtree_to_parent_node(input_bytes, subtree_len, self->key, self->chunk.chunk_counter, self->chunk.flags, cv_pair); hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], self->chunk.chunk_counter + (subtree_chunks / 2)); } self->chunk.chunk_counter += subtree_chunks; input_bytes += subtree_len; input_len -= subtree_len; } // If there's any remaining input less than a full chunk, add it to the chunk // state. In that case, also do a final merge loop to make sure the subtree // stack doesn't contain any unmerged pairs. The remaining input means we // know these merges are non-root. This merge loop isn't strictly necessary // here, because hasher_push_chunk_cv already does its own merge loop, but it // simplifies blake3_hasher_finalize below. if (input_len > 0) { chunk_state_update(&self->chunk, input_bytes, input_len); hasher_merge_cv_stack(self, self->chunk.chunk_counter); } } void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len) { blake3_hasher_finalize_seek(self, 0, out, out_len); } void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len) { // Explicitly checking for zero avoids causing UB by passing a null pointer // to memcpy. This comes up in practice with things like: // std::vector v; // blake3_hasher_finalize(&hasher, v.data(), v.size()); if (out_len == 0) { return; } // If the subtree stack is empty, then the current chunk is the root. if (self->cv_stack_len == 0) { output_t output = chunk_state_output(&self->chunk); output_root_bytes(&output, seek, out, out_len); return; } // If there are any bytes in the chunk state, finalize that chunk and do a // roll-up merge between that chunk hash and every subtree in the stack. In // this case, the extra merge loop at the end of blake3_hasher_update // guarantees that none of the subtrees in the stack need to be merged with // each other first. Otherwise, if there are no bytes in the chunk state, // then the top of the stack is a chunk hash, and we start the merge from // that. output_t output; size_t cvs_remaining; if (chunk_state_len(&self->chunk) > 0) { cvs_remaining = self->cv_stack_len; output = chunk_state_output(&self->chunk); } else { // There are always at least 2 CVs in the stack in this case. cvs_remaining = self->cv_stack_len - 2; output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, self->chunk.flags); } while (cvs_remaining > 0) { cvs_remaining -= 1; uint8_t parent_block[BLAKE3_BLOCK_LEN]; memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); output_chaining_value(&output, &parent_block[32]); output = parent_output(parent_block, self->key, self->chunk.flags); } output_root_bytes(&output, seek, out, out_len); } digest/src/crc32c/0000755000176200001440000000000014660651077013413 5ustar liggesusersdigest/src/crc32c/crc32c_read_le.h0000644000176200001440000000376514423472174016325 0ustar liggesusers// Copyright 2017 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef CRC32C_CRC32C_READ_LE_H_ #define CRC32C_CRC32C_READ_LE_H_ #include #include #include "crc32c/crc32c_config.h" namespace crc32c { // Reads a little-endian 32-bit integer from a 32-bit-aligned buffer. inline uint32_t ReadUint32LE(const uint8_t* buffer) { #if BYTE_ORDER_BIG_ENDIAN return ((static_cast(static_cast(buffer[0]))) | (static_cast(static_cast(buffer[1])) << 8) | (static_cast(static_cast(buffer[2])) << 16) | (static_cast(static_cast(buffer[3])) << 24)); #else // !BYTE_ORDER_BIG_ENDIAN uint32_t result; // This should be optimized to a single instruction. std::memcpy(&result, buffer, sizeof(result)); return result; #endif // BYTE_ORDER_BIG_ENDIAN } // Reads a little-endian 64-bit integer from a 64-bit-aligned buffer. inline uint64_t ReadUint64LE(const uint8_t* buffer) { #if BYTE_ORDER_BIG_ENDIAN return ((static_cast(static_cast(buffer[0]))) | (static_cast(static_cast(buffer[1])) << 8) | (static_cast(static_cast(buffer[2])) << 16) | (static_cast(static_cast(buffer[3])) << 24) | (static_cast(static_cast(buffer[4])) << 32) | (static_cast(static_cast(buffer[5])) << 40) | (static_cast(static_cast(buffer[6])) << 48) | (static_cast(static_cast(buffer[7])) << 56)); #else // !BYTE_ORDER_BIG_ENDIAN uint64_t result; // This should be optimized to a single instruction. std::memcpy(&result, buffer, sizeof(result)); return result; #endif // BYTE_ORDER_BIG_ENDIAN } } // namespace crc32c #endif // CRC32C_CRC32C_READ_LE_H_ digest/src/crc32c/crc32c_config.h0000644000176200001440000000314114446716221016162 0ustar liggesusers// Copyright 2017 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef CRC32C_CRC32C_CONFIG_H_ #define CRC32C_CRC32C_CONFIG_H_ // Define to 1 if building for a big-endian platform and conditions met ... else 0 for little endian #if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define BYTE_ORDER_BIG_ENDIAN 1 #else #define BYTE_ORDER_BIG_ENDIAN 0 #endif // Define to 1 if the compiler has the __builtin_prefetch intrinsic. // edd 2023-06-26 Set to 0 for maximum portability #define HAVE_BUILTIN_PREFETCH 0 // Define to 1 if targeting X86 and the compiler has the _mm_prefetch intrinsic. // edd 2023-06-26 Set to 0 for maximum portability #define HAVE_MM_PREFETCH 0 // Define to 1 if targeting X86 and the compiler has the _mm_crc32_u{8,32,64} // intrinsics. #define HAVE_SSE42 0 // Define to 1 if targeting ARM and the compiler has the __crc32c{b,h,w,d} and // the vmull_p64 intrinsics. #define HAVE_ARM64_CRC32C 0 // Define to 1 if the system libraries have the getauxval function in the // header. Should be true on Linux and Android API level 20+. #define HAVE_STRONG_GETAUXVAL 0 // Define to 1 if the compiler supports defining getauxval as a weak symbol. // Should be true for any compiler that supports __attribute__((weak)). #define HAVE_WEAK_GETAUXVAL 1 // Define to 1 if CRC32C tests have been built with Google Logging. #define CRC32C_TESTS_BUILT_WITH_GLOG 0 #endif // CRC32C_CRC32C_CONFIG_H_ digest/src/crc32c/crc32c_round_up.h0000644000176200001440000000203514423472174016552 0ustar liggesusers// Copyright 2017 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef CRC32C_CRC32C_ROUND_UP_H_ #define CRC32C_CRC32C_ROUND_UP_H_ #include #include namespace crc32c { // Returns the smallest number >= the given number that is evenly divided by N. // // N must be a power of two. template constexpr inline uintptr_t RoundUp(uintptr_t pointer) { static_assert((N & (N - 1)) == 0, "N must be a power of two"); return (pointer + (N - 1)) & ~(N - 1); } // Returns the smallest address >= the given address that is aligned to N bytes. // // N must be a power of two. template constexpr inline const uint8_t* RoundUp(const uint8_t* pointer) { static_assert((N & (N - 1)) == 0, "N must be a power of two"); return reinterpret_cast( RoundUp(reinterpret_cast(pointer))); } } // namespace crc32c #endif // CRC32C_CRC32C_ROUND_UP_H_ digest/src/crc32c/crc32c_internal.h0000644000176200001440000000133714423472174016537 0ustar liggesusers// Copyright 2017 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef CRC32C_CRC32C_INTERNAL_H_ #define CRC32C_CRC32C_INTERNAL_H_ // Internal functions that may change between releases. #include #include namespace crc32c { // Un-accelerated implementation that works on all CPUs. uint32_t ExtendPortable(uint32_t crc, const uint8_t* data, size_t count); // CRCs are pre- and post- conditioned by xoring with all ones. static constexpr const uint32_t kCRC32Xor = static_cast(0xffffffffU); } // namespace crc32c #endif // CRC32C_CRC32C_INTERNAL_H_ digest/src/crc32c/crc32c_prefetch.h0000644000176200001440000000255514423472174016526 0ustar liggesusers// Copyright 2017 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef CRC32C_CRC32C_PREFETCH_H_ #define CRC32C_CRC32C_PREFETCH_H_ #include #include #include "crc32c/crc32c_config.h" #if HAVE_MM_PREFETCH #if defined(_MSC_VER) #include #else // !defined(_MSC_VER) #include #endif // defined(_MSC_VER) #endif // HAVE_MM_PREFETCH namespace crc32c { // Ask the hardware to prefetch the data at the given address into the L1 cache. inline void RequestPrefetch(const uint8_t* address) { #if HAVE_BUILTIN_PREFETCH // Clang and GCC implement the __builtin_prefetch non-standard extension, // which maps to the best instruction on the target architecture. __builtin_prefetch(reinterpret_cast(address), 0 /* Read only. */, 0 /* No temporal locality. */); #elif HAVE_MM_PREFETCH // Visual Studio doesn't implement __builtin_prefetch, but exposes the // PREFETCHNTA instruction via the _mm_prefetch intrinsic. _mm_prefetch(reinterpret_cast(address), _MM_HINT_NTA); #else // No prefetch support. Silence compiler warnings. (void)address; #endif // HAVE_BUILTIN_PREFETCH } } // namespace crc32c #endif // CRC32C_CRC32C_ROUND_UP_H_ digest/src/raes.c0000644000176200001440000000473414657503316013440 0ustar liggesusers /* This is a simple R interface to Christophe Devine's AES implementation */ #include "aes.h" #include #include static aes_context ctx; void encrypt(unsigned char *key, int *len, unsigned char *text) { /* #nocov start */ aes_set_key(&ctx, key, 8*(*len)); aes_encrypt(&ctx, text, text); } void decrypt(unsigned char *key, int *len, unsigned char *code) { aes_set_key(&ctx, key, 8*(*len)); aes_decrypt(&ctx, code, code); } static void AESFinalizer(SEXP ptr) { void *ctx = R_ExternalPtrAddr(ptr); if (!ctx) return; R_Free(ctx); R_ClearExternalPtr(ptr); } /* #nocov end */ SEXP AESinit(SEXP key) { int nbits = 8*length(key); int status; SEXP result; aes_context *ctx; if (TYPEOF(key) != RAWSXP) error("key must be a raw vector"); /* #nocov */ if (nbits != 128 && nbits != 192 && nbits != 256) error("AES only supports 16, 24 and 32 byte keys"); /* #nocov */ ctx = (aes_context*)R_Calloc(sizeof(*ctx), char); status = aes_set_key(ctx, (uint8 *) RAW(key), nbits); if (status) error("AES initialization failed"); /* #nocov */ result = R_MakeExternalPtr(ctx, install("AES_context"), R_NilValue); PROTECT(result); R_RegisterCFinalizerEx(result, AESFinalizer, FALSE); UNPROTECT(1); return result; } SEXP AESencryptECB(SEXP context, SEXP text) { aes_context *ctx = R_ExternalPtrAddr(context); int len = length(text); unsigned char *block; if (!ctx) error("AES context not initialized"); /* #nocov */ if (TYPEOF(text) != RAWSXP) error("Text must be a raw vector"); /* #nocov */ if (len % 16) error("Text length must be a multiple of 16 bytes"); /* #nocov */ if (MAYBE_REFERENCED(text)) text = duplicate(text); block = RAW(text); while (len > 0) { aes_encrypt(ctx, block, block); block += 16; len -= 16; } return(text); } SEXP AESdecryptECB(SEXP context, SEXP ciphertext) { aes_context *ctx = R_ExternalPtrAddr(context); int len = length(ciphertext); unsigned char *block; if (!ctx) error("AES context not initialized"); /* #nocov */ if (TYPEOF(ciphertext) != RAWSXP) error("Ciphertext must be a raw vector"); /* #nocov */ if (len % 16) error("Ciphertext length must be a multiple of 16 bytes"); /* #nocov */ if (MAYBE_REFERENCED(ciphertext)) ciphertext = duplicate(ciphertext); block = RAW(ciphertext); while (len > 0) { aes_decrypt(ctx, block, block); block += 16; len -= 16; } return(ciphertext); } digest/src/zconf.h0000644000176200001440000002217512247504516013625 0ustar liggesusers/* zconf.h -- configuration of the zlib compression library * Copyright (C) 1995-2004 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #ifndef ZCONF_H #define ZCONF_H /* * If you *really* need a unique prefix for all types and library functions, * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. */ #ifdef Z_PREFIX # define deflateInit_ z_deflateInit_ # define deflate z_deflate # define deflateEnd z_deflateEnd # define inflateInit_ z_inflateInit_ # define inflate z_inflate # define inflateEnd z_inflateEnd # define deflateInit2_ z_deflateInit2_ # define deflateSetDictionary z_deflateSetDictionary # define deflateCopy z_deflateCopy # define deflateReset z_deflateReset # define deflateParams z_deflateParams # define deflateBound z_deflateBound # define deflatePrime z_deflatePrime # define inflateInit2_ z_inflateInit2_ # define inflateSetDictionary z_inflateSetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateCopy z_inflateCopy # define inflateReset z_inflateReset # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd # define compress z_compress # define compress2 z_compress2 # define compressBound z_compressBound # define uncompress z_uncompress # define adler32 z_adler32 # define crc32 z_crc32 # define get_crc_table z_get_crc_table # define zError z_zError # define Byte z_Byte # define uInt z_uInt # define uLong z_uLong # define Bytef z_Bytef # define charf z_charf # define intf z_intf # define uIntf z_uIntf # define uLongf z_uLongf # define voidpf z_voidpf # define voidp z_voidp #endif #if defined(__MSDOS__) && !defined(MSDOS) # define MSDOS #endif #if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) # define OS2 #endif #if defined(_WINDOWS) && !defined(WINDOWS) # define WINDOWS #endif #if (defined(_WIN32) || defined(__WIN32__)) && !defined(WIN32) # define WIN32 #endif #if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) # if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) # ifndef SYS16BIT # define SYS16BIT # endif # endif #endif /* * Compile with -DMAXSEG_64K if the alloc function cannot allocate more * than 64k bytes at a time (needed on systems with 16-bit int). */ #ifdef SYS16BIT # define MAXSEG_64K #endif #ifdef MSDOS # define UNALIGNED_OK #endif #ifdef __STDC_VERSION__ # ifndef STDC # define STDC # endif # if __STDC_VERSION__ >= 199901L # ifndef STDC99 # define STDC99 # endif # endif #endif #if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) # define STDC #endif #if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) # define STDC #endif #if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) # define STDC #endif #if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) # define STDC #endif #if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ # define STDC #endif #ifndef STDC # ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ # define const /* note: need a more gentle solution here */ # endif #endif /* Some Mac compilers merge all .h files incorrectly: */ #if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) # define NO_DUMMY_DECL #endif /* Maximum value for memLevel in deflateInit2 */ #ifndef MAX_MEM_LEVEL # ifdef MAXSEG_64K # define MAX_MEM_LEVEL 8 # else # define MAX_MEM_LEVEL 9 # endif #endif /* Maximum value for windowBits in deflateInit2 and inflateInit2. * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files * created by gzip. (Files created by minigzip can still be extracted by * gzip.) */ #ifndef MAX_WBITS # define MAX_WBITS 15 /* 32K LZ77 window */ #endif /* The memory requirements for deflate are (in bytes): (1 << (windowBits+2)) + (1 << (memLevel+9)) that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) plus a few kilobytes for small objects. For example, if you want to reduce the default memory requirements from 256K to 128K, compile with make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits that is, 32K for windowBits=15 (default value) plus a few kilobytes for small objects. */ /* Type declarations */ #ifndef OF /* function prototypes */ # ifdef STDC # define OF(args) args # else # define OF(args) () # endif #endif /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, * just define FAR to be empty. */ #ifdef SYS16BIT # if defined(M_I86SM) || defined(M_I86MM) /* MSC small or medium model */ # define SMALL_MEDIUM # ifdef _MSC_VER # define FAR _far # else # define FAR far # endif # endif # if (defined(__SMALL__) || defined(__MEDIUM__)) /* Turbo C small or medium model */ # define SMALL_MEDIUM # ifdef __BORLANDC__ # define FAR _far # else # define FAR far # endif # endif #endif #if defined(WINDOWS) || defined(WIN32) /* If building or using zlib as a DLL, define ZLIB_DLL. * This is not mandatory, but it offers a little performance increase. */ # ifdef ZLIB_DLL # if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) # ifdef ZLIB_INTERNAL # define ZEXTERN extern __declspec(dllexport) # else # define ZEXTERN extern __declspec(dllimport) # endif # endif # endif /* ZLIB_DLL */ /* If building or using zlib with the WINAPI/WINAPIV calling convention, * define ZLIB_WINAPI. * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. */ # ifdef ZLIB_WINAPI # ifdef FAR # undef FAR # endif # include /* No need for _export, use ZLIB.DEF instead. */ /* For complete Windows compatibility, use WINAPI, not __stdcall. */ # define ZEXPORT WINAPI # ifdef WIN32 # define ZEXPORTVA WINAPIV # else # define ZEXPORTVA FAR CDECL # endif # endif #endif #if defined (__BEOS__) # ifdef ZLIB_DLL # ifdef ZLIB_INTERNAL # define ZEXPORT __declspec(dllexport) # define ZEXPORTVA __declspec(dllexport) # else # define ZEXPORT __declspec(dllimport) # define ZEXPORTVA __declspec(dllimport) # endif # endif #endif #ifndef ZEXTERN # define ZEXTERN extern #endif #ifndef ZEXPORT # define ZEXPORT #endif #ifndef ZEXPORTVA # define ZEXPORTVA #endif #ifndef FAR # define FAR #endif #if !defined(__MACTYPES__) typedef unsigned char Byte; /* 8 bits */ #endif typedef unsigned int uInt; /* 16 bits or more */ typedef unsigned long uLong; /* 32 bits or more */ #ifdef SMALL_MEDIUM /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ # define Bytef Byte FAR #else typedef Byte FAR Bytef; #endif typedef char FAR charf; typedef int FAR intf; typedef uInt FAR uIntf; typedef uLong FAR uLongf; #ifdef STDC typedef void const *voidpc; typedef void FAR *voidpf; typedef void *voidp; #else typedef Byte const *voidpc; typedef Byte FAR *voidpf; typedef Byte *voidp; #endif #if 0 /* HAVE_UNISTD_H -- this line is updated by ./configure */ # include /* for off_t */ # include /* for SEEK_* and off_t */ # ifdef VMS # include /* for off_t */ # endif # define z_off_t off_t #endif #ifndef SEEK_SET # define SEEK_SET 0 /* Seek from beginning of file. */ # define SEEK_CUR 1 /* Seek from current position. */ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ #endif #ifndef z_off_t # define z_off_t long #endif #if defined(__OS400__) # define NO_vsnprintf #endif #if defined(__MVS__) # define NO_vsnprintf # ifdef FAR # undef FAR # endif #endif /* MVS linker does not support external names larger than 8 bytes */ #if defined(__MVS__) # pragma map(deflateInit_,"DEIN") # pragma map(deflateInit2_,"DEIN2") # pragma map(deflateEnd,"DEEND") # pragma map(deflateBound,"DEBND") # pragma map(inflateInit_,"ININ") # pragma map(inflateInit2_,"ININ2") # pragma map(inflateEnd,"INEND") # pragma map(inflateSync,"INSY") # pragma map(inflateSetDictionary,"INSEDI") # pragma map(compressBound,"CMBND") # pragma map(inflate_table,"INTABL") # pragma map(inflate_fast,"INFA") # pragma map(inflate_copyright,"INCOPY") #endif #endif /* ZCONF_H */ digest/src/spooky_serialize.cpp0000644000176200001440000000727414621373425016440 0ustar liggesusers// Copyright (C) 2024 Dirk Eddelbuettel // Copyright (C) 2019 Kendon Bell // Copyright (C) 2014 Gabe Becker // // This file is part of digest and is modified from the original source in // the R package fastdigest under the Artistic License 2.0. // // digest is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 2 of the License, or // (at your option) any later version. // // digest is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with digest. If not, see . #ifdef HAVE_CONFIG_H #include #endif #define R_NO_REMAP #define USE_RINTERNALS #include #include #include #include #include "SpookyV2.h" static void OutCharSpooky(R_outpstream_t stream, int c) { // #nocov start SpookyHash *spooky = (SpookyHash *)stream->data; spooky->Update(&c, 1); } // #nocov end static void OutBytesSpooky(R_outpstream_t stream, void *buf, int length) { SpookyHash *spooky = (SpookyHash *)stream->data; uint8 skipped = 0; uint8 to_skip = 0; spooky->GetSkipCounter(&skipped); spooky->GetToSkip(&to_skip); if (skipped < to_skip){ if ((skipped + length) > to_skip){ Rf_error("Serialization header has an unexpected length. Please file an issue at https://github.com/eddelbuettel/digest/issues."); // #nocov } spooky->UpdateSkipCounter(length); } else { spooky->Update(buf, length); } } static void InitSpookyPStream(R_outpstream_t stream, SpookyHash *spooky, R_pstream_format_t type, int version, SEXP (*phook)(SEXP, SEXP), SEXP pdata) { R_InitOutPStream(stream, (R_pstream_data_t) spooky, type, version, OutCharSpooky, OutBytesSpooky, phook, pdata); } //From serialize.c in R sources /* ought to quote the argument, but it should only be an ENVSXP or STRSXP */ static SEXP CallHook(SEXP x, SEXP fun) { // #nocov start SEXP val, call; PROTECT(call = Rf_lcons(fun, Rf_lcons(x, R_NilValue))); val = Rf_eval(call, R_GlobalEnv); UNPROTECT(1); return val; } // #nocov end extern "C" SEXP spookydigest_impl(SEXP s, SEXP to_skip_r, SEXP seed1_r, SEXP seed2_r, SEXP version_r, SEXP fun) { SpookyHash spooky; double seed1_d = Rf_asReal(seed1_r); double seed2_d = Rf_asReal(seed2_r); uint64_t seed1 = static_cast(seed1_d); uint64_t seed2 = static_cast(seed2_d); uint8_t to_skip = static_cast(Rf_asInteger(to_skip_r)); spooky.Init(seed1, seed2, to_skip); R_outpstream_st spooky_stream; R_pstream_format_t type = R_pstream_binary_format; SEXP (*hook)(SEXP, SEXP); int version = Rf_asInteger(version_r); hook = fun != R_NilValue ? CallHook : NULL; InitSpookyPStream(&spooky_stream, &spooky, type, version, hook, fun); R_Serialize(s, &spooky_stream); //There are two because they are 64 bit ints and the hash is 128 bits!!! uint64 h1, h2; spooky.Final(&h1, &h2); SEXP ans; PROTECT(ans = Rf_allocVector(RAWSXP, 16)); unsigned char *tmp; tmp = (unsigned char *) &h1; for (int i = 0; i < 8; i++) RAW(ans)[i] = tmp[i]; tmp = (unsigned char *) &h2; for (int j = 0; j < 8; j++) RAW(ans)[j + 8] = tmp[j]; UNPROTECT(1); return ans; } digest/src/blake3_dispatch.c0000755000176200001440000001717213743560000015515 0ustar liggesusers/* blake3 license: * The C code is copyright Samuel Neves and Jack O'Connor, 2019-2020. * This work is released into the public domain with CC0 1.0. * Alternatively, it is licensed under the Apache License 2.0. * For full license see * https://raw.githubusercontent.com/BLAKE3-team/BLAKE3/master/LICENSE */ #include #include #include #include "blake3_impl.h" // Commenting out this block as intrinic headers are not need or used // and the 'non-MSC and non-GNU' leads to an error on Solaris // #if defined(IS_X86) // #if defined(_MSC_VER) // #include // #elif defined(__GNUC__) // #include // #else // #error "Unimplemented!" // #endif // #endif // This function is disabled in this R package for now // #if defined(IS_X86) // static uint64_t xgetbv() { // #if defined(_MSC_VER) // return _xgetbv(0); // #else // uint32_t eax = 0, edx = 0; // __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); // return ((uint64_t)edx << 32) | eax; // #endif // } // // static void cpuid(uint32_t out[4], uint32_t id) { // #if defined(_MSC_VER) // __cpuid((int *)out, id); // #elif defined(__i386__) || defined(_M_IX86) // __asm__ __volatile__("movl %%ebx, %1\n" // "cpuid\n" // "xchgl %1, %%ebx\n" // : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) // : "a"(id)); // #else // __asm__ __volatile__("cpuid\n" // : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) // : "a"(id)); // #endif // } // // static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { // #if defined(_MSC_VER) // __cpuidex((int *)out, id, sid); // #elif defined(__i386__) || defined(_M_IX86) // __asm__ __volatile__("movl %%ebx, %1\n" // "cpuid\n" // "xchgl %1, %%ebx\n" // : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) // : "a"(id), "c"(sid)); // #else // __asm__ __volatile__("cpuid\n" // : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) // : "a"(id), "c"(sid)); // #endif // } // // #endif enum cpu_feature { SSE2 = 1 << 0, SSSE3 = 1 << 1, SSE41 = 1 << 2, AVX = 1 << 3, AVX2 = 1 << 4, AVX512F = 1 << 5, AVX512VL = 1 << 6, /* ... */ UNDEFINED = 1 << 30 }; // This function is disabled in this R package for now // #if !defined(BLAKE3_TESTING) // static /* Allow the variable to be controlled manually for testing */ // #endif // enum cpu_feature g_cpu_features = UNDEFINED; // // #if !defined(BLAKE3_TESTING) // static // #endif // enum cpu_feature // get_cpu_features() { // // if (g_cpu_features != UNDEFINED) { // return g_cpu_features; // } else { // #if defined(IS_X86) // uint32_t regs[4] = {0}; // uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; // (void)edx; // enum cpu_feature features = 0; // cpuid(regs, 0); // const int max_id = *eax; // cpuid(regs, 1); // #if defined(__amd64__) || defined(_M_X64) // features |= SSE2; // #else // if (*edx & (1UL << 26)) // features |= SSE2; // #endif // if (*ecx & (1UL << 0)) // features |= SSSE3; // if (*ecx & (1UL << 19)) // features |= SSE41; // // if (*ecx & (1UL << 27)) { // OSXSAVE // const uint64_t mask = xgetbv(); // if ((mask & 6) == 6) { // SSE and AVX states // if (*ecx & (1UL << 28)) // features |= AVX; // if (max_id >= 7) { // cpuidex(regs, 7, 0); // if (*ebx & (1UL << 5)) // features |= AVX2; // if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm // if (*ebx & (1UL << 31)) // features |= AVX512VL; // if (*ebx & (1UL << 16)) // features |= AVX512F; // } // } // } // } // g_cpu_features = features; // return features; // #else // /* How to detect NEON? */ // return 0; // #endif // } // } void blake3_compress_in_place(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags) { #if defined(IS_X86) // const enum cpu_feature features = get_cpu_features(); // never true for this R package #if !defined(BLAKE3_NO_AVX512) if (features & AVX512VL) { blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); return; } #endif #if !defined(BLAKE3_NO_SSE41) if (features & SSE41) { blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); return; } #endif #endif blake3_compress_in_place_portable(cv, block, block_len, counter, flags); } void blake3_compress_xof(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]) { #if defined(IS_X86) // const enum cpu_feature features = get_cpu_features(); // never true for this R package #if !defined(BLAKE3_NO_AVX512) if (features & AVX512VL) { blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); return; } #endif #if !defined(BLAKE3_NO_SSE41) if (features & SSE41) { blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); return; } #endif #endif blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); } void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out) { #if defined(IS_X86) // const enum cpu_feature features = get_cpu_features(); // never true for this R package #if !defined(BLAKE3_NO_AVX512) if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); return; } #endif #if !defined(BLAKE3_NO_AVX2) if (features & AVX2) { blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); return; } #endif #if !defined(BLAKE3_NO_SSE41) if (features & SSE41) { blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); return; } #endif #endif #if defined(BLAKE3_USE_NEON) blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); return; #endif blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out); } // The dynamically detected SIMD degree of the current platform. size_t blake3_simd_degree(void) { #if defined(IS_X86) // const enum cpu_feature features = get_cpu_features(); // never true for this R package #if !defined(BLAKE3_NO_AVX512) if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { return 16; } #endif #if !defined(BLAKE3_NO_AVX2) if (features & AVX2) { return 8; } #endif #if !defined(BLAKE3_NO_SSE41) if (features & SSE41) { return 4; } #endif #endif #if defined(BLAKE3_USE_NEON) return 4; #endif return 1; } digest/src/init.c0000644000176200001440000000265114636007227013442 0ustar liggesusers/* -*- mode: c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; -*- init -- registering the c hash digest functions Copyright (C) 2014 - 2019 Wush Wu and Dirk Eddelbuettel This file is part of digest. digest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. digest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with digest. If not, see . */ #include #include "xxhash.h" #include "pmurhash.h" void R_init_digest(DllInfo *info) { R_RegisterCCallable("digest", "PMurHash32", (DL_FUNC) &PMurHash32); /* tools::package_native_routine_registration_skeleton() reports empty set */ R_registerRoutines(info, NULL, /* slot for .C */ NULL, /* slot for .Call */ NULL, /* slot for .Fortran */ NULL); /* slot for .External */ R_useDynamicSymbols(info, TRUE); /* controls visibility */ } digest/src/sha256.c0000644000176200001440000002560113216563247013511 0ustar liggesusers/* * FIPS-180-2 compliant SHA-256 implementation * * Copyright (C) 2001-2003 Christophe Devine * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include "sha256.h" #define GET_UINT32(n,b,i) \ { \ (n) = ( (uint32) (b)[(i) ] << 24 ) \ | ( (uint32) (b)[(i) + 1] << 16 ) \ | ( (uint32) (b)[(i) + 2] << 8 ) \ | ( (uint32) (b)[(i) + 3] ); \ } #define PUT_UINT32(n,b,i) \ { \ (b)[(i) ] = (uint8) ( (n) >> 24 ); \ (b)[(i) + 1] = (uint8) ( (n) >> 16 ); \ (b)[(i) + 2] = (uint8) ( (n) >> 8 ); \ (b)[(i) + 3] = (uint8) ( (n) ); \ } void sha256_starts( sha256_context *ctx ) { ctx->total[0] = 0; ctx->total[1] = 0; ctx->state[0] = 0x6A09E667; ctx->state[1] = 0xBB67AE85; ctx->state[2] = 0x3C6EF372; ctx->state[3] = 0xA54FF53A; ctx->state[4] = 0x510E527F; ctx->state[5] = 0x9B05688C; ctx->state[6] = 0x1F83D9AB; ctx->state[7] = 0x5BE0CD19; } void sha256_process( sha256_context *ctx, uint8 data[64] ) { uint32 temp1, temp2, W[64]; uint32 A, B, C, D, E, F, G, H; GET_UINT32( W[0], data, 0 ); GET_UINT32( W[1], data, 4 ); GET_UINT32( W[2], data, 8 ); GET_UINT32( W[3], data, 12 ); GET_UINT32( W[4], data, 16 ); GET_UINT32( W[5], data, 20 ); GET_UINT32( W[6], data, 24 ); GET_UINT32( W[7], data, 28 ); GET_UINT32( W[8], data, 32 ); GET_UINT32( W[9], data, 36 ); GET_UINT32( W[10], data, 40 ); GET_UINT32( W[11], data, 44 ); GET_UINT32( W[12], data, 48 ); GET_UINT32( W[13], data, 52 ); GET_UINT32( W[14], data, 56 ); GET_UINT32( W[15], data, 60 ); #define SHR(x,n) ((x & 0xFFFFFFFF) >> n) #define ROTR(x,n) (SHR(x,n) | (x << (32 - n))) #define S0(x) (ROTR(x, 7) ^ ROTR(x,18) ^ SHR(x, 3)) #define S1(x) (ROTR(x,17) ^ ROTR(x,19) ^ SHR(x,10)) #define S2(x) (ROTR(x, 2) ^ ROTR(x,13) ^ ROTR(x,22)) #define S3(x) (ROTR(x, 6) ^ ROTR(x,11) ^ ROTR(x,25)) #define F0(x,y,z) ((x & y) | (z & (x | y))) #define F1(x,y,z) (z ^ (x & (y ^ z))) #define R(t) \ ( \ W[t] = S1(W[t - 2]) + W[t - 7] + \ S0(W[t - 15]) + W[t - 16] \ ) #define P(a,b,c,d,e,f,g,h,x,K) \ { \ temp1 = h + S3(e) + F1(e,f,g) + K + x; \ temp2 = S2(a) + F0(a,b,c); \ d += temp1; h = temp1 + temp2; \ } A = ctx->state[0]; B = ctx->state[1]; C = ctx->state[2]; D = ctx->state[3]; E = ctx->state[4]; F = ctx->state[5]; G = ctx->state[6]; H = ctx->state[7]; P( A, B, C, D, E, F, G, H, W[ 0], 0x428A2F98 ); P( H, A, B, C, D, E, F, G, W[ 1], 0x71374491 ); P( G, H, A, B, C, D, E, F, W[ 2], 0xB5C0FBCF ); P( F, G, H, A, B, C, D, E, W[ 3], 0xE9B5DBA5 ); P( E, F, G, H, A, B, C, D, W[ 4], 0x3956C25B ); P( D, E, F, G, H, A, B, C, W[ 5], 0x59F111F1 ); P( C, D, E, F, G, H, A, B, W[ 6], 0x923F82A4 ); P( B, C, D, E, F, G, H, A, W[ 7], 0xAB1C5ED5 ); P( A, B, C, D, E, F, G, H, W[ 8], 0xD807AA98 ); P( H, A, B, C, D, E, F, G, W[ 9], 0x12835B01 ); P( G, H, A, B, C, D, E, F, W[10], 0x243185BE ); P( F, G, H, A, B, C, D, E, W[11], 0x550C7DC3 ); P( E, F, G, H, A, B, C, D, W[12], 0x72BE5D74 ); P( D, E, F, G, H, A, B, C, W[13], 0x80DEB1FE ); P( C, D, E, F, G, H, A, B, W[14], 0x9BDC06A7 ); P( B, C, D, E, F, G, H, A, W[15], 0xC19BF174 ); P( A, B, C, D, E, F, G, H, R(16), 0xE49B69C1 ); P( H, A, B, C, D, E, F, G, R(17), 0xEFBE4786 ); P( G, H, A, B, C, D, E, F, R(18), 0x0FC19DC6 ); P( F, G, H, A, B, C, D, E, R(19), 0x240CA1CC ); P( E, F, G, H, A, B, C, D, R(20), 0x2DE92C6F ); P( D, E, F, G, H, A, B, C, R(21), 0x4A7484AA ); P( C, D, E, F, G, H, A, B, R(22), 0x5CB0A9DC ); P( B, C, D, E, F, G, H, A, R(23), 0x76F988DA ); P( A, B, C, D, E, F, G, H, R(24), 0x983E5152 ); P( H, A, B, C, D, E, F, G, R(25), 0xA831C66D ); P( G, H, A, B, C, D, E, F, R(26), 0xB00327C8 ); P( F, G, H, A, B, C, D, E, R(27), 0xBF597FC7 ); P( E, F, G, H, A, B, C, D, R(28), 0xC6E00BF3 ); P( D, E, F, G, H, A, B, C, R(29), 0xD5A79147 ); P( C, D, E, F, G, H, A, B, R(30), 0x06CA6351 ); P( B, C, D, E, F, G, H, A, R(31), 0x14292967 ); P( A, B, C, D, E, F, G, H, R(32), 0x27B70A85 ); P( H, A, B, C, D, E, F, G, R(33), 0x2E1B2138 ); P( G, H, A, B, C, D, E, F, R(34), 0x4D2C6DFC ); P( F, G, H, A, B, C, D, E, R(35), 0x53380D13 ); P( E, F, G, H, A, B, C, D, R(36), 0x650A7354 ); P( D, E, F, G, H, A, B, C, R(37), 0x766A0ABB ); P( C, D, E, F, G, H, A, B, R(38), 0x81C2C92E ); P( B, C, D, E, F, G, H, A, R(39), 0x92722C85 ); P( A, B, C, D, E, F, G, H, R(40), 0xA2BFE8A1 ); P( H, A, B, C, D, E, F, G, R(41), 0xA81A664B ); P( G, H, A, B, C, D, E, F, R(42), 0xC24B8B70 ); P( F, G, H, A, B, C, D, E, R(43), 0xC76C51A3 ); P( E, F, G, H, A, B, C, D, R(44), 0xD192E819 ); P( D, E, F, G, H, A, B, C, R(45), 0xD6990624 ); P( C, D, E, F, G, H, A, B, R(46), 0xF40E3585 ); P( B, C, D, E, F, G, H, A, R(47), 0x106AA070 ); P( A, B, C, D, E, F, G, H, R(48), 0x19A4C116 ); P( H, A, B, C, D, E, F, G, R(49), 0x1E376C08 ); P( G, H, A, B, C, D, E, F, R(50), 0x2748774C ); P( F, G, H, A, B, C, D, E, R(51), 0x34B0BCB5 ); P( E, F, G, H, A, B, C, D, R(52), 0x391C0CB3 ); P( D, E, F, G, H, A, B, C, R(53), 0x4ED8AA4A ); P( C, D, E, F, G, H, A, B, R(54), 0x5B9CCA4F ); P( B, C, D, E, F, G, H, A, R(55), 0x682E6FF3 ); P( A, B, C, D, E, F, G, H, R(56), 0x748F82EE ); P( H, A, B, C, D, E, F, G, R(57), 0x78A5636F ); P( G, H, A, B, C, D, E, F, R(58), 0x84C87814 ); P( F, G, H, A, B, C, D, E, R(59), 0x8CC70208 ); P( E, F, G, H, A, B, C, D, R(60), 0x90BEFFFA ); P( D, E, F, G, H, A, B, C, R(61), 0xA4506CEB ); P( C, D, E, F, G, H, A, B, R(62), 0xBEF9A3F7 ); P( B, C, D, E, F, G, H, A, R(63), 0xC67178F2 ); ctx->state[0] += A; ctx->state[1] += B; ctx->state[2] += C; ctx->state[3] += D; ctx->state[4] += E; ctx->state[5] += F; ctx->state[6] += G; ctx->state[7] += H; } void sha256_update( sha256_context *ctx, uint8 *input, uint32 length ) { uint32 left, fill; if( ! length ) return; left = ctx->total[0] & 0x3F; fill = 64 - left; ctx->total[0] += length; ctx->total[0] &= 0xFFFFFFFF; if( ctx->total[0] < length ) ctx->total[1]++; /* #nocov */ if( left && length >= fill ) { memcpy( (void *) (ctx->buffer + left), (void *) input, fill ); sha256_process( ctx, ctx->buffer ); length -= fill; input += fill; left = 0; } while( length >= 64 ) { sha256_process( ctx, input ); length -= 64; input += 64; } if( length ) { memcpy( (void *) (ctx->buffer + left), (void *) input, length ); } } static uint8 sha256_padding[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; void sha256_finish( sha256_context *ctx, uint8 digest[32] ) { uint32 last, padn; uint32 high, low; uint8 msglen[8]; high = ( ctx->total[0] >> 29 ) | ( ctx->total[1] << 3 ); low = ( ctx->total[0] << 3 ); PUT_UINT32( high, msglen, 0 ); PUT_UINT32( low, msglen, 4 ); last = ctx->total[0] & 0x3F; padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); sha256_update( ctx, sha256_padding, padn ); sha256_update( ctx, msglen, 8 ); PUT_UINT32( ctx->state[0], digest, 0 ); PUT_UINT32( ctx->state[1], digest, 4 ); PUT_UINT32( ctx->state[2], digest, 8 ); PUT_UINT32( ctx->state[3], digest, 12 ); PUT_UINT32( ctx->state[4], digest, 16 ); PUT_UINT32( ctx->state[5], digest, 20 ); PUT_UINT32( ctx->state[6], digest, 24 ); PUT_UINT32( ctx->state[7], digest, 28 ); } #ifdef TEST #include #include /* * those are the standard FIPS-180-2 test vectors */ static char *msg[] = { "abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", NULL }; static char *val[] = { "ba7816bf8f01cfea414140de5dae2223" \ "b00361a396177a9cb410ff61f20015ad", "248d6a61d20638b8e5c026930c3e6039" \ "a33ce45964ff2167f6ecedd419db06c1", "cdc76e5c9914fb9281a1c7e284d73e67" \ "f1809a48a497200e046d39ccc7112cd0" }; int main( int argc, char *argv[] ) { FILE *f; int i, j; char output[65]; sha256_context ctx; unsigned char buf[1000]; unsigned char sha256sum[32]; if( argc < 2 ) { printf( "\n SHA-256 Validation Tests:\n\n" ); for( i = 0; i < 3; i++ ) { printf( " Test %d ", i + 1 ); sha256_starts( &ctx ); if( i < 2 ) { sha256_update( &ctx, (uint8 *) msg[i], strlen( msg[i] ) ); } else { memset( buf, 'a', 1000 ); for( j = 0; j < 1000; j++ ) { sha256_update( &ctx, (uint8 *) buf, 1000 ); } } sha256_finish( &ctx, sha256sum ); for( j = 0; j < 32; j++ ) { sprintf( output + j * 2, "%02x", sha256sum[j] ); } if( memcmp( output, val[i], 64 ) ) { printf( "failed!\n" ); return( 1 ); } printf( "passed.\n" ); } printf( "\n" ); } else { if( ! ( f = fopen( argv[1], "rb" ) ) ) { perror( "fopen" ); return( 1 ); } sha256_starts( &ctx ); while( ( i = fread( buf, 1, sizeof( buf ), f ) ) > 0 ) { sha256_update( &ctx, buf, i ); } sha256_finish( &ctx, sha256sum ); for( j = 0; j < 32; j++ ) { printf( "%02x", sha256sum[j] ); } printf( " %s\n", argv[1] ); } return( 0 ); } #endif digest/src/aes.c0000644000176200001440000007520413216563247013255 0ustar liggesusers/* * FIPS-197 compliant AES implementation * * Copyright (C) 2001-2004 Christophe Devine * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include "aes.h" /* uncomment the following line to run the test suite */ /* #define TEST */ /* uncomment the following line to use pre-computed tables */ /* otherwise the tables will be generated at the first run */ #define FIXED_TABLES #ifndef FIXED_TABLES /* forward S-box & tables */ uint32 FSb[256]; uint32 FT0[256]; uint32 FT1[256]; uint32 FT2[256]; uint32 FT3[256]; /* reverse S-box & tables */ uint32 RSb[256]; uint32 RT0[256]; uint32 RT1[256]; uint32 RT2[256]; uint32 RT3[256]; /* round constants */ uint32 RCON[10]; /* tables generation flag */ int do_init = 1; /* tables generation routine */ #define ROTR8(x) ( ( ( x << 24 ) & 0xFFFFFFFF ) | \ ( ( x & 0xFFFFFFFF ) >> 8 ) ) #define XTIME(x) ( ( x << 1 ) ^ ( ( x & 0x80 ) ? 0x1B : 0x00 ) ) #define MUL(x,y) ( ( x && y ) ? pow[(log[x] + log[y]) % 255] : 0 ) void aes_gen_tables( void ) { int i; uint8 x, y; uint8 pow[256]; uint8 log[256]; /* compute pow and log tables over GF(2^8) */ for( i = 0, x = 1; i < 256; i++, x ^= XTIME( x ) ) { pow[i] = x; log[x] = i; } /* calculate the round constants */ for( i = 0, x = 1; i < 10; i++, x = XTIME( x ) ) { RCON[i] = (uint32) x << 24; } /* generate the forward and reverse S-boxes */ FSb[0x00] = 0x63; RSb[0x63] = 0x00; for( i = 1; i < 256; i++ ) { x = pow[255 - log[i]]; y = x; y = ( y << 1 ) | ( y >> 7 ); x ^= y; y = ( y << 1 ) | ( y >> 7 ); x ^= y; y = ( y << 1 ) | ( y >> 7 ); x ^= y; y = ( y << 1 ) | ( y >> 7 ); x ^= y ^ 0x63; FSb[i] = x; RSb[x] = i; } /* generate the forward and reverse tables */ for( i = 0; i < 256; i++ ) { x = (unsigned char) FSb[i]; y = XTIME( x ); FT0[i] = (uint32) ( x ^ y ) ^ ( (uint32) x << 8 ) ^ ( (uint32) x << 16 ) ^ ( (uint32) y << 24 ); FT0[i] &= 0xFFFFFFFF; FT1[i] = ROTR8( FT0[i] ); FT2[i] = ROTR8( FT1[i] ); FT3[i] = ROTR8( FT2[i] ); y = (unsigned char) RSb[i]; RT0[i] = ( (uint32) MUL( 0x0B, y ) ) ^ ( (uint32) MUL( 0x0D, y ) << 8 ) ^ ( (uint32) MUL( 0x09, y ) << 16 ) ^ ( (uint32) MUL( 0x0E, y ) << 24 ); RT0[i] &= 0xFFFFFFFF; RT1[i] = ROTR8( RT0[i] ); RT2[i] = ROTR8( RT1[i] ); RT3[i] = ROTR8( RT2[i] ); } } #else /* forward S-box */ static const uint32 FSb[256] = { 0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76, 0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0, 0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15, 0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75, 0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84, 0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF, 0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8, 0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2, 0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73, 0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB, 0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79, 0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08, 0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A, 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16 }; /* forward tables */ #define FT \ \ V(C6,63,63,A5), V(F8,7C,7C,84), V(EE,77,77,99), V(F6,7B,7B,8D), \ V(FF,F2,F2,0D), V(D6,6B,6B,BD), V(DE,6F,6F,B1), V(91,C5,C5,54), \ V(60,30,30,50), V(02,01,01,03), V(CE,67,67,A9), V(56,2B,2B,7D), \ V(E7,FE,FE,19), V(B5,D7,D7,62), V(4D,AB,AB,E6), V(EC,76,76,9A), \ V(8F,CA,CA,45), V(1F,82,82,9D), V(89,C9,C9,40), V(FA,7D,7D,87), \ V(EF,FA,FA,15), V(B2,59,59,EB), V(8E,47,47,C9), V(FB,F0,F0,0B), \ V(41,AD,AD,EC), V(B3,D4,D4,67), V(5F,A2,A2,FD), V(45,AF,AF,EA), \ V(23,9C,9C,BF), V(53,A4,A4,F7), V(E4,72,72,96), V(9B,C0,C0,5B), \ V(75,B7,B7,C2), V(E1,FD,FD,1C), V(3D,93,93,AE), V(4C,26,26,6A), \ V(6C,36,36,5A), V(7E,3F,3F,41), V(F5,F7,F7,02), V(83,CC,CC,4F), \ V(68,34,34,5C), V(51,A5,A5,F4), V(D1,E5,E5,34), V(F9,F1,F1,08), \ V(E2,71,71,93), V(AB,D8,D8,73), V(62,31,31,53), V(2A,15,15,3F), \ V(08,04,04,0C), V(95,C7,C7,52), V(46,23,23,65), V(9D,C3,C3,5E), \ V(30,18,18,28), V(37,96,96,A1), V(0A,05,05,0F), V(2F,9A,9A,B5), \ V(0E,07,07,09), V(24,12,12,36), V(1B,80,80,9B), V(DF,E2,E2,3D), \ V(CD,EB,EB,26), V(4E,27,27,69), V(7F,B2,B2,CD), V(EA,75,75,9F), \ V(12,09,09,1B), V(1D,83,83,9E), V(58,2C,2C,74), V(34,1A,1A,2E), \ V(36,1B,1B,2D), V(DC,6E,6E,B2), V(B4,5A,5A,EE), V(5B,A0,A0,FB), \ V(A4,52,52,F6), V(76,3B,3B,4D), V(B7,D6,D6,61), V(7D,B3,B3,CE), \ V(52,29,29,7B), V(DD,E3,E3,3E), V(5E,2F,2F,71), V(13,84,84,97), \ V(A6,53,53,F5), V(B9,D1,D1,68), V(00,00,00,00), V(C1,ED,ED,2C), \ V(40,20,20,60), V(E3,FC,FC,1F), V(79,B1,B1,C8), V(B6,5B,5B,ED), \ V(D4,6A,6A,BE), V(8D,CB,CB,46), V(67,BE,BE,D9), V(72,39,39,4B), \ V(94,4A,4A,DE), V(98,4C,4C,D4), V(B0,58,58,E8), V(85,CF,CF,4A), \ V(BB,D0,D0,6B), V(C5,EF,EF,2A), V(4F,AA,AA,E5), V(ED,FB,FB,16), \ V(86,43,43,C5), V(9A,4D,4D,D7), V(66,33,33,55), V(11,85,85,94), \ V(8A,45,45,CF), V(E9,F9,F9,10), V(04,02,02,06), V(FE,7F,7F,81), \ V(A0,50,50,F0), V(78,3C,3C,44), V(25,9F,9F,BA), V(4B,A8,A8,E3), \ V(A2,51,51,F3), V(5D,A3,A3,FE), V(80,40,40,C0), V(05,8F,8F,8A), \ V(3F,92,92,AD), V(21,9D,9D,BC), V(70,38,38,48), V(F1,F5,F5,04), \ V(63,BC,BC,DF), V(77,B6,B6,C1), V(AF,DA,DA,75), V(42,21,21,63), \ V(20,10,10,30), V(E5,FF,FF,1A), V(FD,F3,F3,0E), V(BF,D2,D2,6D), \ V(81,CD,CD,4C), V(18,0C,0C,14), V(26,13,13,35), V(C3,EC,EC,2F), \ V(BE,5F,5F,E1), V(35,97,97,A2), V(88,44,44,CC), V(2E,17,17,39), \ V(93,C4,C4,57), V(55,A7,A7,F2), V(FC,7E,7E,82), V(7A,3D,3D,47), \ V(C8,64,64,AC), V(BA,5D,5D,E7), V(32,19,19,2B), V(E6,73,73,95), \ V(C0,60,60,A0), V(19,81,81,98), V(9E,4F,4F,D1), V(A3,DC,DC,7F), \ V(44,22,22,66), V(54,2A,2A,7E), V(3B,90,90,AB), V(0B,88,88,83), \ V(8C,46,46,CA), V(C7,EE,EE,29), V(6B,B8,B8,D3), V(28,14,14,3C), \ V(A7,DE,DE,79), V(BC,5E,5E,E2), V(16,0B,0B,1D), V(AD,DB,DB,76), \ V(DB,E0,E0,3B), V(64,32,32,56), V(74,3A,3A,4E), V(14,0A,0A,1E), \ V(92,49,49,DB), V(0C,06,06,0A), V(48,24,24,6C), V(B8,5C,5C,E4), \ V(9F,C2,C2,5D), V(BD,D3,D3,6E), V(43,AC,AC,EF), V(C4,62,62,A6), \ V(39,91,91,A8), V(31,95,95,A4), V(D3,E4,E4,37), V(F2,79,79,8B), \ V(D5,E7,E7,32), V(8B,C8,C8,43), V(6E,37,37,59), V(DA,6D,6D,B7), \ V(01,8D,8D,8C), V(B1,D5,D5,64), V(9C,4E,4E,D2), V(49,A9,A9,E0), \ V(D8,6C,6C,B4), V(AC,56,56,FA), V(F3,F4,F4,07), V(CF,EA,EA,25), \ V(CA,65,65,AF), V(F4,7A,7A,8E), V(47,AE,AE,E9), V(10,08,08,18), \ V(6F,BA,BA,D5), V(F0,78,78,88), V(4A,25,25,6F), V(5C,2E,2E,72), \ V(38,1C,1C,24), V(57,A6,A6,F1), V(73,B4,B4,C7), V(97,C6,C6,51), \ V(CB,E8,E8,23), V(A1,DD,DD,7C), V(E8,74,74,9C), V(3E,1F,1F,21), \ V(96,4B,4B,DD), V(61,BD,BD,DC), V(0D,8B,8B,86), V(0F,8A,8A,85), \ V(E0,70,70,90), V(7C,3E,3E,42), V(71,B5,B5,C4), V(CC,66,66,AA), \ V(90,48,48,D8), V(06,03,03,05), V(F7,F6,F6,01), V(1C,0E,0E,12), \ V(C2,61,61,A3), V(6A,35,35,5F), V(AE,57,57,F9), V(69,B9,B9,D0), \ V(17,86,86,91), V(99,C1,C1,58), V(3A,1D,1D,27), V(27,9E,9E,B9), \ V(D9,E1,E1,38), V(EB,F8,F8,13), V(2B,98,98,B3), V(22,11,11,33), \ V(D2,69,69,BB), V(A9,D9,D9,70), V(07,8E,8E,89), V(33,94,94,A7), \ V(2D,9B,9B,B6), V(3C,1E,1E,22), V(15,87,87,92), V(C9,E9,E9,20), \ V(87,CE,CE,49), V(AA,55,55,FF), V(50,28,28,78), V(A5,DF,DF,7A), \ V(03,8C,8C,8F), V(59,A1,A1,F8), V(09,89,89,80), V(1A,0D,0D,17), \ V(65,BF,BF,DA), V(D7,E6,E6,31), V(84,42,42,C6), V(D0,68,68,B8), \ V(82,41,41,C3), V(29,99,99,B0), V(5A,2D,2D,77), V(1E,0F,0F,11), \ V(7B,B0,B0,CB), V(A8,54,54,FC), V(6D,BB,BB,D6), V(2C,16,16,3A) #define V(a,b,c,d) 0x##a##b##c##d static const uint32 FT0[256] = { FT }; #undef V #define V(a,b,c,d) 0x##d##a##b##c static const uint32 FT1[256] = { FT }; #undef V #define V(a,b,c,d) 0x##c##d##a##b static const uint32 FT2[256] = { FT }; #undef V #define V(a,b,c,d) 0x##b##c##d##a static const uint32 FT3[256] = { FT }; #undef V #undef FT /* reverse S-box */ static const uint32 RSb[256] = { 0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38, 0xBF, 0x40, 0xA3, 0x9E, 0x81, 0xF3, 0xD7, 0xFB, 0x7C, 0xE3, 0x39, 0x82, 0x9B, 0x2F, 0xFF, 0x87, 0x34, 0x8E, 0x43, 0x44, 0xC4, 0xDE, 0xE9, 0xCB, 0x54, 0x7B, 0x94, 0x32, 0xA6, 0xC2, 0x23, 0x3D, 0xEE, 0x4C, 0x95, 0x0B, 0x42, 0xFA, 0xC3, 0x4E, 0x08, 0x2E, 0xA1, 0x66, 0x28, 0xD9, 0x24, 0xB2, 0x76, 0x5B, 0xA2, 0x49, 0x6D, 0x8B, 0xD1, 0x25, 0x72, 0xF8, 0xF6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xD4, 0xA4, 0x5C, 0xCC, 0x5D, 0x65, 0xB6, 0x92, 0x6C, 0x70, 0x48, 0x50, 0xFD, 0xED, 0xB9, 0xDA, 0x5E, 0x15, 0x46, 0x57, 0xA7, 0x8D, 0x9D, 0x84, 0x90, 0xD8, 0xAB, 0x00, 0x8C, 0xBC, 0xD3, 0x0A, 0xF7, 0xE4, 0x58, 0x05, 0xB8, 0xB3, 0x45, 0x06, 0xD0, 0x2C, 0x1E, 0x8F, 0xCA, 0x3F, 0x0F, 0x02, 0xC1, 0xAF, 0xBD, 0x03, 0x01, 0x13, 0x8A, 0x6B, 0x3A, 0x91, 0x11, 0x41, 0x4F, 0x67, 0xDC, 0xEA, 0x97, 0xF2, 0xCF, 0xCE, 0xF0, 0xB4, 0xE6, 0x73, 0x96, 0xAC, 0x74, 0x22, 0xE7, 0xAD, 0x35, 0x85, 0xE2, 0xF9, 0x37, 0xE8, 0x1C, 0x75, 0xDF, 0x6E, 0x47, 0xF1, 0x1A, 0x71, 0x1D, 0x29, 0xC5, 0x89, 0x6F, 0xB7, 0x62, 0x0E, 0xAA, 0x18, 0xBE, 0x1B, 0xFC, 0x56, 0x3E, 0x4B, 0xC6, 0xD2, 0x79, 0x20, 0x9A, 0xDB, 0xC0, 0xFE, 0x78, 0xCD, 0x5A, 0xF4, 0x1F, 0xDD, 0xA8, 0x33, 0x88, 0x07, 0xC7, 0x31, 0xB1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xEC, 0x5F, 0x60, 0x51, 0x7F, 0xA9, 0x19, 0xB5, 0x4A, 0x0D, 0x2D, 0xE5, 0x7A, 0x9F, 0x93, 0xC9, 0x9C, 0xEF, 0xA0, 0xE0, 0x3B, 0x4D, 0xAE, 0x2A, 0xF5, 0xB0, 0xC8, 0xEB, 0xBB, 0x3C, 0x83, 0x53, 0x99, 0x61, 0x17, 0x2B, 0x04, 0x7E, 0xBA, 0x77, 0xD6, 0x26, 0xE1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0C, 0x7D }; /* reverse tables */ #define RT \ \ V(51,F4,A7,50), V(7E,41,65,53), V(1A,17,A4,C3), V(3A,27,5E,96), \ V(3B,AB,6B,CB), V(1F,9D,45,F1), V(AC,FA,58,AB), V(4B,E3,03,93), \ V(20,30,FA,55), V(AD,76,6D,F6), V(88,CC,76,91), V(F5,02,4C,25), \ V(4F,E5,D7,FC), V(C5,2A,CB,D7), V(26,35,44,80), V(B5,62,A3,8F), \ V(DE,B1,5A,49), V(25,BA,1B,67), V(45,EA,0E,98), V(5D,FE,C0,E1), \ V(C3,2F,75,02), V(81,4C,F0,12), V(8D,46,97,A3), V(6B,D3,F9,C6), \ V(03,8F,5F,E7), V(15,92,9C,95), V(BF,6D,7A,EB), V(95,52,59,DA), \ V(D4,BE,83,2D), V(58,74,21,D3), V(49,E0,69,29), V(8E,C9,C8,44), \ V(75,C2,89,6A), V(F4,8E,79,78), V(99,58,3E,6B), V(27,B9,71,DD), \ V(BE,E1,4F,B6), V(F0,88,AD,17), V(C9,20,AC,66), V(7D,CE,3A,B4), \ V(63,DF,4A,18), V(E5,1A,31,82), V(97,51,33,60), V(62,53,7F,45), \ V(B1,64,77,E0), V(BB,6B,AE,84), V(FE,81,A0,1C), V(F9,08,2B,94), \ V(70,48,68,58), V(8F,45,FD,19), V(94,DE,6C,87), V(52,7B,F8,B7), \ V(AB,73,D3,23), V(72,4B,02,E2), V(E3,1F,8F,57), V(66,55,AB,2A), \ V(B2,EB,28,07), V(2F,B5,C2,03), V(86,C5,7B,9A), V(D3,37,08,A5), \ V(30,28,87,F2), V(23,BF,A5,B2), V(02,03,6A,BA), V(ED,16,82,5C), \ V(8A,CF,1C,2B), V(A7,79,B4,92), V(F3,07,F2,F0), V(4E,69,E2,A1), \ V(65,DA,F4,CD), V(06,05,BE,D5), V(D1,34,62,1F), V(C4,A6,FE,8A), \ V(34,2E,53,9D), V(A2,F3,55,A0), V(05,8A,E1,32), V(A4,F6,EB,75), \ V(0B,83,EC,39), V(40,60,EF,AA), V(5E,71,9F,06), V(BD,6E,10,51), \ V(3E,21,8A,F9), V(96,DD,06,3D), V(DD,3E,05,AE), V(4D,E6,BD,46), \ V(91,54,8D,B5), V(71,C4,5D,05), V(04,06,D4,6F), V(60,50,15,FF), \ V(19,98,FB,24), V(D6,BD,E9,97), V(89,40,43,CC), V(67,D9,9E,77), \ V(B0,E8,42,BD), V(07,89,8B,88), V(E7,19,5B,38), V(79,C8,EE,DB), \ V(A1,7C,0A,47), V(7C,42,0F,E9), V(F8,84,1E,C9), V(00,00,00,00), \ V(09,80,86,83), V(32,2B,ED,48), V(1E,11,70,AC), V(6C,5A,72,4E), \ V(FD,0E,FF,FB), V(0F,85,38,56), V(3D,AE,D5,1E), V(36,2D,39,27), \ V(0A,0F,D9,64), V(68,5C,A6,21), V(9B,5B,54,D1), V(24,36,2E,3A), \ V(0C,0A,67,B1), V(93,57,E7,0F), V(B4,EE,96,D2), V(1B,9B,91,9E), \ V(80,C0,C5,4F), V(61,DC,20,A2), V(5A,77,4B,69), V(1C,12,1A,16), \ V(E2,93,BA,0A), V(C0,A0,2A,E5), V(3C,22,E0,43), V(12,1B,17,1D), \ V(0E,09,0D,0B), V(F2,8B,C7,AD), V(2D,B6,A8,B9), V(14,1E,A9,C8), \ V(57,F1,19,85), V(AF,75,07,4C), V(EE,99,DD,BB), V(A3,7F,60,FD), \ V(F7,01,26,9F), V(5C,72,F5,BC), V(44,66,3B,C5), V(5B,FB,7E,34), \ V(8B,43,29,76), V(CB,23,C6,DC), V(B6,ED,FC,68), V(B8,E4,F1,63), \ V(D7,31,DC,CA), V(42,63,85,10), V(13,97,22,40), V(84,C6,11,20), \ V(85,4A,24,7D), V(D2,BB,3D,F8), V(AE,F9,32,11), V(C7,29,A1,6D), \ V(1D,9E,2F,4B), V(DC,B2,30,F3), V(0D,86,52,EC), V(77,C1,E3,D0), \ V(2B,B3,16,6C), V(A9,70,B9,99), V(11,94,48,FA), V(47,E9,64,22), \ V(A8,FC,8C,C4), V(A0,F0,3F,1A), V(56,7D,2C,D8), V(22,33,90,EF), \ V(87,49,4E,C7), V(D9,38,D1,C1), V(8C,CA,A2,FE), V(98,D4,0B,36), \ V(A6,F5,81,CF), V(A5,7A,DE,28), V(DA,B7,8E,26), V(3F,AD,BF,A4), \ V(2C,3A,9D,E4), V(50,78,92,0D), V(6A,5F,CC,9B), V(54,7E,46,62), \ V(F6,8D,13,C2), V(90,D8,B8,E8), V(2E,39,F7,5E), V(82,C3,AF,F5), \ V(9F,5D,80,BE), V(69,D0,93,7C), V(6F,D5,2D,A9), V(CF,25,12,B3), \ V(C8,AC,99,3B), V(10,18,7D,A7), V(E8,9C,63,6E), V(DB,3B,BB,7B), \ V(CD,26,78,09), V(6E,59,18,F4), V(EC,9A,B7,01), V(83,4F,9A,A8), \ V(E6,95,6E,65), V(AA,FF,E6,7E), V(21,BC,CF,08), V(EF,15,E8,E6), \ V(BA,E7,9B,D9), V(4A,6F,36,CE), V(EA,9F,09,D4), V(29,B0,7C,D6), \ V(31,A4,B2,AF), V(2A,3F,23,31), V(C6,A5,94,30), V(35,A2,66,C0), \ V(74,4E,BC,37), V(FC,82,CA,A6), V(E0,90,D0,B0), V(33,A7,D8,15), \ V(F1,04,98,4A), V(41,EC,DA,F7), V(7F,CD,50,0E), V(17,91,F6,2F), \ V(76,4D,D6,8D), V(43,EF,B0,4D), V(CC,AA,4D,54), V(E4,96,04,DF), \ V(9E,D1,B5,E3), V(4C,6A,88,1B), V(C1,2C,1F,B8), V(46,65,51,7F), \ V(9D,5E,EA,04), V(01,8C,35,5D), V(FA,87,74,73), V(FB,0B,41,2E), \ V(B3,67,1D,5A), V(92,DB,D2,52), V(E9,10,56,33), V(6D,D6,47,13), \ V(9A,D7,61,8C), V(37,A1,0C,7A), V(59,F8,14,8E), V(EB,13,3C,89), \ V(CE,A9,27,EE), V(B7,61,C9,35), V(E1,1C,E5,ED), V(7A,47,B1,3C), \ V(9C,D2,DF,59), V(55,F2,73,3F), V(18,14,CE,79), V(73,C7,37,BF), \ V(53,F7,CD,EA), V(5F,FD,AA,5B), V(DF,3D,6F,14), V(78,44,DB,86), \ V(CA,AF,F3,81), V(B9,68,C4,3E), V(38,24,34,2C), V(C2,A3,40,5F), \ V(16,1D,C3,72), V(BC,E2,25,0C), V(28,3C,49,8B), V(FF,0D,95,41), \ V(39,A8,01,71), V(08,0C,B3,DE), V(D8,B4,E4,9C), V(64,56,C1,90), \ V(7B,CB,84,61), V(D5,32,B6,70), V(48,6C,5C,74), V(D0,B8,57,42) #define V(a,b,c,d) 0x##a##b##c##d static const uint32 RT0[256] = { RT }; #undef V #define V(a,b,c,d) 0x##d##a##b##c static const uint32 RT1[256] = { RT }; #undef V #define V(a,b,c,d) 0x##c##d##a##b static const uint32 RT2[256] = { RT }; #undef V #define V(a,b,c,d) 0x##b##c##d##a static const uint32 RT3[256] = { RT }; #undef V #undef RT /* round constants */ static const uint32 RCON[10] = { 0x01000000, 0x02000000, 0x04000000, 0x08000000, 0x10000000, 0x20000000, 0x40000000, 0x80000000, 0x1B000000, 0x36000000 }; int do_init = 0; void aes_gen_tables( void ) /* #nocov start */ { } /* #nocov end */ #endif /* platform-independant 32-bit integer manipulation macros */ #define GET_UINT32(n,b,i) \ { \ (n) = ( (uint32) (b)[(i) ] << 24 ) \ | ( (uint32) (b)[(i) + 1] << 16 ) \ | ( (uint32) (b)[(i) + 2] << 8 ) \ | ( (uint32) (b)[(i) + 3] ); \ } #define PUT_UINT32(n,b,i) \ { \ (b)[(i) ] = (uint8) ( (n) >> 24 ); \ (b)[(i) + 1] = (uint8) ( (n) >> 16 ); \ (b)[(i) + 2] = (uint8) ( (n) >> 8 ); \ (b)[(i) + 3] = (uint8) ( (n) ); \ } /* decryption key schedule tables */ int KT_init = 1; uint32 KT0[256]; uint32 KT1[256]; uint32 KT2[256]; uint32 KT3[256]; /* AES key scheduling routine */ int aes_set_key( aes_context *ctx, uint8 *key, int nbits ) { int i; uint32 *RK, *SK; if( do_init ) { aes_gen_tables(); /* #nocov start */ do_init = 0; /* #nocov end */ } switch( nbits ) { case 128: ctx->nr = 10; break; case 192: ctx->nr = 12; break; case 256: ctx->nr = 14; break; default : return( 1 ); /* #nocov */ } RK = ctx->erk; for( i = 0; i < (nbits >> 5); i++ ) { GET_UINT32( RK[i], key, i * 4 ); } /* setup encryption round keys */ switch( nbits ) { case 128: for( i = 0; i < 10; i++, RK += 4 ) { RK[4] = RK[0] ^ RCON[i] ^ ( FSb[ (uint8) ( RK[3] >> 16 ) ] << 24 ) ^ ( FSb[ (uint8) ( RK[3] >> 8 ) ] << 16 ) ^ ( FSb[ (uint8) ( RK[3] ) ] << 8 ) ^ ( FSb[ (uint8) ( RK[3] >> 24 ) ] ); RK[5] = RK[1] ^ RK[4]; RK[6] = RK[2] ^ RK[5]; RK[7] = RK[3] ^ RK[6]; } break; case 192: for( i = 0; i < 8; i++, RK += 6 ) { RK[6] = RK[0] ^ RCON[i] ^ ( FSb[ (uint8) ( RK[5] >> 16 ) ] << 24 ) ^ ( FSb[ (uint8) ( RK[5] >> 8 ) ] << 16 ) ^ ( FSb[ (uint8) ( RK[5] ) ] << 8 ) ^ ( FSb[ (uint8) ( RK[5] >> 24 ) ] ); RK[7] = RK[1] ^ RK[6]; RK[8] = RK[2] ^ RK[7]; RK[9] = RK[3] ^ RK[8]; RK[10] = RK[4] ^ RK[9]; RK[11] = RK[5] ^ RK[10]; } break; case 256: for( i = 0; i < 7; i++, RK += 8 ) { RK[8] = RK[0] ^ RCON[i] ^ ( FSb[ (uint8) ( RK[7] >> 16 ) ] << 24 ) ^ ( FSb[ (uint8) ( RK[7] >> 8 ) ] << 16 ) ^ ( FSb[ (uint8) ( RK[7] ) ] << 8 ) ^ ( FSb[ (uint8) ( RK[7] >> 24 ) ] ); RK[9] = RK[1] ^ RK[8]; RK[10] = RK[2] ^ RK[9]; RK[11] = RK[3] ^ RK[10]; RK[12] = RK[4] ^ ( FSb[ (uint8) ( RK[11] >> 24 ) ] << 24 ) ^ ( FSb[ (uint8) ( RK[11] >> 16 ) ] << 16 ) ^ ( FSb[ (uint8) ( RK[11] >> 8 ) ] << 8 ) ^ ( FSb[ (uint8) ( RK[11] ) ] ); RK[13] = RK[5] ^ RK[12]; RK[14] = RK[6] ^ RK[13]; RK[15] = RK[7] ^ RK[14]; } break; } /* setup decryption round keys */ if( KT_init ) { for( i = 0; i < 256; i++ ) { KT0[i] = RT0[ FSb[i] ]; KT1[i] = RT1[ FSb[i] ]; KT2[i] = RT2[ FSb[i] ]; KT3[i] = RT3[ FSb[i] ]; } KT_init = 0; } SK = ctx->drk; *SK++ = *RK++; *SK++ = *RK++; *SK++ = *RK++; *SK++ = *RK++; for( i = 1; i < ctx->nr; i++ ) { RK -= 8; *SK++ = KT0[ (uint8) ( *RK >> 24 ) ] ^ KT1[ (uint8) ( *RK >> 16 ) ] ^ KT2[ (uint8) ( *RK >> 8 ) ] ^ KT3[ (uint8) ( *RK ) ]; RK++; *SK++ = KT0[ (uint8) ( *RK >> 24 ) ] ^ KT1[ (uint8) ( *RK >> 16 ) ] ^ KT2[ (uint8) ( *RK >> 8 ) ] ^ KT3[ (uint8) ( *RK ) ]; RK++; *SK++ = KT0[ (uint8) ( *RK >> 24 ) ] ^ KT1[ (uint8) ( *RK >> 16 ) ] ^ KT2[ (uint8) ( *RK >> 8 ) ] ^ KT3[ (uint8) ( *RK ) ]; RK++; *SK++ = KT0[ (uint8) ( *RK >> 24 ) ] ^ KT1[ (uint8) ( *RK >> 16 ) ] ^ KT2[ (uint8) ( *RK >> 8 ) ] ^ KT3[ (uint8) ( *RK ) ]; RK++; } RK -= 8; *SK++ = *RK++; *SK++ = *RK++; *SK++ = *RK++; *SK++ = *RK++; return( 0 ); } /* AES 128-bit block encryption routine */ void aes_encrypt( aes_context *ctx, uint8 input[16], uint8 output[16] ) { uint32 *RK, X0, X1, X2, X3, Y0, Y1, Y2, Y3; RK = ctx->erk; GET_UINT32( X0, input, 0 ); X0 ^= RK[0]; GET_UINT32( X1, input, 4 ); X1 ^= RK[1]; GET_UINT32( X2, input, 8 ); X2 ^= RK[2]; GET_UINT32( X3, input, 12 ); X3 ^= RK[3]; #define AES_FROUND(X0,X1,X2,X3,Y0,Y1,Y2,Y3) \ { \ RK += 4; \ \ X0 = RK[0] ^ FT0[ (uint8) ( Y0 >> 24 ) ] ^ \ FT1[ (uint8) ( Y1 >> 16 ) ] ^ \ FT2[ (uint8) ( Y2 >> 8 ) ] ^ \ FT3[ (uint8) ( Y3 ) ]; \ \ X1 = RK[1] ^ FT0[ (uint8) ( Y1 >> 24 ) ] ^ \ FT1[ (uint8) ( Y2 >> 16 ) ] ^ \ FT2[ (uint8) ( Y3 >> 8 ) ] ^ \ FT3[ (uint8) ( Y0 ) ]; \ \ X2 = RK[2] ^ FT0[ (uint8) ( Y2 >> 24 ) ] ^ \ FT1[ (uint8) ( Y3 >> 16 ) ] ^ \ FT2[ (uint8) ( Y0 >> 8 ) ] ^ \ FT3[ (uint8) ( Y1 ) ]; \ \ X3 = RK[3] ^ FT0[ (uint8) ( Y3 >> 24 ) ] ^ \ FT1[ (uint8) ( Y0 >> 16 ) ] ^ \ FT2[ (uint8) ( Y1 >> 8 ) ] ^ \ FT3[ (uint8) ( Y2 ) ]; \ } AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 1 */ AES_FROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 2 */ AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 3 */ AES_FROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 4 */ AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 5 */ AES_FROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 6 */ AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 7 */ AES_FROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 8 */ AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 9 */ if( ctx->nr > 10 ) { AES_FROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 10 */ AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 11 */ } if( ctx->nr > 12 ) { AES_FROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 12 */ AES_FROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 13 */ } /* last round */ RK += 4; X0 = RK[0] ^ ( FSb[ (uint8) ( Y0 >> 24 ) ] << 24 ) ^ ( FSb[ (uint8) ( Y1 >> 16 ) ] << 16 ) ^ ( FSb[ (uint8) ( Y2 >> 8 ) ] << 8 ) ^ ( FSb[ (uint8) ( Y3 ) ] ); X1 = RK[1] ^ ( FSb[ (uint8) ( Y1 >> 24 ) ] << 24 ) ^ ( FSb[ (uint8) ( Y2 >> 16 ) ] << 16 ) ^ ( FSb[ (uint8) ( Y3 >> 8 ) ] << 8 ) ^ ( FSb[ (uint8) ( Y0 ) ] ); X2 = RK[2] ^ ( FSb[ (uint8) ( Y2 >> 24 ) ] << 24 ) ^ ( FSb[ (uint8) ( Y3 >> 16 ) ] << 16 ) ^ ( FSb[ (uint8) ( Y0 >> 8 ) ] << 8 ) ^ ( FSb[ (uint8) ( Y1 ) ] ); X3 = RK[3] ^ ( FSb[ (uint8) ( Y3 >> 24 ) ] << 24 ) ^ ( FSb[ (uint8) ( Y0 >> 16 ) ] << 16 ) ^ ( FSb[ (uint8) ( Y1 >> 8 ) ] << 8 ) ^ ( FSb[ (uint8) ( Y2 ) ] ); PUT_UINT32( X0, output, 0 ); PUT_UINT32( X1, output, 4 ); PUT_UINT32( X2, output, 8 ); PUT_UINT32( X3, output, 12 ); } /* AES 128-bit block decryption routine */ void aes_decrypt( aes_context *ctx, uint8 input[16], uint8 output[16] ) { uint32 *RK, X0, X1, X2, X3, Y0, Y1, Y2, Y3; RK = ctx->drk; GET_UINT32( X0, input, 0 ); X0 ^= RK[0]; GET_UINT32( X1, input, 4 ); X1 ^= RK[1]; GET_UINT32( X2, input, 8 ); X2 ^= RK[2]; GET_UINT32( X3, input, 12 ); X3 ^= RK[3]; #define AES_RROUND(X0,X1,X2,X3,Y0,Y1,Y2,Y3) \ { \ RK += 4; \ \ X0 = RK[0] ^ RT0[ (uint8) ( Y0 >> 24 ) ] ^ \ RT1[ (uint8) ( Y3 >> 16 ) ] ^ \ RT2[ (uint8) ( Y2 >> 8 ) ] ^ \ RT3[ (uint8) ( Y1 ) ]; \ \ X1 = RK[1] ^ RT0[ (uint8) ( Y1 >> 24 ) ] ^ \ RT1[ (uint8) ( Y0 >> 16 ) ] ^ \ RT2[ (uint8) ( Y3 >> 8 ) ] ^ \ RT3[ (uint8) ( Y2 ) ]; \ \ X2 = RK[2] ^ RT0[ (uint8) ( Y2 >> 24 ) ] ^ \ RT1[ (uint8) ( Y1 >> 16 ) ] ^ \ RT2[ (uint8) ( Y0 >> 8 ) ] ^ \ RT3[ (uint8) ( Y3 ) ]; \ \ X3 = RK[3] ^ RT0[ (uint8) ( Y3 >> 24 ) ] ^ \ RT1[ (uint8) ( Y2 >> 16 ) ] ^ \ RT2[ (uint8) ( Y1 >> 8 ) ] ^ \ RT3[ (uint8) ( Y0 ) ]; \ } AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 1 */ AES_RROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 2 */ AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 3 */ AES_RROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 4 */ AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 5 */ AES_RROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 6 */ AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 7 */ AES_RROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 8 */ AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 9 */ if( ctx->nr > 10 ) { AES_RROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 10 */ AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 11 */ } if( ctx->nr > 12 ) { AES_RROUND( X0, X1, X2, X3, Y0, Y1, Y2, Y3 ); /* round 12 */ AES_RROUND( Y0, Y1, Y2, Y3, X0, X1, X2, X3 ); /* round 13 */ } /* last round */ RK += 4; X0 = RK[0] ^ ( RSb[ (uint8) ( Y0 >> 24 ) ] << 24 ) ^ ( RSb[ (uint8) ( Y3 >> 16 ) ] << 16 ) ^ ( RSb[ (uint8) ( Y2 >> 8 ) ] << 8 ) ^ ( RSb[ (uint8) ( Y1 ) ] ); X1 = RK[1] ^ ( RSb[ (uint8) ( Y1 >> 24 ) ] << 24 ) ^ ( RSb[ (uint8) ( Y0 >> 16 ) ] << 16 ) ^ ( RSb[ (uint8) ( Y3 >> 8 ) ] << 8 ) ^ ( RSb[ (uint8) ( Y2 ) ] ); X2 = RK[2] ^ ( RSb[ (uint8) ( Y2 >> 24 ) ] << 24 ) ^ ( RSb[ (uint8) ( Y1 >> 16 ) ] << 16 ) ^ ( RSb[ (uint8) ( Y0 >> 8 ) ] << 8 ) ^ ( RSb[ (uint8) ( Y3 ) ] ); X3 = RK[3] ^ ( RSb[ (uint8) ( Y3 >> 24 ) ] << 24 ) ^ ( RSb[ (uint8) ( Y2 >> 16 ) ] << 16 ) ^ ( RSb[ (uint8) ( Y1 >> 8 ) ] << 8 ) ^ ( RSb[ (uint8) ( Y0 ) ] ); PUT_UINT32( X0, output, 0 ); PUT_UINT32( X1, output, 4 ); PUT_UINT32( X2, output, 8 ); PUT_UINT32( X3, output, 12 ); } #ifdef TEST #include #include /* * Rijndael Monte Carlo Test: ECB mode * source: NIST - rijndael-vals.zip */ static unsigned char AES_enc_test[3][16] = { { 0xA0, 0x43, 0x77, 0xAB, 0xE2, 0x59, 0xB0, 0xD0, 0xB5, 0xBA, 0x2D, 0x40, 0xA5, 0x01, 0x97, 0x1B }, { 0x4E, 0x46, 0xF8, 0xC5, 0x09, 0x2B, 0x29, 0xE2, 0x9A, 0x97, 0x1A, 0x0C, 0xD1, 0xF6, 0x10, 0xFB }, { 0x1F, 0x67, 0x63, 0xDF, 0x80, 0x7A, 0x7E, 0x70, 0x96, 0x0D, 0x4C, 0xD3, 0x11, 0x8E, 0x60, 0x1A } }; static unsigned char AES_dec_test[3][16] = { { 0xF5, 0xBF, 0x8B, 0x37, 0x13, 0x6F, 0x2E, 0x1F, 0x6B, 0xEC, 0x6F, 0x57, 0x20, 0x21, 0xE3, 0xBA }, { 0xF1, 0xA8, 0x1B, 0x68, 0xF6, 0xE5, 0xA6, 0x27, 0x1A, 0x8C, 0xB2, 0x4E, 0x7D, 0x94, 0x91, 0xEF }, { 0x4D, 0xE0, 0xC6, 0xDF, 0x7C, 0xB1, 0x69, 0x72, 0x84, 0x60, 0x4D, 0x60, 0x27, 0x1B, 0xC5, 0x9A } }; int main( void ) { int m, n, i, j; aes_context ctx; unsigned char buf[16]; unsigned char key[32]; for( m = 0; m < 2; m++ ) { printf( "\n Rijndael Monte Carlo Test (ECB mode) - " ); if( m == 0 ) printf( "encryption\n\n" ); if( m == 1 ) printf( "decryption\n\n" ); for( n = 0; n < 3; n++ ) { printf( " Test %d, key size = %3d bits: ", n + 1, 128 + n * 64 ); fflush( stdout ); memset( buf, 0, 16 ); memset( key, 0, 16 + n * 8 ); for( i = 0; i < 400; i++ ) { aes_set_key( &ctx, key, 128 + n * 64 ); for( j = 0; j < 9999; j++ ) { if( m == 0 ) aes_encrypt( &ctx, buf, buf ); if( m == 1 ) aes_decrypt( &ctx, buf, buf ); } if( n > 0 ) { for( j = 0; j < (n << 3); j++ ) { key[j] ^= buf[j + 16 - (n << 3)]; } } if( m == 0 ) aes_encrypt( &ctx, buf, buf ); if( m == 1 ) aes_decrypt( &ctx, buf, buf ); for( j = 0; j < 16; j++ ) { key[j + (n << 3)] ^= buf[j]; } } if( ( m == 0 && memcmp( buf, AES_enc_test[n], 16 ) != 0 ) || ( m == 1 && memcmp( buf, AES_dec_test[n], 16 ) != 0 ) ) { printf( "failed!\n" ); return( 1 ); } printf( "passed.\n" ); } } printf( "\n" ); return( 0 ); } #endif digest/src/crc32c_portable.cpp0000644000176200001440000004673514423472174016022 0ustar liggesusers// Copyright 2008 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "crc32c/crc32c_internal.h" #include #include #include "crc32c/crc32c_prefetch.h" #include "crc32c/crc32c_read_le.h" #include "crc32c/crc32c_round_up.h" namespace { const uint32_t kByteExtensionTable[256] = { 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351}; const uint32_t kStrideExtensionTable0[256] = { 0x00000000, 0x30d23865, 0x61a470ca, 0x517648af, 0xc348e194, 0xf39ad9f1, 0xa2ec915e, 0x923ea93b, 0x837db5d9, 0xb3af8dbc, 0xe2d9c513, 0xd20bfd76, 0x4035544d, 0x70e76c28, 0x21912487, 0x11431ce2, 0x03171d43, 0x33c52526, 0x62b36d89, 0x526155ec, 0xc05ffcd7, 0xf08dc4b2, 0xa1fb8c1d, 0x9129b478, 0x806aa89a, 0xb0b890ff, 0xe1ced850, 0xd11ce035, 0x4322490e, 0x73f0716b, 0x228639c4, 0x125401a1, 0x062e3a86, 0x36fc02e3, 0x678a4a4c, 0x57587229, 0xc566db12, 0xf5b4e377, 0xa4c2abd8, 0x941093bd, 0x85538f5f, 0xb581b73a, 0xe4f7ff95, 0xd425c7f0, 0x461b6ecb, 0x76c956ae, 0x27bf1e01, 0x176d2664, 0x053927c5, 0x35eb1fa0, 0x649d570f, 0x544f6f6a, 0xc671c651, 0xf6a3fe34, 0xa7d5b69b, 0x97078efe, 0x8644921c, 0xb696aa79, 0xe7e0e2d6, 0xd732dab3, 0x450c7388, 0x75de4bed, 0x24a80342, 0x147a3b27, 0x0c5c750c, 0x3c8e4d69, 0x6df805c6, 0x5d2a3da3, 0xcf149498, 0xffc6acfd, 0xaeb0e452, 0x9e62dc37, 0x8f21c0d5, 0xbff3f8b0, 0xee85b01f, 0xde57887a, 0x4c692141, 0x7cbb1924, 0x2dcd518b, 0x1d1f69ee, 0x0f4b684f, 0x3f99502a, 0x6eef1885, 0x5e3d20e0, 0xcc0389db, 0xfcd1b1be, 0xada7f911, 0x9d75c174, 0x8c36dd96, 0xbce4e5f3, 0xed92ad5c, 0xdd409539, 0x4f7e3c02, 0x7fac0467, 0x2eda4cc8, 0x1e0874ad, 0x0a724f8a, 0x3aa077ef, 0x6bd63f40, 0x5b040725, 0xc93aae1e, 0xf9e8967b, 0xa89eded4, 0x984ce6b1, 0x890ffa53, 0xb9ddc236, 0xe8ab8a99, 0xd879b2fc, 0x4a471bc7, 0x7a9523a2, 0x2be36b0d, 0x1b315368, 0x096552c9, 0x39b76aac, 0x68c12203, 0x58131a66, 0xca2db35d, 0xfaff8b38, 0xab89c397, 0x9b5bfbf2, 0x8a18e710, 0xbacadf75, 0xebbc97da, 0xdb6eafbf, 0x49500684, 0x79823ee1, 0x28f4764e, 0x18264e2b, 0x18b8ea18, 0x286ad27d, 0x791c9ad2, 0x49cea2b7, 0xdbf00b8c, 0xeb2233e9, 0xba547b46, 0x8a864323, 0x9bc55fc1, 0xab1767a4, 0xfa612f0b, 0xcab3176e, 0x588dbe55, 0x685f8630, 0x3929ce9f, 0x09fbf6fa, 0x1baff75b, 0x2b7dcf3e, 0x7a0b8791, 0x4ad9bff4, 0xd8e716cf, 0xe8352eaa, 0xb9436605, 0x89915e60, 0x98d24282, 0xa8007ae7, 0xf9763248, 0xc9a40a2d, 0x5b9aa316, 0x6b489b73, 0x3a3ed3dc, 0x0aecebb9, 0x1e96d09e, 0x2e44e8fb, 0x7f32a054, 0x4fe09831, 0xddde310a, 0xed0c096f, 0xbc7a41c0, 0x8ca879a5, 0x9deb6547, 0xad395d22, 0xfc4f158d, 0xcc9d2de8, 0x5ea384d3, 0x6e71bcb6, 0x3f07f419, 0x0fd5cc7c, 0x1d81cddd, 0x2d53f5b8, 0x7c25bd17, 0x4cf78572, 0xdec92c49, 0xee1b142c, 0xbf6d5c83, 0x8fbf64e6, 0x9efc7804, 0xae2e4061, 0xff5808ce, 0xcf8a30ab, 0x5db49990, 0x6d66a1f5, 0x3c10e95a, 0x0cc2d13f, 0x14e49f14, 0x2436a771, 0x7540efde, 0x4592d7bb, 0xd7ac7e80, 0xe77e46e5, 0xb6080e4a, 0x86da362f, 0x97992acd, 0xa74b12a8, 0xf63d5a07, 0xc6ef6262, 0x54d1cb59, 0x6403f33c, 0x3575bb93, 0x05a783f6, 0x17f38257, 0x2721ba32, 0x7657f29d, 0x4685caf8, 0xd4bb63c3, 0xe4695ba6, 0xb51f1309, 0x85cd2b6c, 0x948e378e, 0xa45c0feb, 0xf52a4744, 0xc5f87f21, 0x57c6d61a, 0x6714ee7f, 0x3662a6d0, 0x06b09eb5, 0x12caa592, 0x22189df7, 0x736ed558, 0x43bced3d, 0xd1824406, 0xe1507c63, 0xb02634cc, 0x80f40ca9, 0x91b7104b, 0xa165282e, 0xf0136081, 0xc0c158e4, 0x52fff1df, 0x622dc9ba, 0x335b8115, 0x0389b970, 0x11ddb8d1, 0x210f80b4, 0x7079c81b, 0x40abf07e, 0xd2955945, 0xe2476120, 0xb331298f, 0x83e311ea, 0x92a00d08, 0xa272356d, 0xf3047dc2, 0xc3d645a7, 0x51e8ec9c, 0x613ad4f9, 0x304c9c56, 0x009ea433}; const uint32_t kStrideExtensionTable1[256] = { 0x00000000, 0x54075546, 0xa80eaa8c, 0xfc09ffca, 0x55f123e9, 0x01f676af, 0xfdff8965, 0xa9f8dc23, 0xabe247d2, 0xffe51294, 0x03eced5e, 0x57ebb818, 0xfe13643b, 0xaa14317d, 0x561dceb7, 0x021a9bf1, 0x5228f955, 0x062fac13, 0xfa2653d9, 0xae21069f, 0x07d9dabc, 0x53de8ffa, 0xafd77030, 0xfbd02576, 0xf9cabe87, 0xadcdebc1, 0x51c4140b, 0x05c3414d, 0xac3b9d6e, 0xf83cc828, 0x043537e2, 0x503262a4, 0xa451f2aa, 0xf056a7ec, 0x0c5f5826, 0x58580d60, 0xf1a0d143, 0xa5a78405, 0x59ae7bcf, 0x0da92e89, 0x0fb3b578, 0x5bb4e03e, 0xa7bd1ff4, 0xf3ba4ab2, 0x5a429691, 0x0e45c3d7, 0xf24c3c1d, 0xa64b695b, 0xf6790bff, 0xa27e5eb9, 0x5e77a173, 0x0a70f435, 0xa3882816, 0xf78f7d50, 0x0b86829a, 0x5f81d7dc, 0x5d9b4c2d, 0x099c196b, 0xf595e6a1, 0xa192b3e7, 0x086a6fc4, 0x5c6d3a82, 0xa064c548, 0xf463900e, 0x4d4f93a5, 0x1948c6e3, 0xe5413929, 0xb1466c6f, 0x18beb04c, 0x4cb9e50a, 0xb0b01ac0, 0xe4b74f86, 0xe6add477, 0xb2aa8131, 0x4ea37efb, 0x1aa42bbd, 0xb35cf79e, 0xe75ba2d8, 0x1b525d12, 0x4f550854, 0x1f676af0, 0x4b603fb6, 0xb769c07c, 0xe36e953a, 0x4a964919, 0x1e911c5f, 0xe298e395, 0xb69fb6d3, 0xb4852d22, 0xe0827864, 0x1c8b87ae, 0x488cd2e8, 0xe1740ecb, 0xb5735b8d, 0x497aa447, 0x1d7df101, 0xe91e610f, 0xbd193449, 0x4110cb83, 0x15179ec5, 0xbcef42e6, 0xe8e817a0, 0x14e1e86a, 0x40e6bd2c, 0x42fc26dd, 0x16fb739b, 0xeaf28c51, 0xbef5d917, 0x170d0534, 0x430a5072, 0xbf03afb8, 0xeb04fafe, 0xbb36985a, 0xef31cd1c, 0x133832d6, 0x473f6790, 0xeec7bbb3, 0xbac0eef5, 0x46c9113f, 0x12ce4479, 0x10d4df88, 0x44d38ace, 0xb8da7504, 0xecdd2042, 0x4525fc61, 0x1122a927, 0xed2b56ed, 0xb92c03ab, 0x9a9f274a, 0xce98720c, 0x32918dc6, 0x6696d880, 0xcf6e04a3, 0x9b6951e5, 0x6760ae2f, 0x3367fb69, 0x317d6098, 0x657a35de, 0x9973ca14, 0xcd749f52, 0x648c4371, 0x308b1637, 0xcc82e9fd, 0x9885bcbb, 0xc8b7de1f, 0x9cb08b59, 0x60b97493, 0x34be21d5, 0x9d46fdf6, 0xc941a8b0, 0x3548577a, 0x614f023c, 0x635599cd, 0x3752cc8b, 0xcb5b3341, 0x9f5c6607, 0x36a4ba24, 0x62a3ef62, 0x9eaa10a8, 0xcaad45ee, 0x3eced5e0, 0x6ac980a6, 0x96c07f6c, 0xc2c72a2a, 0x6b3ff609, 0x3f38a34f, 0xc3315c85, 0x973609c3, 0x952c9232, 0xc12bc774, 0x3d2238be, 0x69256df8, 0xc0ddb1db, 0x94dae49d, 0x68d31b57, 0x3cd44e11, 0x6ce62cb5, 0x38e179f3, 0xc4e88639, 0x90efd37f, 0x39170f5c, 0x6d105a1a, 0x9119a5d0, 0xc51ef096, 0xc7046b67, 0x93033e21, 0x6f0ac1eb, 0x3b0d94ad, 0x92f5488e, 0xc6f21dc8, 0x3afbe202, 0x6efcb744, 0xd7d0b4ef, 0x83d7e1a9, 0x7fde1e63, 0x2bd94b25, 0x82219706, 0xd626c240, 0x2a2f3d8a, 0x7e2868cc, 0x7c32f33d, 0x2835a67b, 0xd43c59b1, 0x803b0cf7, 0x29c3d0d4, 0x7dc48592, 0x81cd7a58, 0xd5ca2f1e, 0x85f84dba, 0xd1ff18fc, 0x2df6e736, 0x79f1b270, 0xd0096e53, 0x840e3b15, 0x7807c4df, 0x2c009199, 0x2e1a0a68, 0x7a1d5f2e, 0x8614a0e4, 0xd213f5a2, 0x7beb2981, 0x2fec7cc7, 0xd3e5830d, 0x87e2d64b, 0x73814645, 0x27861303, 0xdb8fecc9, 0x8f88b98f, 0x267065ac, 0x727730ea, 0x8e7ecf20, 0xda799a66, 0xd8630197, 0x8c6454d1, 0x706dab1b, 0x246afe5d, 0x8d92227e, 0xd9957738, 0x259c88f2, 0x719bddb4, 0x21a9bf10, 0x75aeea56, 0x89a7159c, 0xdda040da, 0x74589cf9, 0x205fc9bf, 0xdc563675, 0x88516333, 0x8a4bf8c2, 0xde4cad84, 0x2245524e, 0x76420708, 0xdfbadb2b, 0x8bbd8e6d, 0x77b471a7, 0x23b324e1}; const uint32_t kStrideExtensionTable2[256] = { 0x00000000, 0x678efd01, 0xcf1dfa02, 0xa8930703, 0x9bd782f5, 0xfc597ff4, 0x54ca78f7, 0x334485f6, 0x3243731b, 0x55cd8e1a, 0xfd5e8919, 0x9ad07418, 0xa994f1ee, 0xce1a0cef, 0x66890bec, 0x0107f6ed, 0x6486e636, 0x03081b37, 0xab9b1c34, 0xcc15e135, 0xff5164c3, 0x98df99c2, 0x304c9ec1, 0x57c263c0, 0x56c5952d, 0x314b682c, 0x99d86f2f, 0xfe56922e, 0xcd1217d8, 0xaa9cead9, 0x020fedda, 0x658110db, 0xc90dcc6c, 0xae83316d, 0x0610366e, 0x619ecb6f, 0x52da4e99, 0x3554b398, 0x9dc7b49b, 0xfa49499a, 0xfb4ebf77, 0x9cc04276, 0x34534575, 0x53ddb874, 0x60993d82, 0x0717c083, 0xaf84c780, 0xc80a3a81, 0xad8b2a5a, 0xca05d75b, 0x6296d058, 0x05182d59, 0x365ca8af, 0x51d255ae, 0xf94152ad, 0x9ecfafac, 0x9fc85941, 0xf846a440, 0x50d5a343, 0x375b5e42, 0x041fdbb4, 0x639126b5, 0xcb0221b6, 0xac8cdcb7, 0x97f7ee29, 0xf0791328, 0x58ea142b, 0x3f64e92a, 0x0c206cdc, 0x6bae91dd, 0xc33d96de, 0xa4b36bdf, 0xa5b49d32, 0xc23a6033, 0x6aa96730, 0x0d279a31, 0x3e631fc7, 0x59ede2c6, 0xf17ee5c5, 0x96f018c4, 0xf371081f, 0x94fff51e, 0x3c6cf21d, 0x5be20f1c, 0x68a68aea, 0x0f2877eb, 0xa7bb70e8, 0xc0358de9, 0xc1327b04, 0xa6bc8605, 0x0e2f8106, 0x69a17c07, 0x5ae5f9f1, 0x3d6b04f0, 0x95f803f3, 0xf276fef2, 0x5efa2245, 0x3974df44, 0x91e7d847, 0xf6692546, 0xc52da0b0, 0xa2a35db1, 0x0a305ab2, 0x6dbea7b3, 0x6cb9515e, 0x0b37ac5f, 0xa3a4ab5c, 0xc42a565d, 0xf76ed3ab, 0x90e02eaa, 0x387329a9, 0x5ffdd4a8, 0x3a7cc473, 0x5df23972, 0xf5613e71, 0x92efc370, 0xa1ab4686, 0xc625bb87, 0x6eb6bc84, 0x09384185, 0x083fb768, 0x6fb14a69, 0xc7224d6a, 0xa0acb06b, 0x93e8359d, 0xf466c89c, 0x5cf5cf9f, 0x3b7b329e, 0x2a03aaa3, 0x4d8d57a2, 0xe51e50a1, 0x8290ada0, 0xb1d42856, 0xd65ad557, 0x7ec9d254, 0x19472f55, 0x1840d9b8, 0x7fce24b9, 0xd75d23ba, 0xb0d3debb, 0x83975b4d, 0xe419a64c, 0x4c8aa14f, 0x2b045c4e, 0x4e854c95, 0x290bb194, 0x8198b697, 0xe6164b96, 0xd552ce60, 0xb2dc3361, 0x1a4f3462, 0x7dc1c963, 0x7cc63f8e, 0x1b48c28f, 0xb3dbc58c, 0xd455388d, 0xe711bd7b, 0x809f407a, 0x280c4779, 0x4f82ba78, 0xe30e66cf, 0x84809bce, 0x2c139ccd, 0x4b9d61cc, 0x78d9e43a, 0x1f57193b, 0xb7c41e38, 0xd04ae339, 0xd14d15d4, 0xb6c3e8d5, 0x1e50efd6, 0x79de12d7, 0x4a9a9721, 0x2d146a20, 0x85876d23, 0xe2099022, 0x878880f9, 0xe0067df8, 0x48957afb, 0x2f1b87fa, 0x1c5f020c, 0x7bd1ff0d, 0xd342f80e, 0xb4cc050f, 0xb5cbf3e2, 0xd2450ee3, 0x7ad609e0, 0x1d58f4e1, 0x2e1c7117, 0x49928c16, 0xe1018b15, 0x868f7614, 0xbdf4448a, 0xda7ab98b, 0x72e9be88, 0x15674389, 0x2623c67f, 0x41ad3b7e, 0xe93e3c7d, 0x8eb0c17c, 0x8fb73791, 0xe839ca90, 0x40aacd93, 0x27243092, 0x1460b564, 0x73ee4865, 0xdb7d4f66, 0xbcf3b267, 0xd972a2bc, 0xbefc5fbd, 0x166f58be, 0x71e1a5bf, 0x42a52049, 0x252bdd48, 0x8db8da4b, 0xea36274a, 0xeb31d1a7, 0x8cbf2ca6, 0x242c2ba5, 0x43a2d6a4, 0x70e65352, 0x1768ae53, 0xbffba950, 0xd8755451, 0x74f988e6, 0x137775e7, 0xbbe472e4, 0xdc6a8fe5, 0xef2e0a13, 0x88a0f712, 0x2033f011, 0x47bd0d10, 0x46bafbfd, 0x213406fc, 0x89a701ff, 0xee29fcfe, 0xdd6d7908, 0xbae38409, 0x1270830a, 0x75fe7e0b, 0x107f6ed0, 0x77f193d1, 0xdf6294d2, 0xb8ec69d3, 0x8ba8ec25, 0xec261124, 0x44b51627, 0x233beb26, 0x223c1dcb, 0x45b2e0ca, 0xed21e7c9, 0x8aaf1ac8, 0xb9eb9f3e, 0xde65623f, 0x76f6653c, 0x1178983d}; const uint32_t kStrideExtensionTable3[256] = { 0x00000000, 0xf20c0dfe, 0xe1f46d0d, 0x13f860f3, 0xc604aceb, 0x3408a115, 0x27f0c1e6, 0xd5fccc18, 0x89e52f27, 0x7be922d9, 0x6811422a, 0x9a1d4fd4, 0x4fe183cc, 0xbded8e32, 0xae15eec1, 0x5c19e33f, 0x162628bf, 0xe42a2541, 0xf7d245b2, 0x05de484c, 0xd0228454, 0x222e89aa, 0x31d6e959, 0xc3dae4a7, 0x9fc30798, 0x6dcf0a66, 0x7e376a95, 0x8c3b676b, 0x59c7ab73, 0xabcba68d, 0xb833c67e, 0x4a3fcb80, 0x2c4c517e, 0xde405c80, 0xcdb83c73, 0x3fb4318d, 0xea48fd95, 0x1844f06b, 0x0bbc9098, 0xf9b09d66, 0xa5a97e59, 0x57a573a7, 0x445d1354, 0xb6511eaa, 0x63add2b2, 0x91a1df4c, 0x8259bfbf, 0x7055b241, 0x3a6a79c1, 0xc866743f, 0xdb9e14cc, 0x29921932, 0xfc6ed52a, 0x0e62d8d4, 0x1d9ab827, 0xef96b5d9, 0xb38f56e6, 0x41835b18, 0x527b3beb, 0xa0773615, 0x758bfa0d, 0x8787f7f3, 0x947f9700, 0x66739afe, 0x5898a2fc, 0xaa94af02, 0xb96ccff1, 0x4b60c20f, 0x9e9c0e17, 0x6c9003e9, 0x7f68631a, 0x8d646ee4, 0xd17d8ddb, 0x23718025, 0x3089e0d6, 0xc285ed28, 0x17792130, 0xe5752cce, 0xf68d4c3d, 0x048141c3, 0x4ebe8a43, 0xbcb287bd, 0xaf4ae74e, 0x5d46eab0, 0x88ba26a8, 0x7ab62b56, 0x694e4ba5, 0x9b42465b, 0xc75ba564, 0x3557a89a, 0x26afc869, 0xd4a3c597, 0x015f098f, 0xf3530471, 0xe0ab6482, 0x12a7697c, 0x74d4f382, 0x86d8fe7c, 0x95209e8f, 0x672c9371, 0xb2d05f69, 0x40dc5297, 0x53243264, 0xa1283f9a, 0xfd31dca5, 0x0f3dd15b, 0x1cc5b1a8, 0xeec9bc56, 0x3b35704e, 0xc9397db0, 0xdac11d43, 0x28cd10bd, 0x62f2db3d, 0x90fed6c3, 0x8306b630, 0x710abbce, 0xa4f677d6, 0x56fa7a28, 0x45021adb, 0xb70e1725, 0xeb17f41a, 0x191bf9e4, 0x0ae39917, 0xf8ef94e9, 0x2d1358f1, 0xdf1f550f, 0xcce735fc, 0x3eeb3802, 0xb13145f8, 0x433d4806, 0x50c528f5, 0xa2c9250b, 0x7735e913, 0x8539e4ed, 0x96c1841e, 0x64cd89e0, 0x38d46adf, 0xcad86721, 0xd92007d2, 0x2b2c0a2c, 0xfed0c634, 0x0cdccbca, 0x1f24ab39, 0xed28a6c7, 0xa7176d47, 0x551b60b9, 0x46e3004a, 0xb4ef0db4, 0x6113c1ac, 0x931fcc52, 0x80e7aca1, 0x72eba15f, 0x2ef24260, 0xdcfe4f9e, 0xcf062f6d, 0x3d0a2293, 0xe8f6ee8b, 0x1afae375, 0x09028386, 0xfb0e8e78, 0x9d7d1486, 0x6f711978, 0x7c89798b, 0x8e857475, 0x5b79b86d, 0xa975b593, 0xba8dd560, 0x4881d89e, 0x14983ba1, 0xe694365f, 0xf56c56ac, 0x07605b52, 0xd29c974a, 0x20909ab4, 0x3368fa47, 0xc164f7b9, 0x8b5b3c39, 0x795731c7, 0x6aaf5134, 0x98a35cca, 0x4d5f90d2, 0xbf539d2c, 0xacabfddf, 0x5ea7f021, 0x02be131e, 0xf0b21ee0, 0xe34a7e13, 0x114673ed, 0xc4babff5, 0x36b6b20b, 0x254ed2f8, 0xd742df06, 0xe9a9e704, 0x1ba5eafa, 0x085d8a09, 0xfa5187f7, 0x2fad4bef, 0xdda14611, 0xce5926e2, 0x3c552b1c, 0x604cc823, 0x9240c5dd, 0x81b8a52e, 0x73b4a8d0, 0xa64864c8, 0x54446936, 0x47bc09c5, 0xb5b0043b, 0xff8fcfbb, 0x0d83c245, 0x1e7ba2b6, 0xec77af48, 0x398b6350, 0xcb876eae, 0xd87f0e5d, 0x2a7303a3, 0x766ae09c, 0x8466ed62, 0x979e8d91, 0x6592806f, 0xb06e4c77, 0x42624189, 0x519a217a, 0xa3962c84, 0xc5e5b67a, 0x37e9bb84, 0x2411db77, 0xd61dd689, 0x03e11a91, 0xf1ed176f, 0xe215779c, 0x10197a62, 0x4c00995d, 0xbe0c94a3, 0xadf4f450, 0x5ff8f9ae, 0x8a0435b6, 0x78083848, 0x6bf058bb, 0x99fc5545, 0xd3c39ec5, 0x21cf933b, 0x3237f3c8, 0xc03bfe36, 0x15c7322e, 0xe7cb3fd0, 0xf4335f23, 0x063f52dd, 0x5a26b1e2, 0xa82abc1c, 0xbbd2dcef, 0x49ded111, 0x9c221d09, 0x6e2e10f7, 0x7dd67004, 0x8fda7dfa}; constexpr const ptrdiff_t kPrefetchHorizon = 256; } // namespace namespace crc32c { uint32_t ExtendPortable(uint32_t crc, const uint8_t* data, size_t size) { const uint8_t* p = data; const uint8_t* e = p + size; uint32_t l = crc ^ kCRC32Xor; // Process one byte at a time. #define STEP1 \ do { \ int c = (l & 0xff) ^ *p++; \ l = kByteExtensionTable[c] ^ (l >> 8); \ } while (0) // Process one of the 4 strides of 4-byte data. #define STEP4(s) \ do { \ crc##s = ReadUint32LE(p + s * 4) ^ kStrideExtensionTable3[crc##s & 0xff] ^ \ kStrideExtensionTable2[(crc##s >> 8) & 0xff] ^ \ kStrideExtensionTable1[(crc##s >> 16) & 0xff] ^ \ kStrideExtensionTable0[crc##s >> 24]; \ } while (0) // Process a 16-byte swath of 4 strides, each of which has 4 bytes of data. #define STEP16 \ do { \ STEP4(0); \ STEP4(1); \ STEP4(2); \ STEP4(3); \ p += 16; \ } while (0) // Process 4 bytes that were already loaded into a word. #define STEP4W(w) \ do { \ w ^= l; \ for (size_t i = 0; i < 4; ++i) { \ w = (w >> 8) ^ kByteExtensionTable[w & 0xff]; \ } \ l = w; \ } while (0) // Point x at first 4-byte aligned byte in the buffer. This might be past the // end of the buffer. const uint8_t* x = RoundUp<4>(p); if (x <= e) { // Process bytes p is 4-byte aligned. while (p != x) { STEP1; } } if ((e - p) >= 16) { // #nocov start // Load a 16-byte swath into the stride partial results. uint32_t crc0 = ReadUint32LE(p + 0 * 4) ^ l; uint32_t crc1 = ReadUint32LE(p + 1 * 4); uint32_t crc2 = ReadUint32LE(p + 2 * 4); uint32_t crc3 = ReadUint32LE(p + 3 * 4); p += 16; while ((e - p) > kPrefetchHorizon) { RequestPrefetch(p + kPrefetchHorizon); // Process 64 bytes at a time. STEP16; STEP16; STEP16; STEP16; } // Process one 16-byte swath at a time. while ((e - p) >= 16) { STEP16; } // Advance one word at a time as far as possible. while ((e - p) >= 4) { STEP4(0); uint32_t tmp = crc0; crc0 = crc1; crc1 = crc2; crc2 = crc3; crc3 = tmp; p += 4; } // Combine the 4 partial stride results. l = 0; STEP4W(crc0); STEP4W(crc1); STEP4W(crc2); STEP4W(crc3); } // #nocov end // Process the last few bytes. while (p != e) { STEP1; } #undef STEP4W #undef STEP16 #undef STEP4 #undef STEP1 return l ^ kCRC32Xor; } } // namespace crc32c digest/src/zutil.h0000644000176200001440000000505312247504516013651 0ustar liggesusers/* zutil.h -- internal interface and configuration of the compression library * Copyright (C) 1995-2003 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of the compression library and is subject to change. Applications should only use zlib.h. */ /* @(#) $Id$ */ #ifndef ZUTIL_H #define ZUTIL_H #define ZLIB_INTERNAL #include "zlib.h" #ifdef STDC # include # include # include #endif #ifdef NO_ERRNO_H extern int errno; #else # include #endif #ifndef local # define local static #endif /* compile with -Dlocal if your debugger can't find static symbols */ typedef unsigned char uch; typedef uch FAR uchf; typedef unsigned short ush; typedef ush FAR ushf; typedef unsigned long ulg; extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ #define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] #define ERR_RETURN(strm,err) \ return (strm->msg = (char*)ERR_MSG(err), (err)) /* To be used only when the state is known to be valid */ /* common constants */ #ifndef DEF_WBITS # define DEF_WBITS MAX_WBITS #endif /* default windowBits for decompression. MAX_WBITS is for compression only */ #if MAX_MEM_LEVEL >= 8 # define DEF_MEM_LEVEL 8 #else # define DEF_MEM_LEVEL MAX_MEM_LEVEL #endif /* default memLevel */ #define STORED_BLOCK 0 #define STATIC_TREES 1 #define DYN_TREES 2 /* The three kinds of block type */ #define MIN_MATCH 3 #define MAX_MATCH 258 /* The minimum and maximum match lengths */ #define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ /* target dependencies */ #if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) # define OS_CODE 0x00 # if defined(__TURBOC__) || defined(__BORLANDC__) # if(__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) /* Allow compilation with ANSI keywords only enabled */ void _Cdecl farfree( void *block ); void *_Cdecl farmalloc( unsigned long nbytes ); # else # include # endif # else /* MSC or DJGPP */ # include # endif #endif #if (defined(_MSC_VER) && (_MSC_VER > 600)) # if defined(_WIN32_WCE) # define fdopen(fd,mode) NULL /* No fdopen() */ # ifndef _PTRDIFF_T_DEFINED typedef int ptrdiff_t; # define _PTRDIFF_T_DEFINED # endif # else # define fdopen(fd,type) _fdopen(fd,type) # endif #endif #endif /* ZUTIL_H */ digest/src/crc32c.h0000644000176200001440000000435514423472174013567 0ustar liggesusers/* Copyright 2017 The CRC32C Authors. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. See the AUTHORS file for names of contributors. */ #ifndef CRC32C_CRC32C_H_ #define CRC32C_CRC32C_H_ /* The API exported by the CRC32C project. */ #if defined(__cplusplus) #include #include #include #else /* !defined(__cplusplus) */ #include #include #endif /* !defined(__cplusplus) */ /* The C API. */ #if defined(__cplusplus) extern "C" { #endif /* defined(__cplusplus) */ /* Extends "crc" with the CRC32C of "count" bytes in the buffer pointed by "data" */ uint32_t crc32c_extend(uint32_t crc, const uint8_t* data, size_t count); /* Computes the CRC32C of "count" bytes in the buffer pointed by "data". */ uint32_t crc32c_value(const uint8_t* data, size_t count); #ifdef __cplusplus } /* end extern "C" */ #endif /* defined(__cplusplus) */ /* The C++ API. */ #if defined(__cplusplus) namespace crc32c { // Extends "crc" with the CRC32C of "count" bytes in the buffer pointed by // "data". uint32_t Extend(uint32_t crc, const uint8_t* data, size_t count); // Computes the CRC32C of "count" bytes in the buffer pointed by "data". inline uint32_t Crc32c(const uint8_t* data, size_t count) { return Extend(0, data, count); } // Computes the CRC32C of "count" bytes in the buffer pointed by "data". inline uint32_t Crc32c(const char* data, size_t count) { return Extend(0, reinterpret_cast(data), count); } // Computes the CRC32C of the string's content. inline uint32_t Crc32c(const std::string& string) { return Crc32c(reinterpret_cast(string.data()), string.size()); } } // namespace crc32c #if __cplusplus > 201402L #if __has_include() #include namespace crc32c { // Computes the CRC32C of the bytes in the string_view. inline uint32_t Crc32c(const std::string_view& string_view) { return Crc32c(reinterpret_cast(string_view.data()), string_view.size()); } } // namespace crc32c #endif // __has_include() #endif // __cplusplus > 201402L #endif /* defined(__cplusplus) */ #endif // CRC32C_CRC32C_H_ digest/src/aes.h0000644000176200001440000000104512247504516013247 0ustar liggesusers#ifndef _AES_H #define _AES_H #ifndef uint8 #define uint8 unsigned char #endif #ifndef uint32 #define uint32 unsigned long int #endif typedef struct { uint32 erk[64]; /* encryption round keys */ uint32 drk[64]; /* decryption round keys */ int nr; /* number of rounds */ } aes_context; int aes_set_key( aes_context *ctx, uint8 *key, int nbits ); void aes_encrypt( aes_context *ctx, uint8 input[16], uint8 output[16] ); void aes_decrypt( aes_context *ctx, uint8 input[16], uint8 output[16] ); #endif /* aes.h */ digest/src/Makevars0000644000176200001440000000002414446063403014014 0ustar liggesusers PKG_CPPFLAGS = -I. digest/src/xxhash.c0000644000176200001440000000347714573367376014027 0ustar liggesusers/* * xxHash - Extremely Fast Hash algorithm * Copyright (C) 2012-2023 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at: * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ /* * xxhash.c instantiates functions defined in xxhash.h */ #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ #define XXH_IMPLEMENTATION /* access definitions */ #include "xxhash.h" digest/src/blake3_impl.h0000755000176200001440000002220514635552730014671 0ustar liggesusers/* blake3 license: * The C code is copyright Samuel Neves and Jack O'Connor, 2019-2020. * This work is released into the public domain with CC0 1.0. * Alternatively, it is licensed under the Apache License 2.0. * For full license see * https://raw.githubusercontent.com/BLAKE3-team/BLAKE3/master/LICENSE */ #ifndef BLAKE3_IMPL_H #define BLAKE3_IMPL_H #include #include #include #include #include #include "blake3.h" // internal flags enum blake3_flags { CHUNK_START = 1 << 0, CHUNK_END = 1 << 1, PARENT = 1 << 2, ROOT = 1 << 3, KEYED_HASH = 1 << 4, DERIVE_KEY_CONTEXT = 1 << 5, DERIVE_KEY_MATERIAL = 1 << 6, }; // This C implementation tries to support recent versions of GCC, Clang, and // MSVC. #if defined(_MSC_VER) #define INLINE static __forceinline #else #define INLINE static inline __attribute__((always_inline)) #endif #if defined(__x86_64__) || defined(_M_X64) #define IS_X86 #define IS_X86_64 #endif #if defined(__i386__) || defined(_M_IX86) #define IS_X86 #define IS_X86_32 #endif #if defined(IS_X86) #if defined(_MSC_VER) #include #endif #include #endif #if defined(IS_X86) #define MAX_SIMD_DEGREE 16 #elif defined(BLAKE3_USE_NEON) #define MAX_SIMD_DEGREE 4 #else #define MAX_SIMD_DEGREE 1 #endif // There are some places where we want a static size that's equal to the // MAX_SIMD_DEGREE, but also at least 2. #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL}; static const uint8_t MSG_SCHEDULE[7][16] = { {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, }; /* Find index of the highest set bit */ /* x is assumed to be nonzero. */ static unsigned int highest_one(uint64_t x) { #if defined(__GNUC__) || defined(__clang__) return 63 ^ __builtin_clzll(x); #elif defined(_MSC_VER) && defined(IS_X86_64) unsigned long index; _BitScanReverse64(&index, x); return index; #elif defined(_MSC_VER) && defined(IS_X86_32) if(x >> 32) { unsigned long index; _BitScanReverse(&index, x >> 32); return 32 + index; } else { unsigned long index; _BitScanReverse(&index, x); return index; } #else unsigned int c = 0; if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } if(x & 0x0000000000000002ULL) { c += 1; } return c; #endif } // Count the number of 1 bits. INLINE unsigned int popcnt(uint64_t x) { #if defined(__GNUC__) || defined(__clang__) return __builtin_popcountll(x); #else unsigned int count = 0; while (x != 0) { count += 1; x &= x - 1; } return count; #endif } // Largest power of two less than or equal to x. As a special case, returns 1 // when x is 0. INLINE uint64_t round_down_to_power_of_2(uint64_t x) { return 1ULL << highest_one(x | 1); } INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } INLINE uint32_t counter_high(uint64_t counter) { return (uint32_t)(counter >> 32); } INLINE uint32_t load32(const void *src) { const uint8_t *p = (const uint8_t *)src; return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); } INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8]) { key_words[0] = load32(&key[0 * 4]); key_words[1] = load32(&key[1 * 4]); key_words[2] = load32(&key[2 * 4]); key_words[3] = load32(&key[3 * 4]); key_words[4] = load32(&key[4 * 4]); key_words[5] = load32(&key[5 * 4]); key_words[6] = load32(&key[6 * 4]); key_words[7] = load32(&key[7 * 4]); } INLINE void store32(void *dst, uint32_t w) { uint8_t *p = (uint8_t *)dst; p[0] = (uint8_t)(w >> 0); p[1] = (uint8_t)(w >> 8); p[2] = (uint8_t)(w >> 16); p[3] = (uint8_t)(w >> 24); } INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { store32(&bytes_out[0 * 4], cv_words[0]); store32(&bytes_out[1 * 4], cv_words[1]); store32(&bytes_out[2 * 4], cv_words[2]); store32(&bytes_out[3 * 4], cv_words[3]); store32(&bytes_out[4 * 4], cv_words[4]); store32(&bytes_out[5 * 4], cv_words[5]); store32(&bytes_out[6 * 4], cv_words[6]); store32(&bytes_out[7 * 4], cv_words[7]); } void blake3_compress_in_place(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags); void blake3_compress_xof(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); size_t blake3_simd_degree(void); // Declarations for implementation-specific functions. void blake3_compress_in_place_portable(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags); void blake3_compress_xof_portable(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); #if defined(IS_X86) #if !defined(BLAKE3_NO_SSE41) void blake3_compress_in_place_sse41(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags); void blake3_compress_xof_sse41(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); #endif #if !defined(BLAKE3_NO_AVX2) void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); #endif #if !defined(BLAKE3_NO_AVX512) void blake3_compress_in_place_avx512(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags); void blake3_compress_xof_avx512(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]); void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); #endif #endif #if defined(BLAKE3_USE_NEON) void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out); #endif #endif /* BLAKE3_IMPL_H */ digest/src/crc32.c0000644000176200001440000002171514323254402013405 0ustar liggesusers/* crc32.c -- compute the CRC-32 of a data stream * Copyright (C) 1995-2003 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster * CRC methods: exclusive-oring 32 bits of data at a time, and pre-computing * tables for updating the shift register in one step with three exclusive-ors * instead of four steps with four exclusive-ors. This results about a factor * of two increase in speed on a Power PC G4 (PPC7455) using gcc -O3. */ /* @(#) $Id$ */ /* Note on the use of DYNAMIC_CRC_TABLE: there is no mutex or semaphore protection on the static variables used to control the first-use generation of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should first call get_crc_table() to initialize the tables before allowing more than one thread to use crc32(). */ #include "zutil.h" /* for STDC and FAR definitions */ /* #include "zlib.h" */ /* for STDC and FAR definitions */ #define local static /* Find a four-byte integer type for crc32_little() and crc32_big(). */ #ifndef NOBYFOUR # ifdef STDC /* need ANSI C limits.h to determine sizes */ # include # define BYFOUR # if (UINT_MAX == 0xffffffffUL) typedef unsigned int u4; # else # if (ULONG_MAX == 0xffffffffUL) typedef unsigned long u4; # else # if (USHRT_MAX == 0xffffffffUL) typedef unsigned short u4; # else # undef BYFOUR /* can't find a four-byte integer type! */ # endif # endif # endif # endif /* STDC */ #endif /* !NOBYFOUR */ /* Definitions for doing the crc four data bytes at a time. */ #ifdef BYFOUR # define REV(w) (((w)>>24)+(((w)>>8)&0xff00)+ \ (((w)&0xff00)<<8)+(((w)&0xff)<<24)) local unsigned long digest_crc32_little OF((unsigned long, const unsigned char FAR *, unsigned)); local unsigned long digest_crc32_big OF((unsigned long, const unsigned char FAR *, unsigned)); # define TBLS 8 #else # define TBLS 1 #endif /* BYFOUR */ #ifdef DYNAMIC_CRC_TABLE local volatile int crc_table_empty = 1; local unsigned long FAR crc_table[TBLS][256]; local void digest_make_crc_table OF((void)); /* Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. Polynomials over GF(2) are represented in binary, one bit per coefficient, with the lowest powers in the most significant bit. Then adding polynomials is just exclusive-or, and multiplying a polynomial by x is a right shift by one. If we call the above polynomial p, and represent a byte as the polynomial q, also with the lowest power in the most significant bit (so the byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p, where a mod b means the remainder after dividing a by b. This calculation is done using the shift-register method of multiplying and taking the remainder. The register is initialized to zero, and for each incoming bit, x^32 is added mod p to the register if the bit is a one (where x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by x (which is shifting right by one and adding x^32 mod p if the bit shifted out is a one). We start with the highest power (least significant bit) of q and repeat for all eight bits of q. The first table is simply the CRC of all possible eight bit values. This is all the information needed to generate CRCs on data a byte at a time for all combinations of CRC register values and incoming bytes. The remaining tables allow for word-at-a-time CRC calculation for both big-endian and little- endian machines, where a word is four bytes. */ local void digest_make_crc_table() { unsigned long c; int n, k; unsigned long poly; /* polynomial exclusive-or pattern */ /* terms of polynomial defining this crc (except x^32): */ static volatile int first = 1; /* flag to limit concurrent making */ static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; /* See if another task is already doing this (not thread-safe, but better than nothing -- significantly reduces duration of vulnerability in case the advice about DYNAMIC_CRC_TABLE is ignored) */ if (first) { first = 0; /* make exclusive-or pattern from polynomial (0xedb88320UL) */ poly = 0UL; for (n = 0; n < sizeof(p)/sizeof(unsigned char); n++) poly |= 1UL << (31 - p[n]); /* generate a crc for every 8-bit value */ for (n = 0; n < 256; n++) { c = (unsigned long)n; for (k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >> 1) : c >> 1; crc_table[0][n] = c; } #ifdef BYFOUR /* generate crc for each value followed by one, two, and three zeros, and then the byte reversal of those as well as the first table */ for (n = 0; n < 256; n++) { c = crc_table[0][n]; crc_table[4][n] = REV(c); for (k = 1; k < 4; k++) { c = crc_table[0][c & 0xff] ^ (c >> 8); crc_table[k][n] = c; crc_table[k + 4][n] = REV(c); } } #endif /* BYFOUR */ crc_table_empty = 0; } else { /* not first */ /* wait for the other guy to finish (not efficient, but rare) */ while (crc_table_empty) ; } } #else /* !DYNAMIC_CRC_TABLE */ /* ======================================================================== * Tables of CRC-32s of all single-byte values, made by make_crc_table(). */ #include "crc32.h" #endif /* DYNAMIC_CRC_TABLE */ /* ========================================================================= */ #define DO1 crc = crc_table[0][((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8) #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ unsigned long ZEXPORT digest_crc32(unsigned long crc, const unsigned char FAR *buf, unsigned len) { if (buf == Z_NULL) return 0UL; #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) digest_make_crc_table(); #endif /* DYNAMIC_CRC_TABLE */ #ifdef BYFOUR if (sizeof(void *) == sizeof(ptrdiff_t)) { u4 endian; endian = 1; if (*((unsigned char *)(&endian))) return digest_crc32_little(crc, buf, len); else return digest_crc32_big(crc, buf, len); /* #nocov */ } #endif /* BYFOUR */ crc = crc ^ 0xffffffffUL; while (len >= 8) { DO8; len -= 8; } if (len) do { DO1; } while (--len); return crc ^ 0xffffffffUL; } #ifdef BYFOUR /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24] #define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4 /* ========================================================================= */ local unsigned long digest_crc32_little(unsigned long crc, const unsigned char FAR *buf, unsigned len) { register u4 c; register const u4 FAR *buf4; c = (u4)crc; c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); /* #nocov */ len--; /* #nocov */ } buf4 = (const u4 FAR *)buf; while (len >= 32) { DOLIT32; len -= 32; } while (len >= 4) { DOLIT4; len -= 4; } buf = (const unsigned char FAR *)buf4; if (len) do { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); } while (--len); c = ~c; return (unsigned long)c; } /* ========================================================================= */ #define DOBIG4 c ^= *++buf4; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 /* ========================================================================= */ local unsigned long digest_crc32_big(unsigned long crc, const unsigned char FAR *buf, unsigned len) { /* #nocov start */ register u4 c; register const u4 FAR *buf4; c = REV((u4)crc); c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); len--; } buf4 = (const u4 FAR *)buf; buf4--; while (len >= 32) { DOBIG32; len -= 32; } while (len >= 4) { DOBIG4; len -= 4; } buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); } while (--len); c = ~c; return (unsigned long)(REV(c)); } /* #nocov end */ #endif /* BYFOUR */ digest/src/sha1.h0000644000176200001440000000064712247504516013342 0ustar liggesusers#ifndef _SHA1_H #define _SHA1_H #ifndef uint8 #define uint8 unsigned char #endif #ifndef uint32 #define uint32 unsigned long int #endif typedef struct { uint32 total[2]; uint32 state[5]; uint8 buffer[64]; } sha1_context; void sha1_starts( sha1_context *ctx ); void sha1_update( sha1_context *ctx, uint8 *input, uint32 length ); void sha1_finish( sha1_context *ctx, uint8 digest[20] ); #endif /* sha1.h */ digest/src/zlib.h0000644000176200001440000001733312247504516013446 0ustar liggesusers/* zlib.h -- interface of the 'zlib' general purpose compression library version 1.2.2, October 3rd, 2004 Copyright (C) 1995-2004 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format). */ #ifndef ZLIB_H #define ZLIB_H #include "zconf.h" #ifdef __cplusplus extern "C" { #endif #define ZLIB_VERSION "1.2.2" #define ZLIB_VERNUM 0x1220 /* The 'zlib' compression library provides in-memory compression and decompression functions, including integrity checks of the uncompressed data. This version of the library supports only one compression method (deflation) but other algorithms will be added later and will have the same stream interface. Compression can be done in a single step if the buffers are large enough (for example if an input file is mmap'ed), or can be done by repeated calls of the compression function. In the latter case, the application must provide more input and/or consume the output (providing more output space) before each call. The compressed data format used by default by the in-memory functions is the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped around a deflate stream, which is itself documented in RFC 1951. The library also supports reading and writing files in gzip (.gz) format with an interface similar to that of stdio using the functions that start with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. This library can optionally read and write gzip streams in memory as well. The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- file compression on file systems, has a larger header than zlib to maintain directory information, and uses a different, slower check method than zlib. The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash even in case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); typedef void (*free_func) OF((voidpf opaque, voidpf address)); struct internal_state; typedef struct z_stream_s { Bytef *next_in; /* next input byte */ uInt avail_in; /* number of bytes available at next_in */ uLong total_in; /* total nb of input bytes read so far */ Bytef *next_out; /* next output byte should be put there */ uInt avail_out; /* remaining free space at next_out */ uLong total_out; /* total nb of bytes output so far */ char *msg; /* last error message, NULL if no error */ struct internal_state FAR *state; /* not visible by applications */ alloc_func zalloc; /* used to allocate the internal state */ free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ int data_type; /* best guess about the data type: ascii or binary */ uLong adler; /* adler32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ } z_stream; typedef z_stream FAR *z_streamp; /* The application must update next_in and avail_in when avail_in has dropped to zero. It must update next_out and avail_out when avail_out has dropped to zero. The application must initialize zalloc, zfree and opaque before calling the init function. All other fields are set by the compression library and must not be updated by the application. The opaque value provided by the application will be passed as the first parameter for calls of zalloc and zfree. This can be useful for custom memory management. The compression library attaches no meaning to the opaque value. zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be thread safe. On 16-bit systems, the functions zalloc and zfree must be able to allocate exactly 65536 bytes, but will not be required to allocate more than this if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, pointers returned by zalloc for objects of exactly 65536 bytes *must* have their offset normalized to zero. The default allocation function provided by this library ensures this (see zutil.c). To reduce memory requirements and avoid any allocation of 64K objects, at the expense of compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h). The fields total_in and total_out can be used for statistics or progress reports. After compression, total_in holds the total size of the uncompressed data and may be saved for use in the decompressor (particularly if the decompressor wants to decompress everything in a single step). */ /* constants */ #define Z_NO_FLUSH 0 #define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */ #define Z_SYNC_FLUSH 2 #define Z_FULL_FLUSH 3 #define Z_FINISH 4 #define Z_BLOCK 5 /* Allowed flush values; see deflate() and inflate() below for details */ #define Z_OK 0 #define Z_STREAM_END 1 #define Z_NEED_DICT 2 #define Z_ERRNO (-1) #define Z_STREAM_ERROR (-2) #define Z_DATA_ERROR (-3) #define Z_MEM_ERROR (-4) #define Z_BUF_ERROR (-5) #define Z_VERSION_ERROR (-6) /* Return codes for the compression/decompression functions. Negative * values are errors, positive values are used for special but normal events. */ #define Z_NO_COMPRESSION 0 #define Z_BEST_SPEED 1 #define Z_BEST_COMPRESSION 9 #define Z_DEFAULT_COMPRESSION (-1) /* compression levels */ #define Z_FILTERED 1 #define Z_HUFFMAN_ONLY 2 #define Z_RLE 3 #define Z_DEFAULT_STRATEGY 0 /* compression strategy; see deflateInit2() below for details */ #define Z_BINARY 0 #define Z_ASCII 1 #define Z_UNKNOWN 2 /* Possible values of the data_type field (though see inflate()) */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ #define zlib_version zlibVersion() /* for compatibility with versions < 1.0.2 */ #if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) struct internal_state {int dummy;}; /* hack for buggy compilers */ #endif ZEXTERN const char * ZEXPORT zError OF((int)); ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp z)); ZEXTERN const uLongf * ZEXPORT get_crc_table OF((void)); #ifdef __cplusplus } #endif #endif /* ZLIB_H */ digest/src/sha1.c0000644000176200001440000002150613216563247013335 0ustar liggesusers/* * FIPS-180-1 compliant SHA-1 implementation, * by Christophe Devine ; * this program is licensed under the GPL. */ #include #include "sha1.h" #define GET_UINT32(n,b,i) \ { \ (n) = ( (uint32) (b)[(i) ] << 24 ) \ | ( (uint32) (b)[(i) + 1] << 16 ) \ | ( (uint32) (b)[(i) + 2] << 8 ) \ | ( (uint32) (b)[(i) + 3] ); \ } #define PUT_UINT32(n,b,i) \ { \ (b)[(i) ] = (uint8) ( (n) >> 24 ); \ (b)[(i) + 1] = (uint8) ( (n) >> 16 ); \ (b)[(i) + 2] = (uint8) ( (n) >> 8 ); \ (b)[(i) + 3] = (uint8) ( (n) ); \ } void sha1_starts( sha1_context *ctx ) { ctx->total[0] = 0; ctx->total[1] = 0; ctx->state[0] = 0x67452301; ctx->state[1] = 0xEFCDAB89; ctx->state[2] = 0x98BADCFE; ctx->state[3] = 0x10325476; ctx->state[4] = 0xC3D2E1F0; } void sha1_process( sha1_context *ctx, uint8 data[64] ) { uint32 temp, W[16], A, B, C, D, E; GET_UINT32( W[0], data, 0 ); GET_UINT32( W[1], data, 4 ); GET_UINT32( W[2], data, 8 ); GET_UINT32( W[3], data, 12 ); GET_UINT32( W[4], data, 16 ); GET_UINT32( W[5], data, 20 ); GET_UINT32( W[6], data, 24 ); GET_UINT32( W[7], data, 28 ); GET_UINT32( W[8], data, 32 ); GET_UINT32( W[9], data, 36 ); GET_UINT32( W[10], data, 40 ); GET_UINT32( W[11], data, 44 ); GET_UINT32( W[12], data, 48 ); GET_UINT32( W[13], data, 52 ); GET_UINT32( W[14], data, 56 ); GET_UINT32( W[15], data, 60 ); #define S(x,n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) #define R(t) \ ( \ temp = W[(t - 3) & 0x0F] ^ W[(t - 8) & 0x0F] ^ \ W[(t - 14) & 0x0F] ^ W[ t & 0x0F], \ ( W[t & 0x0F] = S(temp,1) ) \ ) #define P(a,b,c,d,e,x) \ { \ e += S(a,5) + F(b,c,d) + K + x; b = S(b,30); \ } A = ctx->state[0]; B = ctx->state[1]; C = ctx->state[2]; D = ctx->state[3]; E = ctx->state[4]; #define F(x,y,z) (z ^ (x & (y ^ z))) #define K 0x5A827999 P( A, B, C, D, E, W[0] ); P( E, A, B, C, D, W[1] ); P( D, E, A, B, C, W[2] ); P( C, D, E, A, B, W[3] ); P( B, C, D, E, A, W[4] ); P( A, B, C, D, E, W[5] ); P( E, A, B, C, D, W[6] ); P( D, E, A, B, C, W[7] ); P( C, D, E, A, B, W[8] ); P( B, C, D, E, A, W[9] ); P( A, B, C, D, E, W[10] ); P( E, A, B, C, D, W[11] ); P( D, E, A, B, C, W[12] ); P( C, D, E, A, B, W[13] ); P( B, C, D, E, A, W[14] ); P( A, B, C, D, E, W[15] ); P( E, A, B, C, D, R(16) ); P( D, E, A, B, C, R(17) ); P( C, D, E, A, B, R(18) ); P( B, C, D, E, A, R(19) ); #undef K #undef F #define F(x,y,z) (x ^ y ^ z) #define K 0x6ED9EBA1 P( A, B, C, D, E, R(20) ); P( E, A, B, C, D, R(21) ); P( D, E, A, B, C, R(22) ); P( C, D, E, A, B, R(23) ); P( B, C, D, E, A, R(24) ); P( A, B, C, D, E, R(25) ); P( E, A, B, C, D, R(26) ); P( D, E, A, B, C, R(27) ); P( C, D, E, A, B, R(28) ); P( B, C, D, E, A, R(29) ); P( A, B, C, D, E, R(30) ); P( E, A, B, C, D, R(31) ); P( D, E, A, B, C, R(32) ); P( C, D, E, A, B, R(33) ); P( B, C, D, E, A, R(34) ); P( A, B, C, D, E, R(35) ); P( E, A, B, C, D, R(36) ); P( D, E, A, B, C, R(37) ); P( C, D, E, A, B, R(38) ); P( B, C, D, E, A, R(39) ); #undef K #undef F #define F(x,y,z) ((x & y) | (z & (x | y))) #define K 0x8F1BBCDC P( A, B, C, D, E, R(40) ); P( E, A, B, C, D, R(41) ); P( D, E, A, B, C, R(42) ); P( C, D, E, A, B, R(43) ); P( B, C, D, E, A, R(44) ); P( A, B, C, D, E, R(45) ); P( E, A, B, C, D, R(46) ); P( D, E, A, B, C, R(47) ); P( C, D, E, A, B, R(48) ); P( B, C, D, E, A, R(49) ); P( A, B, C, D, E, R(50) ); P( E, A, B, C, D, R(51) ); P( D, E, A, B, C, R(52) ); P( C, D, E, A, B, R(53) ); P( B, C, D, E, A, R(54) ); P( A, B, C, D, E, R(55) ); P( E, A, B, C, D, R(56) ); P( D, E, A, B, C, R(57) ); P( C, D, E, A, B, R(58) ); P( B, C, D, E, A, R(59) ); #undef K #undef F #define F(x,y,z) (x ^ y ^ z) #define K 0xCA62C1D6 P( A, B, C, D, E, R(60) ); P( E, A, B, C, D, R(61) ); P( D, E, A, B, C, R(62) ); P( C, D, E, A, B, R(63) ); P( B, C, D, E, A, R(64) ); P( A, B, C, D, E, R(65) ); P( E, A, B, C, D, R(66) ); P( D, E, A, B, C, R(67) ); P( C, D, E, A, B, R(68) ); P( B, C, D, E, A, R(69) ); P( A, B, C, D, E, R(70) ); P( E, A, B, C, D, R(71) ); P( D, E, A, B, C, R(72) ); P( C, D, E, A, B, R(73) ); P( B, C, D, E, A, R(74) ); P( A, B, C, D, E, R(75) ); P( E, A, B, C, D, R(76) ); P( D, E, A, B, C, R(77) ); P( C, D, E, A, B, R(78) ); P( B, C, D, E, A, R(79) ); #undef K #undef F ctx->state[0] += A; ctx->state[1] += B; ctx->state[2] += C; ctx->state[3] += D; ctx->state[4] += E; } void sha1_update( sha1_context *ctx, uint8 *input, uint32 length ) { uint32 left, fill; if( ! length ) return; left = ctx->total[0] & 0x3F; fill = 64 - left; ctx->total[0] += length; ctx->total[0] &= 0xFFFFFFFF; if( ctx->total[0] < length ) ctx->total[1]++; /* #nocov */ if( left && length >= fill ) { memcpy( (void *) (ctx->buffer + left), (void *) input, fill ); sha1_process( ctx, ctx->buffer ); length -= fill; input += fill; left = 0; } while( length >= 64 ) { sha1_process( ctx, input ); length -= 64; input += 64; } if( length ) { memcpy( (void *) (ctx->buffer + left), (void *) input, length ); } } static uint8 sha1_padding[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; void sha1_finish( sha1_context *ctx, uint8 digest[20] ) { uint32 last, padn; uint32 high, low; uint8 msglen[8]; high = ( ctx->total[0] >> 29 ) | ( ctx->total[1] << 3 ); low = ( ctx->total[0] << 3 ); PUT_UINT32( high, msglen, 0 ); PUT_UINT32( low, msglen, 4 ); last = ctx->total[0] & 0x3F; padn = ( last < 56 ) ? ( 56 - last ) : ( 120 - last ); sha1_update( ctx, sha1_padding, padn ); sha1_update( ctx, msglen, 8 ); PUT_UINT32( ctx->state[0], digest, 0 ); PUT_UINT32( ctx->state[1], digest, 4 ); PUT_UINT32( ctx->state[2], digest, 8 ); PUT_UINT32( ctx->state[3], digest, 12 ); PUT_UINT32( ctx->state[4], digest, 16 ); } #ifdef TEST #include #include /* * those are the standard FIPS-180-1 test vectors */ static char *msg[] = { "abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", NULL }; static char *val[] = { "a9993e364706816aba3e25717850c26c9cd0d89d", "84983e441c3bd26ebaae4aa1f95129e5e54670f1", "34aa973cd4c4daa4f61eeb2bdbad27316534016f" }; int main( int argc, char *argv[] ) { FILE *f; int i, j; char output[41]; sha1_context ctx; unsigned char buf[1000]; unsigned char sha1sum[20]; if( argc < 2 ) { printf( "\n SHA-1 Validation Tests:\n\n" ); for( i = 0; i < 3; i++ ) { printf( " Test %d ", i + 1 ); sha1_starts( &ctx ); if( i < 2 ) { sha1_update( &ctx, (uint8 *) msg[i], strlen( msg[i] ) ); } else { memset( buf, 'a', 1000 ); for( j = 0; j < 1000; j++ ) { sha1_update( &ctx, (uint8 *) buf, 1000 ); } } sha1_finish( &ctx, sha1sum ); for( j = 0; j < 20; j++ ) { sprintf( output + j * 2, "%02x", sha1sum[j] ); } if( memcmp( output, val[i], 40 ) ) { printf( "failed!\n" ); return( 1 ); } printf( "passed.\n" ); } printf( "\n" ); } else { if( ! ( f = fopen( argv[1], "rb" ) ) ) { perror( "fopen" ); return( 1 ); } sha1_starts( &ctx ); while( ( i = fread( buf, 1, sizeof( buf ), f ) ) > 0 ) { sha1_update( &ctx, buf, i ); } sha1_finish( &ctx, sha1sum ); for( j = 0; j < 20; j++ ) { printf( "%02x", sha1sum[j] ); } printf( " %s\n", argv[1] ); } return( 0 ); } #endif digest/src/xxhash.h0000644000176200001440000100267414573367376014033 0ustar liggesusers/* * xxHash - Extremely Fast Hash algorithm * Header File * Copyright (C) 2012-2023 Yann Collet * * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * You can contact the author at: * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ /*! * @mainpage xxHash * * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed * limits. * * It is proposed in four flavors, in three families: * 1. @ref XXH32_family * - Classic 32-bit hash function. Simple, compact, and runs on almost all * 32-bit and 64-bit systems. * 2. @ref XXH64_family * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most * 64-bit systems (but _not_ 32-bit systems). * 3. @ref XXH3_family * - Modern 64-bit and 128-bit hash function family which features improved * strength and performance across the board, especially on smaller data. * It benefits greatly from SIMD and 64-bit without requiring it. * * Benchmarks * --- * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. * The open source benchmark program is compiled with clang v10.0 using -O3 flag. * * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | * | -------------------- | ------- | ----: | ---------------: | ------------------: | * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | * | RAM sequential read | | N/A | 28.0 GB/s | N/A | * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | * | City64 | | 64 | 22.0 GB/s | 76.6 | * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | * | City128 | | 128 | 21.7 GB/s | 57.7 | * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | * | XXH64() | | 64 | 19.4 GB/s | 71.0 | * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | * | Mum | | 64 | 18.0 GB/s | 67.0 | * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | * | XXH32() | | 32 | 9.7 GB/s | 71.9 | * | City32 | | 32 | 9.1 GB/s | 66.0 | * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | * | SipHash* | | 64 | 3.0 GB/s | 43.2 | * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | * | FNV64 | | 64 | 1.2 GB/s | 62.7 | * | Blake2* | | 256 | 1.1 GB/s | 5.1 | * | SHA1* | | 160 | 0.8 GB/s | 5.6 | * | MD5* | | 128 | 0.6 GB/s | 7.8 | * @note * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, * even though it is mandatory on x64. * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic * by modern standards. * - Small data velocity is a rough average of algorithm's efficiency for small * data. For more accurate information, see the wiki. * - More benchmarks and strength tests are found on the wiki: * https://github.com/Cyan4973/xxHash/wiki * * Usage * ------ * All xxHash variants use a similar API. Changing the algorithm is a trivial * substitution. * * @pre * For functions which take an input and length parameter, the following * requirements are assumed: * - The range from [`input`, `input + length`) is valid, readable memory. * - The only exception is if the `length` is `0`, `input` may be `NULL`. * - For C++, the objects must have the *TriviallyCopyable* property, as the * functions access bytes directly as if it was an array of `unsigned char`. * * @anchor single_shot_example * **Single Shot** * * These functions are stateless functions which hash a contiguous block of memory, * immediately returning the result. They are the easiest and usually the fastest * option. * * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() * * @code{.c} * #include * #include "xxhash.h" * * // Example for a function which hashes a null terminated string with XXH32(). * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) * { * // NULL pointers are only valid if the length is zero * size_t length = (string == NULL) ? 0 : strlen(string); * return XXH32(string, length, seed); * } * @endcode * * * @anchor streaming_example * **Streaming** * * These groups of functions allow incremental hashing of unknown size, even * more than what would fit in a size_t. * * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() * * @code{.c} * #include * #include * #include "xxhash.h" * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). * XXH64_hash_t hashFile(FILE* f) * { * // Allocate a state struct. Do not just use malloc() or new. * XXH3_state_t* state = XXH3_createState(); * assert(state != NULL && "Out of memory!"); * // Reset the state to start a new hashing session. * XXH3_64bits_reset(state); * char buffer[4096]; * size_t count; * // Read the file in chunks * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { * // Run update() as many times as necessary to process the data * XXH3_64bits_update(state, buffer, count); * } * // Retrieve the finalized hash. This will not change the state. * XXH64_hash_t result = XXH3_64bits_digest(state); * // Free the state. Do not use free(). * XXH3_freeState(state); * return result; * } * @endcode * * Streaming functions generate the xxHash value from an incremental input. * This method is slower than single-call functions, due to state management. * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. * * An XXH state must first be allocated using `XXH*_createState()`. * * Start a new hash by initializing the state with a seed using `XXH*_reset()`. * * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. * * The function returns an error code, with 0 meaning OK, and any other value * meaning there is an error. * * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. * This function returns the nn-bits hash as an int or long long. * * It's still possible to continue inserting input into the hash state after a * digest, and generate new hash values later on by invoking `XXH*_digest()`. * * When done, release the state using `XXH*_freeState()`. * * * @anchor canonical_representation_example * **Canonical Representation** * * The default return values from XXH functions are unsigned 32, 64 and 128 bit * integers. * This the simplest and fastest format for further post-processing. * * However, this leaves open the question of what is the order on the byte level, * since little and big endian conventions will store the same number differently. * * The canonical representation settles this issue by mandating big-endian * convention, the same convention as human-readable numbers (large digits first). * * When writing hash values to storage, sending them over a network, or printing * them, it's highly recommended to use the canonical representation to ensure * portability across a wider range of systems, present and future. * * The following functions allow transformation of hash values to and from * canonical format. * * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), * * @code{.c} * #include * #include "xxhash.h" * * // Example for a function which prints XXH32_hash_t in human readable format * void printXxh32(XXH32_hash_t hash) * { * XXH32_canonical_t cano; * XXH32_canonicalFromHash(&cano, hash); * size_t i; * for(i = 0; i < sizeof(cano.digest); ++i) { * printf("%02x", cano.digest[i]); * } * printf("\n"); * } * * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) * { * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); * return hash; * } * @endcode * * * @file xxhash.h * xxHash prototypes and implementation */ #if defined (__cplusplus) extern "C" { #endif /* **************************** * INLINE mode ******************************/ /*! * @defgroup public Public API * Contains details on the public xxHash functions. * @{ */ #ifdef XXH_DOXYGEN /*! * @brief Gives access to internal state declaration, required for static allocation. * * Incompatible with dynamic linking, due to risks of ABI changes. * * Usage: * @code{.c} * #define XXH_STATIC_LINKING_ONLY * #include "xxhash.h" * @endcode */ # define XXH_STATIC_LINKING_ONLY /* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ /*! * @brief Gives access to internal definitions. * * Usage: * @code{.c} * #define XXH_STATIC_LINKING_ONLY * #define XXH_IMPLEMENTATION * #include "xxhash.h" * @endcode */ # define XXH_IMPLEMENTATION /* Do not undef XXH_IMPLEMENTATION for Doxygen */ /*! * @brief Exposes the implementation and marks all functions as `inline`. * * Use these build macros to inline xxhash into the target unit. * Inlining improves performance on small inputs, especially when the length is * expressed as a compile-time constant: * * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html * * It also keeps xxHash symbols private to the unit, so they are not exported. * * Usage: * @code{.c} * #define XXH_INLINE_ALL * #include "xxhash.h" * @endcode * Do not compile and link xxhash.o as a separate object, as it is not useful. */ # define XXH_INLINE_ALL # undef XXH_INLINE_ALL /*! * @brief Exposes the implementation without marking functions as inline. */ # define XXH_PRIVATE_API # undef XXH_PRIVATE_API /*! * @brief Emulate a namespace by transparently prefixing all symbols. * * If you want to include _and expose_ xxHash functions from within your own * library, but also want to avoid symbol collisions with other libraries which * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE * (therefore, avoid empty or numeric values). * * Note that no change is required within the calling program as long as it * includes `xxhash.h`: Regular symbol names will be automatically translated * by this header. */ # define XXH_NAMESPACE /* YOUR NAME HERE */ # undef XXH_NAMESPACE #endif #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ && !defined(XXH_INLINE_ALL_31684351384) /* this section should be traversed only once */ # define XXH_INLINE_ALL_31684351384 /* give access to the advanced API, required to compile implementations */ # undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ # define XXH_STATIC_LINKING_ONLY /* make all functions private */ # undef XXH_PUBLIC_API # if defined(__GNUC__) # define XXH_PUBLIC_API static __inline __attribute__((unused)) # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # define XXH_PUBLIC_API static inline # elif defined(_MSC_VER) # define XXH_PUBLIC_API static __inline # else /* note: this version may generate warnings for unused static functions */ # define XXH_PUBLIC_API static # endif /* * This part deals with the special case where a unit wants to inline xxHash, * but "xxhash.h" has previously been included without XXH_INLINE_ALL, * such as part of some previously included *.h header file. * Without further action, the new include would just be ignored, * and functions would effectively _not_ be inlined (silent failure). * The following macros solve this situation by prefixing all inlined names, * avoiding naming collision with previous inclusions. */ /* Before that, we unconditionally #undef all symbols, * in case they were already defined with XXH_NAMESPACE. * They will then be redefined for XXH_INLINE_ALL */ # undef XXH_versionNumber /* XXH32 */ # undef XXH32 # undef XXH32_createState # undef XXH32_freeState # undef XXH32_reset # undef XXH32_update # undef XXH32_digest # undef XXH32_copyState # undef XXH32_canonicalFromHash # undef XXH32_hashFromCanonical /* XXH64 */ # undef XXH64 # undef XXH64_createState # undef XXH64_freeState # undef XXH64_reset # undef XXH64_update # undef XXH64_digest # undef XXH64_copyState # undef XXH64_canonicalFromHash # undef XXH64_hashFromCanonical /* XXH3_64bits */ # undef XXH3_64bits # undef XXH3_64bits_withSecret # undef XXH3_64bits_withSeed # undef XXH3_64bits_withSecretandSeed # undef XXH3_createState # undef XXH3_freeState # undef XXH3_copyState # undef XXH3_64bits_reset # undef XXH3_64bits_reset_withSeed # undef XXH3_64bits_reset_withSecret # undef XXH3_64bits_update # undef XXH3_64bits_digest # undef XXH3_generateSecret /* XXH3_128bits */ # undef XXH128 # undef XXH3_128bits # undef XXH3_128bits_withSeed # undef XXH3_128bits_withSecret # undef XXH3_128bits_reset # undef XXH3_128bits_reset_withSeed # undef XXH3_128bits_reset_withSecret # undef XXH3_128bits_reset_withSecretandSeed # undef XXH3_128bits_update # undef XXH3_128bits_digest # undef XXH128_isEqual # undef XXH128_cmp # undef XXH128_canonicalFromHash # undef XXH128_hashFromCanonical /* Finally, free the namespace itself */ # undef XXH_NAMESPACE /* employ the namespace for XXH_INLINE_ALL */ # define XXH_NAMESPACE XXH_INLINE_ /* * Some identifiers (enums, type names) are not symbols, * but they must nonetheless be renamed to avoid redeclaration. * Alternative solution: do not redeclare them. * However, this requires some #ifdefs, and has a more dispersed impact. * Meanwhile, renaming can be achieved in a single place. */ # define XXH_IPREF(Id) XXH_NAMESPACE ## Id # define XXH_OK XXH_IPREF(XXH_OK) # define XXH_ERROR XXH_IPREF(XXH_ERROR) # define XXH_errorcode XXH_IPREF(XXH_errorcode) # define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) # define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) # define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) # define XXH32_state_s XXH_IPREF(XXH32_state_s) # define XXH32_state_t XXH_IPREF(XXH32_state_t) # define XXH64_state_s XXH_IPREF(XXH64_state_s) # define XXH64_state_t XXH_IPREF(XXH64_state_t) # define XXH3_state_s XXH_IPREF(XXH3_state_s) # define XXH3_state_t XXH_IPREF(XXH3_state_t) # define XXH128_hash_t XXH_IPREF(XXH128_hash_t) /* Ensure the header is parsed again, even if it was previously included */ # undef XXHASH_H_5627135585666179 # undef XXHASH_H_STATIC_13879238742 #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ /* **************************************************************** * Stable API *****************************************************************/ #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 /*! @brief Marks a global symbol. */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) # if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) # ifdef XXH_EXPORT # define XXH_PUBLIC_API __declspec(dllexport) # elif XXH_IMPORT # define XXH_PUBLIC_API __declspec(dllimport) # endif # else # define XXH_PUBLIC_API /* do nothing */ # endif #endif #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) /* XXH32 */ # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) /* XXH64 */ # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) /* XXH3_64bits */ # define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) # define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) # define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) # define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) # define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) # define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) # define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) # define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) # define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) # define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) # define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) # define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) # define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) # define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) # define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) /* XXH3_128bits */ # define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) # define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) # define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) # define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) # define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) # define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) # define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) # define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) # define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) # define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) # define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) # define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) # define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) # define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) # define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) #endif /* ************************************* * Compiler specifics ***************************************/ /* specific declaration modes for Windows */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) # if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) # ifdef XXH_EXPORT # define XXH_PUBLIC_API __declspec(dllexport) # elif XXH_IMPORT # define XXH_PUBLIC_API __declspec(dllimport) # endif # else # define XXH_PUBLIC_API /* do nothing */ # endif #endif #if defined (__GNUC__) # define XXH_CONSTF __attribute__((const)) # define XXH_PUREF __attribute__((pure)) # define XXH_MALLOCF __attribute__((malloc)) #else # define XXH_CONSTF /* disable */ # define XXH_PUREF # define XXH_MALLOCF #endif /* ************************************* * Version ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 8 #define XXH_VERSION_RELEASE 2 /*! @brief Version number, encoded as two digits each */ #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) /*! * @brief Obtains the xxHash version. * * This is mostly useful when xxHash is compiled as a shared library, * since the returned value comes from the library, as opposed to header file. * * @return @ref XXH_VERSION_NUMBER of the invoked library. */ XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); /* **************************** * Common basic types ******************************/ #include /* size_t */ /*! * @brief Exit code for the streaming API. */ typedef enum { XXH_OK = 0, /*!< OK */ XXH_ERROR /*!< Error */ } XXH_errorcode; /*-********************************************************************** * 32-bit hash ************************************************************************/ #if defined(XXH_DOXYGEN) /* Don't show include */ /*! * @brief An unsigned 32-bit integer. * * Not necessarily defined to `uint32_t` but functionally equivalent. */ typedef uint32_t XXH32_hash_t; #elif !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint32_t XXH32_hash_t; #else # include # if UINT_MAX == 0xFFFFFFFFUL typedef unsigned int XXH32_hash_t; # elif ULONG_MAX == 0xFFFFFFFFUL typedef unsigned long XXH32_hash_t; # else # error "unsupported platform: need a 32-bit type" # endif #endif /*! * @} * * @defgroup XXH32_family XXH32 family * @ingroup public * Contains functions used in the classic 32-bit xxHash algorithm. * * @note * XXH32 is useful for older platforms, with no or poor 64-bit performance. * Note that the @ref XXH3_family provides competitive speed for both 32-bit * and 64-bit systems, and offers true 64/128 bit hash results. * * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families * @see @ref XXH32_impl for implementation details * @{ */ /*! * @brief Calculates the 32-bit hash of @p input using xxHash32. * * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 32-bit seed to alter the hash's output predictably. * * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 32-bit xxHash32 value. * * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); #ifndef XXH_NO_STREAM /*! * @typedef struct XXH32_state_s XXH32_state_t * @brief The opaque state struct for the XXH32 streaming API. * * @see XXH32_state_s for details. * @see @ref streaming_example "Streaming Example" */ typedef struct XXH32_state_s XXH32_state_t; /*! * @brief Allocates an @ref XXH32_state_t. * * @return An allocated pointer of @ref XXH32_state_t on success. * @return `NULL` on failure. * * @note Must be freed with XXH32_freeState(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); /*! * @brief Frees an @ref XXH32_state_t. * * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). * * @return @ref XXH_OK. * * @note @p statePtr must be allocated with XXH32_createState(). * * @see @ref streaming_example "Streaming Example" * */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); /*! * @brief Copies one @ref XXH32_state_t to another. * * @param dst_state The state to copy to. * @param src_state The state to copy from. * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); /*! * @brief Resets an @ref XXH32_state_t to begin a new hash. * * @param statePtr The state struct to reset. * @param seed The 32-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note This function resets and seeds a state. Call it before @ref XXH32_update(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); /*! * @brief Consumes a block of @p input to an @ref XXH32_state_t. * * @param statePtr The state struct to update. * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre * @p statePtr must not be `NULL`. * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note Call this to incrementally consume blocks of data. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); /*! * @brief Returns the calculated hash value from an @ref XXH32_state_t. * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * * @return The calculated 32-bit xxHash32 value from that state. * * @note * Calling XXH32_digest() will not affect @p statePtr, so you can update, * digest, and update again. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ /*! * @brief Canonical (big endian) representation of @ref XXH32_hash_t. */ typedef struct { unsigned char digest[4]; /*!< Hash bytes, big endian */ } XXH32_canonical_t; /*! * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. * * @param dst The @ref XXH32_canonical_t pointer to be stored to. * @param hash The @ref XXH32_hash_t to be converted. * * @pre * @p dst must not be `NULL`. * * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); /*! * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. * * @param src The @ref XXH32_canonical_t to convert. * * @pre * @p src must not be `NULL`. * * @return The converted hash. * * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); /*! @cond Doxygen ignores this part */ #ifdef __has_attribute # define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) #else # define XXH_HAS_ATTRIBUTE(x) 0 #endif /*! @endcond */ /*! @cond Doxygen ignores this part */ /* * C23 __STDC_VERSION__ number hasn't been specified yet. For now * leave as `201711L` (C17 + 1). * TODO: Update to correct value when its been specified. */ #define XXH_C23_VN 201711L /*! @endcond */ /*! @cond Doxygen ignores this part */ /* C-language Attributes are added in C23. */ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) # define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) #else # define XXH_HAS_C_ATTRIBUTE(x) 0 #endif /*! @endcond */ /*! @cond Doxygen ignores this part */ #if defined(__cplusplus) && defined(__has_cpp_attribute) # define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) #else # define XXH_HAS_CPP_ATTRIBUTE(x) 0 #endif /*! @endcond */ /*! @cond Doxygen ignores this part */ /* * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute * introduced in CPP17 and C23. * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough */ #if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) # define XXH_FALLTHROUGH [[fallthrough]] #elif XXH_HAS_ATTRIBUTE(__fallthrough__) # define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) #else # define XXH_FALLTHROUGH /* fallthrough */ #endif /*! @endcond */ /*! @cond Doxygen ignores this part */ /* * Define XXH_NOESCAPE for annotated pointers in public API. * https://clang.llvm.org/docs/AttributeReference.html#noescape * As of writing this, only supported by clang. */ #if XXH_HAS_ATTRIBUTE(noescape) # define XXH_NOESCAPE __attribute__((noescape)) #else # define XXH_NOESCAPE #endif /*! @endcond */ /*! * @} * @ingroup public * @{ */ #ifndef XXH_NO_LONG_LONG /*-********************************************************************** * 64-bit hash ************************************************************************/ #if defined(XXH_DOXYGEN) /* don't include */ /*! * @brief An unsigned 64-bit integer. * * Not necessarily defined to `uint64_t` but functionally equivalent. */ typedef uint64_t XXH64_hash_t; #elif !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint64_t XXH64_hash_t; #else # include # if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL /* LP64 ABI says uint64_t is unsigned long */ typedef unsigned long XXH64_hash_t; # else /* the following type must have a width of 64-bit */ typedef unsigned long long XXH64_hash_t; # endif #endif /*! * @} * * @defgroup XXH64_family XXH64 family * @ingroup public * @{ * Contains functions used in the classic 64-bit xxHash algorithm. * * @note * XXH3 provides competitive speed for both 32-bit and 64-bit systems, * and offers true 64/128 bit hash results. * It provides better speed for systems with vector processing capabilities. */ /*! * @brief Calculates the 64-bit hash of @p input using xxHash64. * * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 64-bit seed to alter the hash's output predictably. * * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 64-bit xxHash64 value. * * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); /******* Streaming *******/ #ifndef XXH_NO_STREAM /*! * @brief The opaque state struct for the XXH64 streaming API. * * @see XXH64_state_s for details. * @see @ref streaming_example "Streaming Example" */ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ /*! * @brief Allocates an @ref XXH64_state_t. * * @return An allocated pointer of @ref XXH64_state_t on success. * @return `NULL` on failure. * * @note Must be freed with XXH64_freeState(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); /*! * @brief Frees an @ref XXH64_state_t. * * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). * * @return @ref XXH_OK. * * @note @p statePtr must be allocated with XXH64_createState(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); /*! * @brief Copies one @ref XXH64_state_t to another. * * @param dst_state The state to copy to. * @param src_state The state to copy from. * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); /*! * @brief Resets an @ref XXH64_state_t to begin a new hash. * * @param statePtr The state struct to reset. * @param seed The 64-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note This function resets and seeds a state. Call it before @ref XXH64_update(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); /*! * @brief Consumes a block of @p input to an @ref XXH64_state_t. * * @param statePtr The state struct to update. * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre * @p statePtr must not be `NULL`. * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note Call this to incrementally consume blocks of data. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); /*! * @brief Returns the calculated hash value from an @ref XXH64_state_t. * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * * @return The calculated 64-bit xxHash64 value from that state. * * @note * Calling XXH64_digest() will not affect @p statePtr, so you can update, * digest, and update again. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ /*! * @brief Canonical (big endian) representation of @ref XXH64_hash_t. */ typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; /*! * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. * * @param dst The @ref XXH64_canonical_t pointer to be stored to. * @param hash The @ref XXH64_hash_t to be converted. * * @pre * @p dst must not be `NULL`. * * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); /*! * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. * * @param src The @ref XXH64_canonical_t to convert. * * @pre * @p src must not be `NULL`. * * @return The converted hash. * * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); #ifndef XXH_NO_XXH3 /*! * @} * ************************************************************************ * @defgroup XXH3_family XXH3 family * @ingroup public * @{ * * XXH3 is a more recent hash algorithm featuring: * - Improved speed for both small and large inputs * - True 64-bit and 128-bit outputs * - SIMD acceleration * - Improved 32-bit viability * * Speed analysis methodology is explained here: * * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html * * Compared to XXH64, expect XXH3 to run approximately * ~2x faster on large inputs and >3x faster on small ones, * exact differences vary depending on platform. * * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, * but does not require it. * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 * at competitive speeds, even without vector support. Further details are * explained in the implementation. * * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD * implementations for many common platforms: * - AVX512 * - AVX2 * - SSE2 * - ARM NEON * - WebAssembly SIMD128 * - POWER8 VSX * - s390x ZVector * This can be controlled via the @ref XXH_VECTOR macro, but it automatically * selects the best version according to predefined macros. For the x86 family, an * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. * * XXH3 implementation is portable: * it has a generic C90 formulation that can be compiled on any platform, * all implementations generate exactly the same hash value on all platforms. * Starting from v0.8.0, it's also labelled "stable", meaning that * any future version will also generate the same hash value. * * XXH3 offers 2 variants, _64bits and _128bits. * * When only 64 bits are needed, prefer invoking the _64bits variant, as it * reduces the amount of mixing, resulting in faster speed on small inputs. * It's also generally simpler to manipulate a scalar return type than a struct. * * The API supports one-shot hashing, streaming mode, and custom secrets. */ /*-********************************************************************** * XXH3 64-bit variant ************************************************************************/ /*! * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. * * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 64-bit XXH3 hash value. * * @note * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however * it may have slightly better performance due to constant propagation of the * defaults. * * @see * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); /*! * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. * * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * @param seed The 64-bit seed to alter the hash result predictably. * * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 64-bit XXH3 hash value. * * @note * seed == 0 produces the same results as @ref XXH3_64bits(). * * This variant generates a custom secret on the fly based on default secret * altered using the @p seed value. * * While this operation is decently fast, note that it's not completely free. * * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); /*! * The bare minimum size for a custom secret. * * @see * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). */ #define XXH3_SECRET_SIZE_MIN 136 /*! * @brief Calculates 64-bit variant of XXH3 with a custom "secret". * * @param data The block of data to be hashed, at least @p len bytes in size. * @param len The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * * @return The calculated 64-bit XXH3 hash value. * * @pre * The memory between @p data and @p data + @p len must be valid, * readable, contiguous memory. However, if @p length is `0`, @p data may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * It's possible to provide any blob of bytes as a "secret" to generate the hash. * This makes it more difficult for an external actor to prepare an intentional collision. * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). * However, the quality of the secret impacts the dispersion of the hash algorithm. * Therefore, the secret _must_ look like a bunch of random bytes. * Avoid "trivial" or structured data such as repeated sequences or a text document. * Whenever in doubt about the "randomness" of the blob of bytes, * consider employing @ref XXH3_generateSecret() instead (see below). * It will generate a proper high entropy secret derived from the blob of bytes. * Another advantage of using XXH3_generateSecret() is that * it guarantees that all bits within the initial blob of bytes * will impact every bit of the output. * This is not necessarily the case when using the blob of bytes directly * because, when hashing _small_ inputs, only a portion of the secret is employed. * * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); /******* Streaming *******/ #ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. * As a consequence, streaming is slower than one-shot hashing. * For better performance, prefer one-shot functions whenever applicable. */ /*! * @brief The opaque state struct for the XXH3 streaming API. * * @see XXH3_state_s for details. * @see @ref streaming_example "Streaming Example" */ typedef struct XXH3_state_s XXH3_state_t; XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); /*! * @brief Copies one @ref XXH3_state_t to another. * * @param dst_state The state to copy to. * @param src_state The state to copy from. * @pre * @p dst_state and @p src_state must not be `NULL` and must not overlap. */ XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); /*! * @brief Resets an @ref XXH3_state_t to begin a new hash. * * @param statePtr The state struct to reset. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note * - This function resets `statePtr` and generate a secret with default parameters. * - Call this function before @ref XXH3_64bits_update(). * - Digest will be equivalent to `XXH3_64bits()`. * * @see @ref streaming_example "Streaming Example" * */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); /*! * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. * * @param statePtr The state struct to reset. * @param seed The 64-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note * - This function resets `statePtr` and generate a secret from `seed`. * - Call this function before @ref XXH3_64bits_update(). * - Digest will be equivalent to `XXH3_64bits_withSeed()`. * * @see @ref streaming_example "Streaming Example" * */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * * @param statePtr The state struct to reset. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note * `secret` is referenced, it _must outlive_ the hash streaming session. * * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, * and the quality of produced hash values depends on secret's entropy * (secret's content should look like a bunch of random bytes). * When in doubt about the randomness of a candidate `secret`, * consider employing `XXH3_generateSecret()` instead (see below). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); /*! * @brief Consumes a block of @p input to an @ref XXH3_state_t. * * @param statePtr The state struct to update. * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre * @p statePtr must not be `NULL`. * @pre * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note Call this to incrementally consume blocks of data. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); /*! * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * * @return The calculated XXH3 64-bit hash value from that state. * * @note * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, * digest, and update again. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); #endif /* !XXH_NO_STREAM */ /* note : canonical representation of XXH3 is the same as XXH64 * since they both produce XXH64_hash_t values */ /*-********************************************************************** * XXH3 128-bit variant ************************************************************************/ /*! * @brief The return value from 128-bit hashes. * * Stored in little endian order, although the fields themselves are in native * endianness. */ typedef struct { XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ XXH64_hash_t high64; /*!< `value >> 64` */ } XXH128_hash_t; /*! * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. * * @param data The block of data to be hashed, at least @p length bytes in size. * @param len The length of @p data, in bytes. * * @return The calculated 128-bit variant of XXH3 value. * * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead * for shorter inputs. * * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however * it may have slightly better performance due to constant propagation of the * defaults. * * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); /*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. * * @param data The block of data to be hashed, at least @p length bytes in size. * @param len The length of @p data, in bytes. * @param seed The 64-bit seed to alter the hash result predictably. * * @return The calculated 128-bit variant of XXH3 value. * * @note * seed == 0 produces the same results as @ref XXH3_64bits(). * * This variant generates a custom secret on the fly based on default secret * altered using the @p seed value. * * While this operation is decently fast, note that it's not completely free. * * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); /*! * @brief Calculates 128-bit variant of XXH3 with a custom "secret". * * @param data The block of data to be hashed, at least @p len bytes in size. * @param len The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * * @return The calculated 128-bit variant of XXH3 value. * * It's possible to provide any blob of bytes as a "secret" to generate the hash. * This makes it more difficult for an external actor to prepare an intentional collision. * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). * However, the quality of the secret impacts the dispersion of the hash algorithm. * Therefore, the secret _must_ look like a bunch of random bytes. * Avoid "trivial" or structured data such as repeated sequences or a text document. * Whenever in doubt about the "randomness" of the blob of bytes, * consider employing @ref XXH3_generateSecret() instead (see below). * It will generate a proper high entropy secret derived from the blob of bytes. * Another advantage of using XXH3_generateSecret() is that * it guarantees that all bits within the initial blob of bytes * will impact every bit of the output. * This is not necessarily the case when using the blob of bytes directly * because, when hashing _small_ inputs, only a portion of the secret is employed. * * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); /******* Streaming *******/ #ifndef XXH_NO_STREAM /* * Streaming requires state maintenance. * This operation costs memory and CPU. * As a consequence, streaming is slower than one-shot hashing. * For better performance, prefer one-shot functions whenever applicable. * * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). * Use already declared XXH3_createState() and XXH3_freeState(). * * All reset and streaming functions have same meaning as their 64-bit counterpart. */ /*! * @brief Resets an @ref XXH3_state_t to begin a new hash. * * @param statePtr The state struct to reset. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note * - This function resets `statePtr` and generate a secret with default parameters. * - Call it before @ref XXH3_128bits_update(). * - Digest will be equivalent to `XXH3_128bits()`. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); /*! * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. * * @param statePtr The state struct to reset. * @param seed The 64-bit seed to alter the hash result predictably. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note * - This function resets `statePtr` and generate a secret from `seed`. * - Call it before @ref XXH3_128bits_update(). * - Digest will be equivalent to `XXH3_128bits_withSeed()`. * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * * @param statePtr The state struct to reset. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * `secret` is referenced, it _must outlive_ the hash streaming session. * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, * and the quality of produced hash values depends on secret's entropy * (secret's content should look like a bunch of random bytes). * When in doubt about the randomness of a candidate `secret`, * consider employing `XXH3_generateSecret()` instead (see below). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); /*! * @brief Consumes a block of @p input to an @ref XXH3_state_t. * * Call this to incrementally consume blocks of data. * * @param statePtr The state struct to update. * @param input The block of data to be hashed, at least @p length bytes in size. * @param length The length of @p input, in bytes. * * @pre * @p statePtr must not be `NULL`. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @note * The memory between @p input and @p input + @p length must be valid, * readable, contiguous memory. However, if @p length is `0`, @p input may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); /*! * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. * * @param statePtr The state struct to calculate the hash from. * * @pre * @p statePtr must not be `NULL`. * * @return The calculated XXH3 128-bit hash value from that state. * * @note * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, * digest, and update again. * */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); #endif /* !XXH_NO_STREAM */ /* Following helper functions make it possible to compare XXH128_hast_t values. * Since XXH128_hash_t is a structure, this capability is not offered by the language. * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ /*! * @brief Check equality of two XXH128_hash_t values * * @param h1 The 128-bit hash value. * @param h2 Another 128-bit hash value. * * @return `1` if `h1` and `h2` are equal. * @return `0` if they are not. */ XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); /*! * @brief Compares two @ref XXH128_hash_t * * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. * * @param h128_1 Left-hand side value * @param h128_2 Right-hand side value * * @return >0 if @p h128_1 > @p h128_2 * @return =0 if @p h128_1 == @p h128_2 * @return <0 if @p h128_1 < @p h128_2 */ XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); /******* Canonical representation *******/ typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; /*! * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. * * @param dst The @ref XXH128_canonical_t pointer to be stored to. * @param hash The @ref XXH128_hash_t to be converted. * * @pre * @p dst must not be `NULL`. * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); /*! * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. * * @param src The @ref XXH128_canonical_t to convert. * * @pre * @p src must not be `NULL`. * * @return The converted hash. * @see @ref canonical_representation_example "Canonical Representation Example" */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); #endif /* !XXH_NO_XXH3 */ #endif /* XXH_NO_LONG_LONG */ /*! * @} */ #endif /* XXHASH_H_5627135585666179 */ #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) #define XXHASH_H_STATIC_13879238742 /* **************************************************************************** * This section contains declarations which are not guaranteed to remain stable. * They may change in future versions, becoming incompatible with a different * version of the library. * These declarations should only be used with static linking. * Never use them in association with dynamic linking! ***************************************************************************** */ /* * These definitions are only present to allow static allocation * of XXH states, on stack or in a struct, for example. * Never **ever** access their members directly. */ /*! * @internal * @brief Structure for XXH32 streaming API. * * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is * an opaque type. This allows fields to safely be changed. * * Typedef'd to @ref XXH32_state_t. * Do not access the members of this struct directly. * @see XXH64_state_s, XXH3_state_s */ struct XXH32_state_s { XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ XXH32_hash_t v[4]; /*!< Accumulator lanes */ XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ }; /* typedef'd to XXH32_state_t */ #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ /*! * @internal * @brief Structure for XXH64 streaming API. * * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is * an opaque type. This allows fields to safely be changed. * * Typedef'd to @ref XXH64_state_t. * Do not access the members of this struct directly. * @see XXH32_state_s, XXH3_state_s */ struct XXH64_state_s { XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ XXH64_hash_t v[4]; /*!< Accumulator lanes */ XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ }; /* typedef'd to XXH64_state_t */ #ifndef XXH_NO_XXH3 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ # include # define XXH_ALIGN(n) alignas(n) #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ /* In C++ alignas() is a keyword */ # define XXH_ALIGN(n) alignas(n) #elif defined(__GNUC__) # define XXH_ALIGN(n) __attribute__ ((aligned(n))) #elif defined(_MSC_VER) # define XXH_ALIGN(n) __declspec(align(n)) #else # define XXH_ALIGN(n) /* disabled */ #endif /* Old GCC versions only accept the attribute after the type in structures. */ #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ && defined(__GNUC__) # define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) #else # define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type #endif /*! * @brief The size of the internal XXH3 buffer. * * This is the optimal update size for incremental hashing. * * @see XXH3_64b_update(), XXH3_128b_update(). */ #define XXH3_INTERNALBUFFER_SIZE 256 /*! * @internal * @brief Default size of the secret buffer (and @ref XXH3_kSecret). * * This is the size used in @ref XXH3_kSecret and the seeded functions. * * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. */ #define XXH3_SECRET_DEFAULT_SIZE 192 /*! * @internal * @brief Structure for XXH3 streaming API. * * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. * Otherwise it is an opaque type. * Never use this definition in combination with dynamic library. * This allows fields to safely be changed in the future. * * @note ** This structure has a strict alignment requirement of 64 bytes!! ** * Do not allocate this with `malloc()` or `new`, * it will not be sufficiently aligned. * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. * * Typedef'd to @ref XXH3_state_t. * Do never access the members of this struct directly. * * @see XXH3_INITSTATE() for stack initialization. * @see XXH3_createState(), XXH3_freeState(). * @see XXH32_state_s, XXH64_state_s */ struct XXH3_state_s { XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); /*!< Used to store a custom secret generated from a seed. */ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); /*!< The internal buffer. @see XXH32_state_s::mem32 */ XXH32_hash_t bufferedSize; /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ XXH32_hash_t useSeed; /*!< Reserved field. Needed for padding on 64-bit. */ size_t nbStripesSoFar; /*!< Number or stripes processed. */ XXH64_hash_t totalLen; /*!< Total length hashed. 64-bit even on 32-bit targets. */ size_t nbStripesPerBlock; /*!< Number of stripes per block. */ size_t secretLimit; /*!< Size of @ref customSecret or @ref extSecret */ XXH64_hash_t seed; /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ XXH64_hash_t reserved64; /*!< Reserved field. */ const unsigned char* extSecret; /*!< Reference to an external secret for the _withSecret variants, NULL * for other variants. */ /* note: there may be some padding at the end due to alignment on 64 bytes */ }; /* typedef'd to XXH3_state_t */ #undef XXH_ALIGN_MEMBER /*! * @brief Initializes a stack-allocated `XXH3_state_s`. * * When the @ref XXH3_state_t structure is merely emplaced on stack, * it should be initialized with XXH3_INITSTATE() or a memset() * in case its first reset uses XXH3_NNbits_reset_withSeed(). * This init can be omitted if the first reset uses default or _withSecret mode. * This operation isn't necessary when the state is created with XXH3_createState(). * Note that this doesn't prepare the state for a streaming operation, * it's still necessary to use XXH3_NNbits_reset*() afterwards. */ #define XXH3_INITSTATE(XXH3_state_ptr) \ do { \ XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ tmp_xxh3_state_ptr->seed = 0; \ tmp_xxh3_state_ptr->extSecret = NULL; \ } while(0) /*! * @brief Calculates the 128-bit hash of @p data using XXH3. * * @param data The block of data to be hashed, at least @p len bytes in size. * @param len The length of @p data, in bytes. * @param seed The 64-bit seed to alter the hash's output predictably. * * @pre * The memory between @p data and @p data + @p len must be valid, * readable, contiguous memory. However, if @p len is `0`, @p data may be * `NULL`. In C++, this also must be *TriviallyCopyable*. * * @return The calculated 128-bit XXH3 value. * * @see @ref single_shot_example "Single Shot Example" for an example. */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); /* === Experimental API === */ /* Symbols defined below must be considered tied to a specific library version. */ /*! * @brief Derive a high-entropy secret from any user-defined content, named customSeed. * * @param secretBuffer A writable buffer for derived high-entropy secret data. * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_SIZE_MIN. * @param customSeed A user-defined content. * @param customSeedSize Size of customSeed, in bytes. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * The generated secret can be used in combination with `*_withSecret()` functions. * The `_withSecret()` variants are useful to provide a higher level of protection * than 64-bit seed, as it becomes much more difficult for an external actor to * guess how to impact the calculation logic. * * The function accepts as input a custom seed of any length and any content, * and derives from it a high-entropy secret of length @p secretSize into an * already allocated buffer @p secretBuffer. * * The generated secret can then be used with any `*_withSecret()` variant. * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() * are part of this list. They all accept a `secret` parameter * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) * _and_ feature very high entropy (consist of random-looking bytes). * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can * be employed to ensure proper quality. * * @p customSeed can be anything. It can have any size, even small ones, * and its content can be anything, even "poor entropy" sources such as a bunch * of zeroes. The resulting `secret` will nonetheless provide all required qualities. * * @pre * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. * * Example code: * @code{.c} * #include * #include * #include * #define XXH_STATIC_LINKING_ONLY // expose unstable API * #include "xxhash.h" * // Hashes argv[2] using the entropy from argv[1]. * int main(int argc, char* argv[]) * { * char secret[XXH3_SECRET_SIZE_MIN]; * if (argv != 3) { return 1; } * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); * XXH64_hash_t h = XXH3_64bits_withSecret( * argv[2], strlen(argv[2]), * secret, sizeof(secret) * ); * printf("%016llx\n", (unsigned long long) h); * } * @endcode */ XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); /*! * @brief Generate the same secret as the _withSeed() variants. * * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes * @param seed The 64-bit seed to alter the hash result predictably. * * The generated secret can be used in combination with *`*_withSecret()` and `_withSecretandSeed()` variants. * * Example C++ `std::string` hash class: * @code{.cpp} * #include * #define XXH_STATIC_LINKING_ONLY // expose unstable API * #include "xxhash.h" * // Slow, seeds each time * class HashSlow { * XXH64_hash_t seed; * public: * HashSlow(XXH64_hash_t s) : seed{s} {} * size_t operator()(const std::string& x) const { * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; * } * }; * // Fast, caches the seeded secret for future uses. * class HashFast { * unsigned char secret[XXH3_SECRET_DEFAULT_SIZE]; * public: * HashFast(XXH64_hash_t s) { * XXH3_generateSecret_fromSeed(secret, seed); * } * size_t operator()(const std::string& x) const { * return size_t{ * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) * }; * } * }; * @endcode */ XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); /*! * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. * * @param data The block of data to be hashed, at least @p len bytes in size. * @param len The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * @param seed The 64-bit seed to alter the hash result predictably. * * These variants generate hash values using either * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). * * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. * `_withSeed()` has to generate the secret on the fly for "large" keys. * It's fast, but can be perceptible for "not so large" keys (< 1 KB). * `_withSecret()` has to generate the masks on the fly for "small" keys, * which requires more instructions than _withSeed() variants. * Therefore, _withSecretandSeed variant combines the best of both worlds. * * When @p secret has been generated by XXH3_generateSecret_fromSeed(), * this variant produces *exactly* the same results as `_withSeed()` variant, * hence offering only a pure speed benefit on "large" input, * by skipping the need to regenerate the secret for every large input. * * Another usage scenario is to hash the secret to a 64-bit hash value, * for example with XXH3_64bits(), which then becomes the seed, * and then employ both the seed and the secret in _withSecretandSeed(). * On top of speed, an added benefit is that each bit in the secret * has a 50% chance to swap each bit in the output, via its impact to the seed. * * This is not guaranteed when using the secret directly in "small data" scenarios, * because only portions of the secret are employed for small data. */ XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed); /*! * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. * * @param input The block of data to be hashed, at least @p len bytes in size. * @param length The length of @p data, in bytes. * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * @param seed64 The 64-bit seed to alter the hash result predictably. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @see XXH3_64bits_withSecretandSeed() */ XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64); #ifndef XXH_NO_STREAM /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * @param seed64 The 64-bit seed to alter the hash result predictably. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @see XXH3_64bits_withSecretandSeed() */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64); /*! * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. * * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). * @param secret The secret data. * @param secretSize The length of @p secret, in bytes. * @param seed64 The 64-bit seed to alter the hash result predictably. * * @return @ref XXH_OK on success. * @return @ref XXH_ERROR on failure. * * @see XXH3_64bits_withSecretandSeed() */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64); #endif /* !XXH_NO_STREAM */ #endif /* !XXH_NO_XXH3 */ #endif /* XXH_NO_LONG_LONG */ #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) # define XXH_IMPLEMENTATION #endif #endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ /* ======================================================================== */ /* ======================================================================== */ /* ======================================================================== */ /*-********************************************************************** * xxHash implementation *-********************************************************************** * xxHash's implementation used to be hosted inside xxhash.c. * * However, inlining requires implementation to be visible to the compiler, * hence be included alongside the header. * Previously, implementation was hosted inside xxhash.c, * which was then #included when inlining was activated. * This construction created issues with a few build and install systems, * as it required xxhash.c to be stored in /include directory. * * xxHash implementation is now directly integrated within xxhash.h. * As a consequence, xxhash.c is no longer needed in /include. * * xxhash.c is still available and is still useful. * In a "normal" setup, when xxhash is not inlined, * xxhash.h only exposes the prototypes and public symbols, * while xxhash.c can be built into an object file xxhash.o * which can then be linked into the final binary. ************************************************************************/ #if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) # define XXH_IMPLEM_13a8737387 /* ************************************* * Tuning parameters ***************************************/ /*! * @defgroup tuning Tuning parameters * @{ * * Various macros to control xxHash's behavior. */ #ifdef XXH_DOXYGEN /*! * @brief Define this to disable 64-bit code. * * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. */ # define XXH_NO_LONG_LONG # undef XXH_NO_LONG_LONG /* don't actually */ /*! * @brief Controls how unaligned memory is accessed. * * By default, access to unaligned memory is controlled by `memcpy()`, which is * safe and portable. * * Unfortunately, on some target/compiler combinations, the generated assembly * is sub-optimal. * * The below switch allow selection of a different access method * in the search for improved performance. * * @par Possible options: * * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` * @par * Use `memcpy()`. Safe and portable. Note that most modern compilers will * eliminate the function call and treat it as an unaligned access. * * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` * @par * Depends on compiler extensions and is therefore not portable. * This method is safe _if_ your compiler supports it, * and *generally* as fast or faster than `memcpy`. * * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast * @par * Casts directly and dereferences. This method doesn't depend on the * compiler, but it violates the C standard as it directly dereferences an * unaligned pointer. It can generate buggy code on targets which do not * support unaligned memory accesses, but in some circumstances, it's the * only known way to get the most performance. * * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift * @par * Also portable. This can generate the best code on old compilers which don't * inline small `memcpy()` calls, and it might also be faster on big-endian * systems which lack a native byteswap instruction. However, some compilers * will emit literal byteshifts even if the target supports unaligned access. * * * @warning * Methods 1 and 2 rely on implementation-defined behavior. Use these with * care, as what works on one compiler/platform/optimization level may cause * another to read garbage data or even crash. * * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. * * Prefer these methods in priority order (0 > 3 > 1 > 2) */ # define XXH_FORCE_MEMORY_ACCESS 0 /*! * @def XXH_SIZE_OPT * @brief Controls how much xxHash optimizes for size. * * xxHash, when compiled, tends to result in a rather large binary size. This * is mostly due to heavy usage to forced inlining and constant folding of the * @ref XXH3_family to increase performance. * * However, some developers prefer size over speed. This option can * significantly reduce the size of the generated code. When using the `-Os` * or `-Oz` options on GCC or Clang, this is defined to 1 by default, * otherwise it is defined to 0. * * Most of these size optimizations can be controlled manually. * * This is a number from 0-2. * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed * comes first. * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more * conservative and disables hacks that increase code size. It implies the * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, * and @ref XXH3_NEON_LANES == 8 if they are not already defined. * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. * Performance may cry. For example, the single shot functions just use the * streaming API. */ # define XXH_SIZE_OPT 0 /*! * @def XXH_FORCE_ALIGN_CHECK * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() * and XXH64() only). * * This is an important performance trick for architectures without decent * unaligned memory access performance. * * It checks for input alignment, and when conditions are met, uses a "fast * path" employing direct 32-bit/64-bit reads, resulting in _dramatically * faster_ read speed. * * The check costs one initial branch per hash, which is generally negligible, * but not zero. * * Moreover, it's not useful to generate an additional code path if memory * access uses the same instruction for both aligned and unaligned * addresses (e.g. x86 and aarch64). * * In these cases, the alignment check can be removed by setting this macro to 0. * Then the code will always use unaligned memory access. * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips * which are platforms known to offer good unaligned memory accesses performance. * * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. * * This option does not affect XXH3 (only XXH32 and XXH64). */ # define XXH_FORCE_ALIGN_CHECK 0 /*! * @def XXH_NO_INLINE_HINTS * @brief When non-zero, sets all functions to `static`. * * By default, xxHash tries to force the compiler to inline almost all internal * functions. * * This can usually improve performance due to reduced jumping and improved * constant folding, but significantly increases the size of the binary which * might not be favorable. * * Additionally, sometimes the forced inlining can be detrimental to performance, * depending on the architecture. * * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the * compiler full control on whether to inline or not. * * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. */ # define XXH_NO_INLINE_HINTS 0 /*! * @def XXH3_INLINE_SECRET * @brief Determines whether to inline the XXH3 withSecret code. * * When the secret size is known, the compiler can improve the performance * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). * * However, if the secret size is not known, it doesn't have any benefit. This * happens when xxHash is compiled into a global symbol. Therefore, if * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. * * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers * that are *sometimes* force inline on -Og, and it is impossible to automatically * detect this optimization level. */ # define XXH3_INLINE_SECRET 0 /*! * @def XXH32_ENDJMP * @brief Whether to use a jump for `XXH32_finalize`. * * For performance, `XXH32_finalize` uses multiple branches in the finalizer. * This is generally preferable for performance, * but depending on exact architecture, a jmp may be preferable. * * This setting is only possibly making a difference for very small inputs. */ # define XXH32_ENDJMP 0 /*! * @internal * @brief Redefines old internal names. * * For compatibility with code that uses xxHash's internals before the names * were changed to improve namespacing. There is no other reason to use this. */ # define XXH_OLD_NAMES # undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ /*! * @def XXH_NO_STREAM * @brief Disables the streaming API. * * When xxHash is not inlined and the streaming functions are not used, disabling * the streaming functions can improve code size significantly, especially with * the @ref XXH3_family which tends to make constant folded copies of itself. */ # define XXH_NO_STREAM # undef XXH_NO_STREAM /* don't actually */ #endif /* XXH_DOXYGEN */ /*! * @} */ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ /* prefer __packed__ structures (method 1) for GCC * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy * which for some reason does unaligned loads. */ # if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif #ifndef XXH_SIZE_OPT /* default to 1 for -Os or -Oz */ # if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) # define XXH_SIZE_OPT 1 # else # define XXH_SIZE_OPT 0 # endif #endif #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ # if XXH_SIZE_OPT >= 1 || \ defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ # define XXH_FORCE_ALIGN_CHECK 0 # else # define XXH_FORCE_ALIGN_CHECK 1 # endif #endif #ifndef XXH_NO_INLINE_HINTS # if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ # define XXH_NO_INLINE_HINTS 1 # else # define XXH_NO_INLINE_HINTS 0 # endif #endif #ifndef XXH3_INLINE_SECRET # if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ || !defined(XXH_INLINE_ALL) # define XXH3_INLINE_SECRET 0 # else # define XXH3_INLINE_SECRET 1 # endif #endif #ifndef XXH32_ENDJMP /* generally preferable for performance */ # define XXH32_ENDJMP 0 #endif /*! * @defgroup impl Implementation * @{ */ /* ************************************* * Includes & Memory related functions ***************************************/ #if defined(XXH_NO_STREAM) /* nothing */ #elif defined(XXH_NO_STDLIB) /* When requesting to disable any mention of stdlib, * the library loses the ability to invoked malloc / free. * In practice, it means that functions like `XXH*_createState()` * will always fail, and return NULL. * This flag is useful in situations where * xxhash.h is integrated into some kernel, embedded or limited environment * without access to dynamic allocation. */ static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } static void XXH_free(void* p) { (void)p; } #else /* * Modify the local functions below should you wish to use * different memory routines for malloc() and free() */ #include /*! * @internal * @brief Modify this function to use a different routine than malloc(). */ static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } /*! * @internal * @brief Modify this function to use a different routine than free(). */ static void XXH_free(void* p) { free(p); } #endif /* XXH_NO_STDLIB */ #include /*! * @internal * @brief Modify this function to use a different routine than memcpy(). */ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } #include /* ULLONG_MAX */ /* ************************************* * Compiler Specific Options ***************************************/ #ifdef _MSC_VER /* Visual Studio warning fix */ # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif #if XXH_NO_INLINE_HINTS /* disable inlining hints */ # if defined(__GNUC__) || defined(__clang__) # define XXH_FORCE_INLINE static __attribute__((unused)) # else # define XXH_FORCE_INLINE static # endif # define XXH_NO_INLINE static /* enable inlining hints */ #elif defined(__GNUC__) || defined(__clang__) # define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) # define XXH_NO_INLINE static __attribute__((noinline)) #elif defined(_MSC_VER) /* Visual Studio */ # define XXH_FORCE_INLINE static __forceinline # define XXH_NO_INLINE static __declspec(noinline) #elif defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ # define XXH_FORCE_INLINE static inline # define XXH_NO_INLINE static #else # define XXH_FORCE_INLINE static # define XXH_NO_INLINE static #endif #if XXH3_INLINE_SECRET # define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE #else # define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE #endif /* ************************************* * Debug ***************************************/ /*! * @ingroup tuning * @def XXH_DEBUGLEVEL * @brief Sets the debugging level. * * XXH_DEBUGLEVEL is expected to be defined externally, typically via the * compiler's command line options. The value must be a number. */ #ifndef XXH_DEBUGLEVEL # ifdef DEBUGLEVEL /* backwards compat */ # define XXH_DEBUGLEVEL DEBUGLEVEL # else # define XXH_DEBUGLEVEL 0 # endif #endif #if (XXH_DEBUGLEVEL>=1) # include /* note: can still be disabled with NDEBUG */ # define XXH_ASSERT(c) assert(c) #else # if defined(__INTEL_COMPILER) # define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) # else # define XXH_ASSERT(c) XXH_ASSUME(c) # endif #endif /* note: use after variable declarations */ #ifndef XXH_STATIC_ASSERT # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) # elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) # else # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) # endif # define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) #endif /*! * @internal * @def XXH_COMPILER_GUARD(var) * @brief Used to prevent unwanted optimizations for @p var. * * It uses an empty GCC inline assembly statement with a register constraint * which forces @p var into a general purpose register (eg eax, ebx, ecx * on x86) and marks it as modified. * * This is used in a few places to avoid unwanted autovectorization (e.g. * XXH32_round()). All vectorization we want is explicit via intrinsics, * and _usually_ isn't wanted elsewhere. * * We also use it to prevent unwanted constant folding for AArch64 in * XXH3_initCustomSecret_scalar(). */ #if defined(__GNUC__) || defined(__clang__) # define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) #else # define XXH_COMPILER_GUARD(var) ((void)0) #endif /* Specifically for NEON vectors which use the "w" constraint, on * Clang. */ #if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) # define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) #else # define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) #endif /* ************************************* * Basic Types ***************************************/ #if !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include typedef uint8_t xxh_u8; #else typedef unsigned char xxh_u8; #endif typedef XXH32_hash_t xxh_u32; #ifdef XXH_OLD_NAMES # warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" # define BYTE xxh_u8 # define U8 xxh_u8 # define U32 xxh_u32 #endif /* *** Memory access *** */ /*! * @internal * @fn xxh_u32 XXH_read32(const void* ptr) * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * * @param ptr The pointer to read from. * @return The 32-bit native endian integer from the bytes at @p ptr. */ /*! * @internal * @fn xxh_u32 XXH_readLE32(const void* ptr) * @brief Reads an unaligned 32-bit little endian integer from @p ptr. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * * @param ptr The pointer to read from. * @return The 32-bit little endian integer from the bytes at @p ptr. */ /*! * @internal * @fn xxh_u32 XXH_readBE32(const void* ptr) * @brief Reads an unaligned 32-bit big endian integer from @p ptr. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * * @param ptr The pointer to read from. * @return The 32-bit big endian integer from the bytes at @p ptr. */ /*! * @internal * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. * * Affected by @ref XXH_FORCE_MEMORY_ACCESS. * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is * always @ref XXH_alignment::XXH_unaligned. * * @param ptr The pointer to read from. * @param align Whether @p ptr is aligned. * @pre * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte * aligned. * @return The 32-bit little endian integer from the bytes at @p ptr. */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. * We actually directly use XXH_readLE32 and XXH_readBE32. */ #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) /* * Force direct memory access. Only works on CPU which support unaligned memory * access in hardware. */ static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) /* * __attribute__((aligned(1))) is supported by gcc and clang. Originally the * documentation claimed that it only increased the alignment, but actually it * can decrease it on gcc, clang, and icc: * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, * https://gcc.godbolt.org/z/xYez1j67Y. */ #ifdef XXH_OLD_NAMES typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; #endif static xxh_u32 XXH_read32(const void* ptr) { typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; return *((const xxh_unalign32*)ptr); } #else /* * Portable and safe solution. Generally efficient. * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ static xxh_u32 XXH_read32(const void* memPtr) { xxh_u32 val; XXH_memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ /* *** Endianness *** */ /*! * @ingroup tuning * @def XXH_CPU_LITTLE_ENDIAN * @brief Whether the target is little endian. * * Defined to 1 if the target is little endian, or 0 if it is big endian. * It can be defined externally, for example on the compiler command line. * * If it is not defined, * a runtime check (which is usually constant folded) is used instead. * * @note * This is not necessarily defined to an integer constant. * * @see XXH_isLittleEndian() for the runtime check. */ #ifndef XXH_CPU_LITTLE_ENDIAN /* * Try to detect endianness automatically, to avoid the nonstandard behavior * in `XXH_isLittleEndian()` */ # if defined(_WIN32) /* Windows is always little endian */ \ || defined(__LITTLE_ENDIAN__) \ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) # define XXH_CPU_LITTLE_ENDIAN 1 # elif defined(__BIG_ENDIAN__) \ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) # define XXH_CPU_LITTLE_ENDIAN 0 # else /*! * @internal * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. * * Most compilers will constant fold this. */ static int XXH_isLittleEndian(void) { /* * Portable and well-defined behavior. * Don't use static: it is detrimental to performance. */ const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; return one.c[0]; } # define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() # endif #endif /* **************************************** * Compiler-specific Functions and Macros ******************************************/ #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) #ifdef __has_builtin # define XXH_HAS_BUILTIN(x) __has_builtin(x) #else # define XXH_HAS_BUILTIN(x) 0 #endif /* * C23 and future versions have standard "unreachable()". * Once it has been implemented reliably we can add it as an * additional case: * * ``` * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) * # include * # ifdef unreachable * # define XXH_UNREACHABLE() unreachable() * # endif * #endif * ``` * * Note C++23 also has std::unreachable() which can be detected * as follows: * ``` * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) * # include * # define XXH_UNREACHABLE() std::unreachable() * #endif * ``` * NB: `__cpp_lib_unreachable` is defined in the `` header. * We don't use that as including `` in `extern "C"` blocks * doesn't work on GCC12 */ #if XXH_HAS_BUILTIN(__builtin_unreachable) # define XXH_UNREACHABLE() __builtin_unreachable() #elif defined(_MSC_VER) # define XXH_UNREACHABLE() __assume(0) #else # define XXH_UNREACHABLE() #endif #if XXH_HAS_BUILTIN(__builtin_assume) # define XXH_ASSUME(c) __builtin_assume(c) #else # define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } #endif /*! * @internal * @def XXH_rotl32(x,r) * @brief 32-bit rotate left. * * @param x The 32-bit integer to be rotated. * @param r The number of bits to rotate. * @pre * @p r > 0 && @p r < 32 * @note * @p x and @p r may be evaluated multiple times. * @return The rotated result. */ #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ && XXH_HAS_BUILTIN(__builtin_rotateleft64) # define XXH_rotl32 __builtin_rotateleft32 # define XXH_rotl64 __builtin_rotateleft64 /* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ #elif defined(_MSC_VER) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) #else # define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) # define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) #endif /*! * @internal * @fn xxh_u32 XXH_swap32(xxh_u32 x) * @brief A 32-bit byteswap. * * @param x The 32-bit integer to byteswap. * @return @p x, byteswapped. */ #if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap32 _byteswap_ulong #elif XXH_GCC_VERSION >= 403 # define XXH_swap32 __builtin_bswap32 #else static xxh_u32 XXH_swap32 (xxh_u32 x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | ((x >> 8) & 0x0000ff00 ) | ((x >> 24) & 0x000000ff ); } #endif /* *************************** * Memory reads *****************************/ /*! * @internal * @brief Enum to indicate whether a pointer is aligned. */ typedef enum { XXH_aligned, /*!< Aligned */ XXH_unaligned /*!< Possibly unaligned */ } XXH_alignment; /* * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. * * This is ideal for older compilers which don't inline memcpy. */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; return bytePtr[0] | ((xxh_u32)bytePtr[1] << 8) | ((xxh_u32)bytePtr[2] << 16) | ((xxh_u32)bytePtr[3] << 24); } XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; return bytePtr[3] | ((xxh_u32)bytePtr[2] << 8) | ((xxh_u32)bytePtr[1] << 16) | ((xxh_u32)bytePtr[0] << 24); } #else XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); } static xxh_u32 XXH_readBE32(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); } #endif XXH_FORCE_INLINE xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) { if (align==XXH_unaligned) { return XXH_readLE32(ptr); } else { return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); } } /* ************************************* * Misc ***************************************/ /*! @ingroup public */ XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } /* ******************************************************************* * 32-bit hash functions *********************************************************************/ /*! * @} * @defgroup XXH32_impl XXH32 implementation * @ingroup impl * * Details on the XXH32 implementation. * @{ */ /* #define instead of static const, to be used as initializers */ #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ #ifdef XXH_OLD_NAMES # define PRIME32_1 XXH_PRIME32_1 # define PRIME32_2 XXH_PRIME32_2 # define PRIME32_3 XXH_PRIME32_3 # define PRIME32_4 XXH_PRIME32_4 # define PRIME32_5 XXH_PRIME32_5 #endif /*! * @internal * @brief Normal stripe processing routine. * * This shuffles the bits so that any bit from @p input impacts several bits in * @p acc. * * @param acc The accumulator lane. * @param input The stripe of input to mix. * @return The mixed accumulator lane. */ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { acc += input * XXH_PRIME32_2; acc = XXH_rotl32(acc, 13); acc *= XXH_PRIME32_1; #if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) /* * UGLY HACK: * A compiler fence is used to prevent GCC and Clang from * autovectorizing the XXH32 loop (pragmas and attributes don't work for some * reason) without globally disabling SSE4.1. * * The reason we want to avoid vectorization is because despite working on * 4 integers at a time, there are multiple factors slowing XXH32 down on * SSE4: * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on * newer chips!) making it slightly slower to multiply four integers at * once compared to four integers independently. Even when pmulld was * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE * just to multiply unless doing a long operation. * * - Four instructions are required to rotate, * movqda tmp, v // not required with VEX encoding * pslld tmp, 13 // tmp <<= 13 * psrld v, 19 // x >>= 19 * por v, tmp // x |= tmp * compared to one for scalar: * roll v, 13 // reliably fast across the board * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason * * - Instruction level parallelism is actually more beneficial here because * the SIMD actually serializes this operation: While v1 is rotating, v2 * can load data, while v3 can multiply. SSE forces them to operate * together. * * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing * the loop. NEON is only faster on the A53, and with the newer cores, it is less * than half the speed. * * Additionally, this is used on WASM SIMD128 because it JITs to the same * SIMD instructions and has the same issue. */ XXH_COMPILER_GUARD(acc); #endif return acc; } /*! * @internal * @brief Mixes all bits to finalize the hash. * * The final mix ensures that all input bits have a chance to impact any bit in * the output digest, resulting in an unbiased distribution. * * @param hash The hash to avalanche. * @return The avalanched hash. */ static xxh_u32 XXH32_avalanche(xxh_u32 hash) { hash ^= hash >> 15; hash *= XXH_PRIME32_2; hash ^= hash >> 13; hash *= XXH_PRIME32_3; hash ^= hash >> 16; return hash; } #define XXH_get32bits(p) XXH_readLE32_align(p, align) /*! * @internal * @brief Processes the last 0-15 bytes of @p ptr. * * There may be up to 15 bytes remaining to consume from the input. * This final stage will digest them to ensure that all input bytes are present * in the final mix. * * @param hash The hash to finalize. * @param ptr The pointer to the remaining input. * @param len The remaining length, modulo 16. * @param align Whether @p ptr is aligned. * @return The finalized hash. * @see XXH64_finalize(). */ static XXH_PUREF xxh_u32 XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) { #define XXH_PROCESS1 do { \ hash += (*ptr++) * XXH_PRIME32_5; \ hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ } while (0) #define XXH_PROCESS4 do { \ hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ ptr += 4; \ hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ } while (0) if (ptr==NULL) XXH_ASSERT(len == 0); /* Compact rerolled version; generally faster */ if (!XXH32_ENDJMP) { len &= 15; while (len >= 4) { XXH_PROCESS4; len -= 4; } while (len > 0) { XXH_PROCESS1; --len; } return XXH32_avalanche(hash); } else { switch(len&15) /* or switch(bEnd - p) */ { case 12: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 8: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 4: XXH_PROCESS4; return XXH32_avalanche(hash); case 13: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 9: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 5: XXH_PROCESS4; XXH_PROCESS1; return XXH32_avalanche(hash); case 14: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 10: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 6: XXH_PROCESS4; XXH_PROCESS1; XXH_PROCESS1; return XXH32_avalanche(hash); case 15: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 11: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 7: XXH_PROCESS4; XXH_FALLTHROUGH; /* fallthrough */ case 3: XXH_PROCESS1; XXH_FALLTHROUGH; /* fallthrough */ case 2: XXH_PROCESS1; XXH_FALLTHROUGH; /* fallthrough */ case 1: XXH_PROCESS1; XXH_FALLTHROUGH; /* fallthrough */ case 0: return XXH32_avalanche(hash); } XXH_ASSERT(0); return hash; /* reaching this point is deemed impossible */ } } #ifdef XXH_OLD_NAMES # define PROCESS1 XXH_PROCESS1 # define PROCESS4 XXH_PROCESS4 #else # undef XXH_PROCESS1 # undef XXH_PROCESS4 #endif /*! * @internal * @brief The implementation for @ref XXH32(). * * @param input , len , seed Directly passed from @ref XXH32(). * @param align Whether @p input is aligned. * @return The calculated hash. */ XXH_FORCE_INLINE XXH_PUREF xxh_u32 XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) { xxh_u32 h32; if (input==NULL) XXH_ASSERT(len == 0); if (len>=16) { const xxh_u8* const bEnd = input + len; const xxh_u8* const limit = bEnd - 15; xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; xxh_u32 v2 = seed + XXH_PRIME32_2; xxh_u32 v3 = seed + 0; xxh_u32 v4 = seed - XXH_PRIME32_1; do { v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; } while (input < limit); h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } else { h32 = seed + XXH_PRIME32_5; } h32 += (xxh_u32)len; return XXH32_finalize(h32, input, len&15, align); } /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) { #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH32_state_t state; XXH32_reset(&state, seed); XXH32_update(&state, (const xxh_u8*)input, len); return XXH32_digest(&state); #else if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); } } return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); #endif } /******* Hash streaming *******/ #ifndef XXH_NO_STREAM /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) { return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } /*! @ingroup XXH32_family */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) { XXH_memcpy(dstState, srcState, sizeof(*dstState)); } /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) { XXH_ASSERT(statePtr != NULL); memset(statePtr, 0, sizeof(*statePtr)); statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; statePtr->v[1] = seed + XXH_PRIME32_2; statePtr->v[2] = seed + 0; statePtr->v[3] = seed - XXH_PRIME32_1; return XXH_OK; } /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t* state, const void* input, size_t len) { if (input==NULL) { XXH_ASSERT(len == 0); return XXH_OK; } { const xxh_u8* p = (const xxh_u8*)input; const xxh_u8* const bEnd = p + len; state->total_len_32 += (XXH32_hash_t)len; state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); if (state->memsize + len < 16) { /* fill in tmp buffer */ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); state->memsize += (XXH32_hash_t)len; return XXH_OK; } if (state->memsize) { /* some data left from previous update */ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); { const xxh_u32* p32 = state->mem32; state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); } p += 16-state->memsize; state->memsize = 0; } if (p <= bEnd-16) { const xxh_u8* const limit = bEnd - 16; do { state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; } while (p<=limit); } if (p < bEnd) { XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); state->memsize = (unsigned)(bEnd-p); } } return XXH_OK; } /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) { xxh_u32 h32; if (state->large_len) { h32 = XXH_rotl32(state->v[0], 1) + XXH_rotl32(state->v[1], 7) + XXH_rotl32(state->v[2], 12) + XXH_rotl32(state->v[3], 18); } else { h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; } h32 += state->total_len_32; return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); } #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ /*! @ingroup XXH32_family */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); XXH_memcpy(dst, &hash, sizeof(*dst)); } /*! @ingroup XXH32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) { return XXH_readBE32(src); } #ifndef XXH_NO_LONG_LONG /* ******************************************************************* * 64-bit hash functions *********************************************************************/ /*! * @} * @ingroup impl * @{ */ /******* Memory access *******/ typedef XXH64_hash_t xxh_u64; #ifdef XXH_OLD_NAMES # define U64 xxh_u64 #endif #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. * We actually directly use XXH_readLE64 and XXH_readBE64. */ #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ static xxh_u64 XXH_read64(const void* memPtr) { return *(const xxh_u64*) memPtr; } #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) /* * __attribute__((aligned(1))) is supported by gcc and clang. Originally the * documentation claimed that it only increased the alignment, but actually it * can decrease it on gcc, clang, and icc: * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, * https://gcc.godbolt.org/z/xYez1j67Y. */ #ifdef XXH_OLD_NAMES typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; #endif static xxh_u64 XXH_read64(const void* ptr) { typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; return *((const xxh_unalign64*)ptr); } #else /* * Portable and safe solution. Generally efficient. * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ static xxh_u64 XXH_read64(const void* memPtr) { xxh_u64 val; XXH_memcpy(&val, memPtr, sizeof(val)); return val; } #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ #if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap64 _byteswap_uint64 #elif XXH_GCC_VERSION >= 403 # define XXH_swap64 __builtin_bswap64 #else static xxh_u64 XXH_swap64(xxh_u64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | ((x << 24) & 0x0000ff0000000000ULL) | ((x << 8) & 0x000000ff00000000ULL) | ((x >> 8) & 0x00000000ff000000ULL) | ((x >> 24) & 0x0000000000ff0000ULL) | ((x >> 40) & 0x000000000000ff00ULL) | ((x >> 56) & 0x00000000000000ffULL); } #endif /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; return bytePtr[0] | ((xxh_u64)bytePtr[1] << 8) | ((xxh_u64)bytePtr[2] << 16) | ((xxh_u64)bytePtr[3] << 24) | ((xxh_u64)bytePtr[4] << 32) | ((xxh_u64)bytePtr[5] << 40) | ((xxh_u64)bytePtr[6] << 48) | ((xxh_u64)bytePtr[7] << 56); } XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) { const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; return bytePtr[7] | ((xxh_u64)bytePtr[6] << 8) | ((xxh_u64)bytePtr[5] << 16) | ((xxh_u64)bytePtr[4] << 24) | ((xxh_u64)bytePtr[3] << 32) | ((xxh_u64)bytePtr[2] << 40) | ((xxh_u64)bytePtr[1] << 48) | ((xxh_u64)bytePtr[0] << 56); } #else XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); } static xxh_u64 XXH_readBE64(const void* ptr) { return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); } #endif XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void* ptr, XXH_alignment align) { if (align==XXH_unaligned) return XXH_readLE64(ptr); else return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); } /******* xxh64 *******/ /*! * @} * @defgroup XXH64_impl XXH64 implementation * @ingroup impl * * Details on the XXH64 implementation. * @{ */ /* #define rather that static const, to be used as initializers */ #define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ #define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ #define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ #define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ #define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ #ifdef XXH_OLD_NAMES # define PRIME64_1 XXH_PRIME64_1 # define PRIME64_2 XXH_PRIME64_2 # define PRIME64_3 XXH_PRIME64_3 # define PRIME64_4 XXH_PRIME64_4 # define PRIME64_5 XXH_PRIME64_5 #endif /*! @copydoc XXH32_round */ static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) { acc += input * XXH_PRIME64_2; acc = XXH_rotl64(acc, 31); acc *= XXH_PRIME64_1; #if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) /* * DISABLE AUTOVECTORIZATION: * A compiler fence is used to prevent GCC and Clang from * autovectorizing the XXH64 loop (pragmas and attributes don't work for some * reason) without globally disabling AVX512. * * Autovectorization of XXH64 tends to be detrimental, * though the exact outcome may change depending on exact cpu and compiler version. * For information, it has been reported as detrimental for Skylake-X, * but possibly beneficial for Zen4. * * The default is to disable auto-vectorization, * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. */ XXH_COMPILER_GUARD(acc); #endif return acc; } static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) { val = XXH64_round(0, val); acc ^= val; acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; return acc; } /*! @copydoc XXH32_avalanche */ static xxh_u64 XXH64_avalanche(xxh_u64 hash) { hash ^= hash >> 33; hash *= XXH_PRIME64_2; hash ^= hash >> 29; hash *= XXH_PRIME64_3; hash ^= hash >> 32; return hash; } #define XXH_get64bits(p) XXH_readLE64_align(p, align) /*! * @internal * @brief Processes the last 0-31 bytes of @p ptr. * * There may be up to 31 bytes remaining to consume from the input. * This final stage will digest them to ensure that all input bytes are present * in the final mix. * * @param hash The hash to finalize. * @param ptr The pointer to the remaining input. * @param len The remaining length, modulo 32. * @param align Whether @p ptr is aligned. * @return The finalized hash * @see XXH32_finalize(). */ static XXH_PUREF xxh_u64 XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) { if (ptr==NULL) XXH_ASSERT(len == 0); len &= 31; while (len >= 8) { xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); ptr += 8; hash ^= k1; hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; len -= 8; } if (len >= 4) { hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; ptr += 4; hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; len -= 4; } while (len > 0) { hash ^= (*ptr++) * XXH_PRIME64_5; hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; --len; } return XXH64_avalanche(hash); } #ifdef XXH_OLD_NAMES # define PROCESS1_64 XXH_PROCESS1_64 # define PROCESS4_64 XXH_PROCESS4_64 # define PROCESS8_64 XXH_PROCESS8_64 #else # undef XXH_PROCESS1_64 # undef XXH_PROCESS4_64 # undef XXH_PROCESS8_64 #endif /*! * @internal * @brief The implementation for @ref XXH64(). * * @param input , len , seed Directly passed from @ref XXH64(). * @param align Whether @p input is aligned. * @return The calculated hash. */ XXH_FORCE_INLINE XXH_PUREF xxh_u64 XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) { xxh_u64 h64; if (input==NULL) XXH_ASSERT(len == 0); if (len>=32) { const xxh_u8* const bEnd = input + len; const xxh_u8* const limit = bEnd - 31; xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; xxh_u64 v2 = seed + XXH_PRIME64_2; xxh_u64 v3 = seed + 0; xxh_u64 v4 = seed - XXH_PRIME64_1; do { v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; } while (input= 2 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, (const xxh_u8*)input, len); return XXH64_digest(&state); #else if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); } } return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); #endif } /******* Hash Streaming *******/ #ifndef XXH_NO_STREAM /*! @ingroup XXH64_family*/ XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } /*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } /*! @ingroup XXH64_family */ XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) { XXH_memcpy(dstState, srcState, sizeof(*dstState)); } /*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) { XXH_ASSERT(statePtr != NULL); memset(statePtr, 0, sizeof(*statePtr)); statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; statePtr->v[1] = seed + XXH_PRIME64_2; statePtr->v[2] = seed + 0; statePtr->v[3] = seed - XXH_PRIME64_1; return XXH_OK; } /*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) { if (input==NULL) { XXH_ASSERT(len == 0); return XXH_OK; } { const xxh_u8* p = (const xxh_u8*)input; const xxh_u8* const bEnd = p + len; state->total_len += len; if (state->memsize + len < 32) { /* fill in tmp buffer */ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); state->memsize += (xxh_u32)len; return XXH_OK; } if (state->memsize) { /* tmp buffer is full */ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); p += 32 - state->memsize; state->memsize = 0; } if (p+32 <= bEnd) { const xxh_u8* const limit = bEnd - 32; do { state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; } while (p<=limit); } if (p < bEnd) { XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); state->memsize = (unsigned)(bEnd-p); } } return XXH_OK; } /*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) { xxh_u64 h64; if (state->total_len >= 32) { h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); h64 = XXH64_mergeRound(h64, state->v[0]); h64 = XXH64_mergeRound(h64, state->v[1]); h64 = XXH64_mergeRound(h64, state->v[2]); h64 = XXH64_mergeRound(h64, state->v[3]); } else { h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; } h64 += (xxh_u64) state->total_len; return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); } #endif /* !XXH_NO_STREAM */ /******* Canonical representation *******/ /*! @ingroup XXH64_family */ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); XXH_memcpy(dst, &hash, sizeof(*dst)); } /*! @ingroup XXH64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) { return XXH_readBE64(src); } #ifndef XXH_NO_XXH3 /* ********************************************************************* * XXH3 * New generation hash designed for speed on small keys and vectorization ************************************************************************ */ /*! * @} * @defgroup XXH3_impl XXH3 implementation * @ingroup impl * @{ */ /* === Compiler specifics === */ #if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ # define XXH_RESTRICT /* disable */ #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ # define XXH_RESTRICT restrict #elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ || (defined (__clang__)) \ || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) /* * There are a LOT more compilers that recognize __restrict but this * covers the major ones. */ # define XXH_RESTRICT __restrict #else # define XXH_RESTRICT /* disable */ #endif #if (defined(__GNUC__) && (__GNUC__ >= 3)) \ || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ || defined(__clang__) # define XXH_likely(x) __builtin_expect(x, 1) # define XXH_unlikely(x) __builtin_expect(x, 0) #else # define XXH_likely(x) (x) # define XXH_unlikely(x) (x) #endif #ifndef XXH_HAS_INCLUDE # ifdef __has_include /* * Not defined as XXH_HAS_INCLUDE(x) (function-like) because * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) */ # define XXH_HAS_INCLUDE __has_include # else # define XXH_HAS_INCLUDE(x) 0 # endif #endif #if defined(__GNUC__) || defined(__clang__) # if defined(__ARM_FEATURE_SVE) # include # endif # if defined(__ARM_NEON__) || defined(__ARM_NEON) \ || (defined(_M_ARM) && _M_ARM >= 7) \ || defined(_M_ARM64) || defined(_M_ARM64EC) \ || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ # define inline __inline__ /* circumvent a clang bug */ # include # undef inline # elif defined(__AVX2__) # include # elif defined(__SSE2__) # include # endif #endif #if defined(_MSC_VER) # include #endif /* * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while * remaining a true 64-bit/128-bit hash function. * * This is done by prioritizing a subset of 64-bit operations that can be * emulated without too many steps on the average 32-bit machine. * * For example, these two lines seem similar, and run equally fast on 64-bit: * * xxh_u64 x; * x ^= (x >> 47); // good * x ^= (x >> 13); // bad * * However, to a 32-bit machine, there is a major difference. * * x ^= (x >> 47) looks like this: * * x.lo ^= (x.hi >> (47 - 32)); * * while x ^= (x >> 13) looks like this: * * // note: funnel shifts are not usually cheap. * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); * x.hi ^= (x.hi >> 13); * * The first one is significantly faster than the second, simply because the * shift is larger than 32. This means: * - All the bits we need are in the upper 32 bits, so we can ignore the lower * 32 bits in the shift. * - The shift result will always fit in the lower 32 bits, and therefore, * we can ignore the upper 32 bits in the xor. * * Thanks to this optimization, XXH3 only requires these features to be efficient: * * - Usable unaligned access * - A 32-bit or 64-bit ALU * - If 32-bit, a decent ADC instruction * - A 32 or 64-bit multiply with a 64-bit result * - For the 128-bit variant, a decent byteswap helps short inputs. * * The first two are already required by XXH32, and almost all 32-bit and 64-bit * platforms which can run XXH32 can run XXH3 efficiently. * * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one * notable exception. * * First of all, Thumb-1 lacks support for the UMULL instruction which * performs the important long multiply. This means numerous __aeabi_lmul * calls. * * Second of all, the 8 functional registers are just not enough. * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need * Lo registers, and this shuffling results in thousands more MOVs than A32. * * A32 and T32 don't have this limitation. They can access all 14 registers, * do a 32->64 multiply with UMULL, and the flexible operand allowing free * shifts is helpful, too. * * Therefore, we do a quick sanity check. * * If compiling Thumb-1 for a target which supports ARM instructions, we will * emit a warning, as it is not a "sane" platform to compile for. * * Usually, if this happens, it is because of an accident and you probably need * to specify -march, as you likely meant to compile for a newer architecture. * * Credit: large sections of the vectorial and asm source code paths * have been contributed by @easyaspi314 */ #if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) # warning "XXH3 is highly inefficient without ARM or Thumb-2." #endif /* ========================================== * Vectorization detection * ========================================== */ #ifdef XXH_DOXYGEN /*! * @ingroup tuning * @brief Overrides the vectorization implementation chosen for XXH3. * * Can be defined to 0 to disable SIMD or any of the values mentioned in * @ref XXH_VECTOR_TYPE. * * If this is not defined, it uses predefined macros to determine the best * implementation. */ # define XXH_VECTOR XXH_SCALAR /*! * @ingroup tuning * @brief Possible values for @ref XXH_VECTOR. * * Note that these are actually implemented as macros. * * If this is not defined, it is detected automatically. * internal macro XXH_X86DISPATCH overrides this. */ enum XXH_VECTOR_TYPE /* fake enum */ { XXH_SCALAR = 0, /*!< Portable scalar version */ XXH_SSE2 = 1, /*!< * SSE2 for Pentium 4, Opteron, all x86_64. * * @note SSE2 is also guaranteed on Windows 10, macOS, and * Android x86. */ XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ XXH_NEON = 4, /*!< * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 * via the SIMDeverywhere polyfill provided with the * Emscripten SDK. */ XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ }; /*! * @ingroup tuning * @brief Selects the minimum alignment for XXH3's accumulators. * * When using SIMD, this should match the alignment required for said vector * type, so, for example, 32 for AVX2. * * Default: Auto detected. */ # define XXH_ACC_ALIGN 8 #endif /* Actual definition */ #ifndef XXH_DOXYGEN # define XXH_SCALAR 0 # define XXH_SSE2 1 # define XXH_AVX2 2 # define XXH_AVX512 3 # define XXH_NEON 4 # define XXH_VSX 5 # define XXH_SVE 6 #endif #ifndef XXH_VECTOR /* can be defined on command line */ # if defined(__ARM_FEATURE_SVE) # define XXH_VECTOR XXH_SVE # elif ( \ defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ ) && ( \ defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ ) # define XXH_VECTOR XXH_NEON # elif defined(__AVX512F__) # define XXH_VECTOR XXH_AVX512 # elif defined(__AVX2__) # define XXH_VECTOR XXH_AVX2 # elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) # define XXH_VECTOR XXH_SSE2 # elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ || (defined(__s390x__) && defined(__VEC__)) \ && defined(__GNUC__) /* TODO: IBM XL */ # define XXH_VECTOR XXH_VSX # else # define XXH_VECTOR XXH_SCALAR # endif #endif /* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ #if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) # ifdef _MSC_VER # pragma warning(once : 4606) # else # warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." # endif # undef XXH_VECTOR # define XXH_VECTOR XXH_SCALAR #endif /* * Controls the alignment of the accumulator, * for compatibility with aligned vector loads, which are usually faster. */ #ifndef XXH_ACC_ALIGN # if defined(XXH_X86DISPATCH) # define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ # elif XXH_VECTOR == XXH_SCALAR /* scalar */ # define XXH_ACC_ALIGN 8 # elif XXH_VECTOR == XXH_SSE2 /* sse2 */ # define XXH_ACC_ALIGN 16 # elif XXH_VECTOR == XXH_AVX2 /* avx2 */ # define XXH_ACC_ALIGN 32 # elif XXH_VECTOR == XXH_NEON /* neon */ # define XXH_ACC_ALIGN 16 # elif XXH_VECTOR == XXH_VSX /* vsx */ # define XXH_ACC_ALIGN 16 # elif XXH_VECTOR == XXH_AVX512 /* avx512 */ # define XXH_ACC_ALIGN 64 # elif XXH_VECTOR == XXH_SVE /* sve */ # define XXH_ACC_ALIGN 64 # endif #endif #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 # define XXH_SEC_ALIGN XXH_ACC_ALIGN #elif XXH_VECTOR == XXH_SVE # define XXH_SEC_ALIGN XXH_ACC_ALIGN #else # define XXH_SEC_ALIGN 8 #endif #if defined(__GNUC__) || defined(__clang__) # define XXH_ALIASING __attribute__((may_alias)) #else # define XXH_ALIASING /* nothing */ #endif /* * UGLY HACK: * GCC usually generates the best code with -O3 for xxHash. * * However, when targeting AVX2, it is overzealous in its unrolling resulting * in code roughly 3/4 the speed of Clang. * * There are other issues, such as GCC splitting _mm256_loadu_si256 into * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which * only applies to Sandy and Ivy Bridge... which don't even support AVX2. * * That is why when compiling the AVX2 version, it is recommended to use either * -O2 -mavx2 -march=haswell * or * -O2 -mavx2 -mno-avx256-split-unaligned-load * for decent performance, or to use Clang instead. * * Fortunately, we can control the first one with a pragma that forces GCC into * -O2, but the other one we can't control without "failed to inline always * inline function due to target mismatch" warnings. */ #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ # pragma GCC push_options # pragma GCC optimize("-O2") #endif #if XXH_VECTOR == XXH_NEON /* * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 * optimizes out the entire hashLong loop because of the aliasing violation. * * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, * so the only option is to mark it as aliasing. */ typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; /*! * @internal * @brief `vld1q_u64` but faster and alignment-safe. * * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). * * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it * prohibits load-store optimizations. Therefore, a direct dereference is used. * * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe * unaligned load. */ #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ { return *(xxh_aliasing_uint64x2_t const *)ptr; } #else XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) { return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); } #endif /*! * @internal * @brief `vmlal_u32` on low and high halves of a vector. * * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` * with `vmlal_u32`. */ #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 XXH_FORCE_INLINE uint64x2_t XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { /* Inline assembly is the only way */ __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); return acc; } XXH_FORCE_INLINE uint64x2_t XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { /* This intrinsic works as expected */ return vmlal_high_u32(acc, lhs, rhs); } #else /* Portable intrinsic versions */ XXH_FORCE_INLINE uint64x2_t XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); } /*! @copydoc XXH_vmlal_low_u32 * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ XXH_FORCE_INLINE uint64x2_t XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) { return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); } #endif /*! * @ingroup tuning * @brief Controls the NEON to scalar ratio for XXH3 * * This can be set to 2, 4, 6, or 8. * * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. * * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU * bandwidth. * * This is even more noticeable on the more advanced cores like the Cortex-A76 which * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. * * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes * and 2 scalar lanes, which is chosen by default. * * This does not apply to Apple processors or 32-bit processors, which run better with * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. * * This change benefits CPUs with large micro-op buffers without negatively affecting * most other CPUs: * * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | * |:----------------------|:--------------------|----------:|-----------:|------:| * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | * * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. * * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning * it effectively becomes worse 4. * * @see XXH3_accumulate_512_neon() */ # ifndef XXH3_NEON_LANES # if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 # define XXH3_NEON_LANES 6 # else # define XXH3_NEON_LANES XXH_ACC_NB # endif # endif #endif /* XXH_VECTOR == XXH_NEON */ /* * VSX and Z Vector helpers. * * This is very messy, and any pull requests to clean this up are welcome. * * There are a lot of problems with supporting VSX and s390x, due to * inconsistent intrinsics, spotty coverage, and multiple endiannesses. */ #if XXH_VECTOR == XXH_VSX /* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, * and `pixel`. This is a problem for obvious reasons. * * These keywords are unnecessary; the spec literally says they are * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd * after including the header. * * We use pragma push_macro/pop_macro to keep the namespace clean. */ # pragma push_macro("bool") # pragma push_macro("vector") # pragma push_macro("pixel") /* silence potential macro redefined warnings */ # undef bool # undef vector # undef pixel # if defined(__s390x__) # include # else # include # endif /* Restore the original macro values, if applicable. */ # pragma pop_macro("pixel") # pragma pop_macro("vector") # pragma pop_macro("bool") typedef __vector unsigned long long xxh_u64x2; typedef __vector unsigned char xxh_u8x16; typedef __vector unsigned xxh_u32x4; /* * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. */ typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; # ifndef XXH_VSX_BE # if defined(__BIG_ENDIAN__) \ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) # define XXH_VSX_BE 1 # elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ # warning "-maltivec=be is not recommended. Please use native endianness." # define XXH_VSX_BE 1 # else # define XXH_VSX_BE 0 # endif # endif /* !defined(XXH_VSX_BE) */ # if XXH_VSX_BE # if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) # define XXH_vec_revb vec_revb # else /*! * A polyfill for POWER9's vec_revb(). */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; return vec_perm(val, val, vByteSwap); } # endif # endif /* XXH_VSX_BE */ /*! * Performs an unaligned vector load and byte swaps it on big endian. */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { xxh_u64x2 ret; XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); # if XXH_VSX_BE ret = XXH_vec_revb(ret); # endif return ret; } /* * vec_mulo and vec_mule are very problematic intrinsics on PowerPC * * These intrinsics weren't added until GCC 8, despite existing for a while, * and they are endian dependent. Also, their meaning swap depending on version. * */ # if defined(__s390x__) /* s390x is always big endian, no issue on this platform */ # define XXH_vec_mulo vec_mulo # define XXH_vec_mule vec_mule # elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) /* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ # define XXH_vec_mulo __builtin_altivec_vmulouw # define XXH_vec_mule __builtin_altivec_vmuleuw # else /* gcc needs inline assembly */ /* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) { xxh_u64x2 result; __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); return result; } XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { xxh_u64x2 result; __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); return result; } # endif /* XXH_vec_mulo, XXH_vec_mule */ #endif /* XXH_VECTOR == XXH_VSX */ #if XXH_VECTOR == XXH_SVE #define ACCRND(acc, offset) \ do { \ svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ acc = svadd_u64_x(mask, acc, mul); \ } while (0) #endif /* XXH_VECTOR == XXH_SVE */ /* prefetch * can be disabled, by declaring XXH_NO_PREFETCH build macro */ #if defined(XXH_NO_PREFETCH) # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ #else # if XXH_SIZE_OPT >= 1 # define XXH_PREFETCH(ptr) (void)(ptr) # elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) # define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) # else # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ # endif #endif /* XXH_NO_PREFETCH */ /* ========================================== * XXH3 default settings * ========================================== */ #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) # error "default keyset is not large enough" #endif /*! Pseudorandom secret taken directly from FARSH. */ XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, }; static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ #ifdef XXH_OLD_NAMES # define kSecret XXH3_kSecret #endif #ifdef XXH_DOXYGEN /*! * @brief Calculates a 32-bit to 64-bit long multiply. * * Implemented as a macro. * * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't * need to (but it shouldn't need to anyways, it is about 7 instructions to do * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we * use that instead of the normal method. * * If you are compiling for platforms like Thumb-1 and don't have a better option, * you may also want to write your own long multiply routine here. * * @param x, y Numbers to be multiplied * @return 64-bit product of the low 32 bits of @p x and @p y. */ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); } #elif defined(_MSC_VER) && defined(_M_IX86) # define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) #else /* * Downcast + upcast is usually better than masking on older compilers like * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. * * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands * and perform a full 64x64 multiply -- entirely redundant on 32-bit. */ # define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) #endif /*! * @brief Calculates a 64->128-bit long multiply. * * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar * version. * * @param lhs , rhs The 64-bit integers to be multiplied * @return The 128-bit result represented in an @ref XXH128_hash_t. */ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { /* * GCC/Clang __uint128_t method. * * On most 64-bit targets, GCC and Clang define a __uint128_t type. * This is usually the best way as it usually uses a native long 64-bit * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. * * Usually. * * Despite being a 32-bit platform, Clang (and emscripten) define this type * despite not having the arithmetic for it. This results in a laggy * compiler builtin call which calculates a full 128-bit multiply. * In that case it is best to use the portable one. * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 */ #if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ && defined(__SIZEOF_INT128__) \ || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; XXH128_hash_t r128; r128.low64 = (xxh_u64)(product); r128.high64 = (xxh_u64)(product >> 64); return r128; /* * MSVC for x64's _umul128 method. * * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); * * This compiles to single operand MUL on x64. */ #elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) #ifndef _MSC_VER # pragma intrinsic(_umul128) #endif xxh_u64 product_high; xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); XXH128_hash_t r128; r128.low64 = product_low; r128.high64 = product_high; return r128; /* * MSVC for ARM64's __umulh method. * * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. */ #elif defined(_M_ARM64) || defined(_M_ARM64EC) #ifndef _MSC_VER # pragma intrinsic(__umulh) #endif XXH128_hash_t r128; r128.low64 = lhs * rhs; r128.high64 = __umulh(lhs, rhs); return r128; #else /* * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. * * This is a fast and simple grade school multiply, which is shown below * with base 10 arithmetic instead of base 0x100000000. * * 9 3 // D2 lhs = 93 * x 7 5 // D2 rhs = 75 * ---------- * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 * --------- * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 * --------- * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 * * The reasons for adding the products like this are: * 1. It avoids manual carry tracking. Just like how * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. * This avoids a lot of complexity. * * 2. It hints for, and on Clang, compiles to, the powerful UMAAL * instruction available in ARM's Digital Signal Processing extension * in 32-bit ARMv6 and later, which is shown below: * * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) * { * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); * *RdHi = (xxh_u32)(product >> 32); * } * * This instruction was designed for efficient long multiplication, and * allows this to be calculated in only 4 instructions at speeds * comparable to some 64-bit ALUs. * * 3. It isn't terrible on other platforms. Usually this will be a couple * of 32-bit ADD/ADCs. */ /* First calculate all of the cross products. */ xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); /* Now add the products together. These will never overflow. */ xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); XXH128_hash_t r128; r128.low64 = lower; r128.high64 = upper; return r128; #endif } /*! * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. * * The reason for the separate function is to prevent passing too many structs * around by value. This will hopefully inline the multiply, but we don't force it. * * @param lhs , rhs The 64-bit integers to multiply * @return The low 64 bits of the product XOR'd by the high 64 bits. * @see XXH_mult64to128() */ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { XXH128_hash_t product = XXH_mult64to128(lhs, rhs); return product.low64 ^ product.high64; } /*! Seems to produce slightly better code on GCC for some reason. */ XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { XXH_ASSERT(0 <= shift && shift < 64); return v64 ^ (v64 >> shift); } /* * This is a fast avalanche stage, * suitable when input bits are already partially mixed */ static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) { h64 = XXH_xorshift64(h64, 37); h64 *= PRIME_MX1; h64 = XXH_xorshift64(h64, 32); return h64; } /* * This is a stronger avalanche, * inspired by Pelle Evensen's rrmxmx * preferable when input has not been previously mixed */ static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) { /* this mix is inspired by Pelle Evensen's rrmxmx */ h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); h64 *= PRIME_MX2; h64 ^= (h64 >> 35) + len ; h64 *= PRIME_MX2; return XXH_xorshift64(h64, 28); } /* ========================================== * Short keys * ========================================== * One of the shortcomings of XXH32 and XXH64 was that their performance was * sub-optimal on short lengths. It used an iterative algorithm which strongly * favored lengths that were a multiple of 4 or 8. * * Instead of iterating over individual inputs, we use a set of single shot * functions which piece together a range of lengths and operate in constant time. * * Additionally, the number of multiplies has been significantly reduced. This * reduces latency, especially when emulating 64-bit multiplies on 32-bit. * * Depending on the platform, this may or may not be faster than XXH32, but it * is almost guaranteed to be faster than XXH64. */ /* * At very short lengths, there isn't enough input to fully hide secrets, or use * the entire secret. * * There is also only a limited amount of mixing we can do before significantly * impacting performance. * * Therefore, we use different sections of the secret and always mix two secret * samples with an XOR. This should have no effect on performance on the * seedless or withSeed variants because everything _should_ be constant folded * by modern compilers. * * The XOR mixing hides individual parts of the secret and increases entropy. * * This adds an extra layer of strength for custom secrets. */ XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(1 <= len && len <= 3); XXH_ASSERT(secret != NULL); /* * len = 1: combined = { input[0], 0x01, input[0], input[0] } * len = 2: combined = { input[1], 0x02, input[0], input[1] } * len = 3: combined = { input[2], 0x03, input[0], input[1] } */ { xxh_u8 const c1 = input[0]; xxh_u8 const c2 = input[len >> 1]; xxh_u8 const c3 = input[len - 1]; xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; return XXH64_avalanche(keyed); } } XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(4 <= len && len <= 8); seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; { xxh_u32 const input1 = XXH_readLE32(input); xxh_u32 const input2 = XXH_readLE32(input + len - 4); xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); xxh_u64 const keyed = input64 ^ bitflip; return XXH3_rrmxmx(keyed, len); } } XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(9 <= len && len <= 16); { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; xxh_u64 const acc = len + XXH_swap64(input_lo) + input_hi + XXH3_mul128_fold64(input_lo, input_hi); return XXH3_avalanche(acc); } } XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(len <= 16); { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); if (len) return XXH3_len_1to3_64b(input, len, secret, seed); return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); } } /* * DISCLAIMER: There are known *seed-dependent* multicollisions here due to * multiplication by zero, affecting hashes of lengths 17 to 240. * * However, they are very unlikely. * * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all * unseeded non-cryptographic hashes, it does not attempt to defend itself * against specially crafted inputs, only random inputs. * * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes * cancelling out the secret is taken an arbitrary number of times (addressed * in XXH3_accumulate_512), this collision is very unlikely with random inputs * and/or proper seeding: * * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a * function that is only called up to 16 times per hash with up to 240 bytes of * input. * * This is not too bad for a non-cryptographic hash function, especially with * only 64 bit outputs. * * The 128-bit variant (which trades some speed for strength) is NOT affected * by this, although it is always a good idea to use a proper seed if you care * about strength. */ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) { #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ /* * UGLY HACK: * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in * slower code. * * By forcing seed64 into a register, we disrupt the cost model and * cause it to scalarize. See `XXH32_round()` * * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on * GCC 9.2, despite both emitting scalar code. * * GCC generates much better scalar code than Clang for the rest of XXH3, * which is why finding a more optimal codepath is an interest. */ XXH_COMPILER_GUARD(seed64); #endif { xxh_u64 const input_lo = XXH_readLE64(input); xxh_u64 const input_hi = XXH_readLE64(input+8); return XXH3_mul128_fold64( input_lo ^ (XXH_readLE64(secret) + seed64), input_hi ^ (XXH_readLE64(secret+8) - seed64) ); } } /* For mid range keys, XXH3 uses a Mum-hash variant. */ XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(16 < len && len <= 128); { xxh_u64 acc = len * XXH_PRIME64_1; #if XXH_SIZE_OPT >= 1 /* Smaller and cleaner, but slightly slower. */ unsigned int i = (unsigned int)(len - 1) / 32; do { acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); } while (i-- != 0); #else if (len > 32) { if (len > 64) { if (len > 96) { acc += XXH3_mix16B(input+48, secret+96, seed); acc += XXH3_mix16B(input+len-64, secret+112, seed); } acc += XXH3_mix16B(input+32, secret+64, seed); acc += XXH3_mix16B(input+len-48, secret+80, seed); } acc += XXH3_mix16B(input+16, secret+32, seed); acc += XXH3_mix16B(input+len-32, secret+48, seed); } acc += XXH3_mix16B(input+0, secret+0, seed); acc += XXH3_mix16B(input+len-16, secret+16, seed); #endif return XXH3_avalanche(acc); } } /*! * @brief Maximum size of "short" key in bytes. */ #define XXH3_MIDSIZE_MAX 240 XXH_NO_INLINE XXH_PUREF XXH64_hash_t XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); #define XXH3_MIDSIZE_STARTOFFSET 3 #define XXH3_MIDSIZE_LASTOFFSET 17 { xxh_u64 acc = len * XXH_PRIME64_1; xxh_u64 acc_end; unsigned int const nbRounds = (unsigned int)len / 16; unsigned int i; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); for (i=0; i<8; i++) { acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); } /* last bytes */ acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); XXH_ASSERT(nbRounds >= 8); acc = XXH3_avalanche(acc); #if defined(__clang__) /* Clang */ \ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. * In everywhere else, it uses scalar code. * * For 64->128-bit multiplies, even if the NEON was 100% optimal, it * would still be slower than UMAAL (see XXH_mult64to128). * * Unfortunately, Clang doesn't handle the long multiplies properly and * converts them to the nonexistent "vmulq_u64" intrinsic, which is then * scalarized into an ugly mess of VMOV.32 instructions. * * This mess is difficult to avoid without turning autovectorization * off completely, but they are usually relatively minor and/or not * worth it to fix. * * This loop is the easiest to fix, as unlike XXH32, this pragma * _actually works_ because it is a loop vectorization instead of an * SLP vectorization. */ #pragma clang loop vectorize(disable) #endif for (i=8 ; i < nbRounds; i++) { /* * Prevents clang for unrolling the acc loop and interleaving with this one. */ XXH_COMPILER_GUARD(acc); acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); } return XXH3_avalanche(acc + acc_end); } } /* ======= Long Keys ======= */ #define XXH_STRIPE_LEN 64 #define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) #ifdef XXH_OLD_NAMES # define STRIPE_LEN XXH_STRIPE_LEN # define ACC_NB XXH_ACC_NB #endif #ifndef XXH_PREFETCH_DIST # ifdef __clang__ # define XXH_PREFETCH_DIST 320 # else # if (XXH_VECTOR == XXH_AVX512) # define XXH_PREFETCH_DIST 512 # else # define XXH_PREFETCH_DIST 384 # endif # endif /* __clang__ */ #endif /* XXH_PREFETCH_DIST */ /* * These macros are to generate an XXH3_accumulate() function. * The two arguments select the name suffix and target attribute. * * The name of this symbol is XXH3_accumulate_() and it calls * XXH3_accumulate_512_(). * * It may be useful to hand implement this function if the compiler fails to * optimize the inline function. */ #define XXH3_ACCUMULATE_TEMPLATE(name) \ void \ XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ const xxh_u8* XXH_RESTRICT input, \ const xxh_u8* XXH_RESTRICT secret, \ size_t nbStripes) \ { \ size_t n; \ for (n = 0; n < nbStripes; n++ ) { \ const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ XXH3_accumulate_512_##name( \ acc, \ in, \ secret + n*XXH_SECRET_CONSUME_RATE); \ } \ } XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) { if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); XXH_memcpy(dst, &v64, sizeof(v64)); } /* Several intrinsic functions below are supposed to accept __int64 as argument, * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . * However, several environments do not define __int64 type, * requiring a workaround. */ #if !defined (__VMS) \ && (defined (__cplusplus) \ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) typedef int64_t xxh_i64; #else /* the following type must have a width of 64-bit */ typedef long long xxh_i64; #endif /* * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. * * It is a hardened version of UMAC, based off of FARSH's implementation. * * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD * implementations, and it is ridiculously fast. * * We harden it by mixing the original input to the accumulators as well as the product. * * This means that in the (relatively likely) case of a multiply by zero, the * original input is preserved. * * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve * cross-pollination, as otherwise the upper and lower halves would be * essentially independent. * * This doesn't matter on 64-bit hashes since they all get merged together in * the end, so we skip the extra step. * * Both XXH3_64bits and XXH3_128bits use this subroutine. */ #if (XXH_VECTOR == XXH_AVX512) \ || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) #ifndef XXH_TARGET_AVX512 # define XXH_TARGET_AVX512 /* disable attribute target */ #endif XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { __m512i* const xacc = (__m512i *) acc; XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); { /* data_vec = input[0]; */ __m512i const data_vec = _mm512_loadu_si512 (input); /* key_vec = secret[0]; */ __m512i const key_vec = _mm512_loadu_si512 (secret); /* data_key = data_vec ^ key_vec; */ __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); /* data_key_lo = data_key >> 32; */ __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); /* xacc[0] += swap(data_vec); */ __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); __m512i const sum = _mm512_add_epi64(*xacc, data_swap); /* xacc[0] += product; */ *xacc = _mm512_add_epi64(product, sum); } } XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) /* * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. * * Multiplication isn't perfect, as explained by Google in HighwayHash: * * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to * // varying degrees. In descending order of goodness, bytes * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. * // As expected, the upper and lower bytes are much worse. * * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 * * Since our algorithm uses a pseudorandom secret to add some variance into the * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. * * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid * extraction. * * Both XXH3_64bits and XXH3_128bits use this subroutine. */ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); { __m512i* const xacc = (__m512i*) acc; const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); /* xacc[0] ^= (xacc[0] >> 47) */ __m512i const acc_vec = *xacc; __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); /* xacc[0] ^= secret; */ __m512i const key_vec = _mm512_loadu_si512 (secret); __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); /* xacc[0] *= XXH_PRIME32_1; */ __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); } } XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); XXH_ASSERT(((size_t)customSecret & 63) == 0); (void)(&XXH_writeLE64); { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); __m512i* const dest = ( __m512i*) customSecret; int i; XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ XXH_ASSERT(((size_t)dest & 63) == 0); for (i=0; i < nbRounds; ++i) { dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); } } } #endif #if (XXH_VECTOR == XXH_AVX2) \ || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) #ifndef XXH_TARGET_AVX2 # define XXH_TARGET_AVX2 /* disable attribute target */ #endif XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 31) == 0); { __m256i* const xacc = (__m256i *) acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i* const xinput = (const __m256i *) input; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i* const xsecret = (const __m256i *) secret; size_t i; for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { /* data_vec = xinput[i]; */ __m256i const data_vec = _mm256_loadu_si256 (xinput+i); /* key_vec = xsecret[i]; */ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); /* data_key = data_vec ^ key_vec; */ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); /* data_key_lo = data_key >> 32; */ __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); /* xacc[i] += swap(data_vec); */ __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); /* xacc[i] += product; */ xacc[i] = _mm256_add_epi64(product, sum); } } } XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 31) == 0); { __m256i* const xacc = (__m256i*) acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i* const xsecret = (const __m256i *) secret; const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); size_t i; for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { /* xacc[i] ^= (xacc[i] >> 47) */ __m256i const acc_vec = xacc[i]; __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); /* xacc[i] ^= xsecret; */ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1; */ __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); } } } XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); (void)(&XXH_writeLE64); XXH_PREFETCH(customSecret); { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); __m256i* dest = ( __m256i*) customSecret; # if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack */ XXH_COMPILER_GUARD(dest); # endif XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ XXH_ASSERT(((size_t)dest & 31) == 0); /* GCC -O2 need unroll loop manually */ dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); } } #endif /* x86dispatch always generates SSE2 */ #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) #ifndef XXH_TARGET_SSE2 # define XXH_TARGET_SSE2 /* disable attribute target */ #endif XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { /* SSE2 is just a half-scale version of the AVX2 version. */ XXH_ASSERT((((size_t)acc) & 15) == 0); { __m128i* const xacc = (__m128i *) acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i* const xinput = (const __m128i *) input; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i* const xsecret = (const __m128i *) secret; size_t i; for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { /* data_vec = xinput[i]; */ __m128i const data_vec = _mm_loadu_si128 (xinput+i); /* key_vec = xsecret[i]; */ __m128i const key_vec = _mm_loadu_si128 (xsecret+i); /* data_key = data_vec ^ key_vec; */ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); /* data_key_lo = data_key >> 32; */ __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); /* xacc[i] += swap(data_vec); */ __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); __m128i const sum = _mm_add_epi64(xacc[i], data_swap); /* xacc[i] += product; */ xacc[i] = _mm_add_epi64(product, sum); } } } XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { __m128i* const xacc = (__m128i*) acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i* const xsecret = (const __m128i *) secret; const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); size_t i; for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { /* xacc[i] ^= (xacc[i] >> 47) */ __m128i const acc_vec = xacc[i]; __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); /* xacc[i] ^= xsecret[i]; */ __m128i const key_vec = _mm_loadu_si128 (xsecret+i); __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1; */ __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); } } } XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); (void)(&XXH_writeLE64); { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); # if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); # else __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); # endif int i; const void* const src16 = XXH3_kSecret; __m128i* dst16 = (__m128i*) customSecret; # if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack */ XXH_COMPILER_GUARD(dst16); # endif XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ XXH_ASSERT(((size_t)dst16 & 15) == 0); for (i=0; i < nbRounds; ++i) { dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); } } } #endif #if (XXH_VECTOR == XXH_NEON) /* forward declarations for the scalar routines */ XXH_FORCE_INLINE void XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, void const* XXH_RESTRICT secret, size_t lane); XXH_FORCE_INLINE void XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT secret, size_t lane); /*! * @internal * @brief The bulk processing loop for NEON and WASM SIMD128. * * The NEON code path is actually partially scalar when running on AArch64. This * is to optimize the pipelining and can have up to 15% speedup depending on the * CPU, and it also mitigates some GCC codegen issues. * * @see XXH3_NEON_LANES for configuring this and details about this optimization. * * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit * integers instead of the other platforms which mask full 64-bit vectors, * so the setup is more complicated than just shifting right. * * Additionally, there is an optimization for 4 lanes at once noted below. * * Since, as stated, the most optimal amount of lanes for Cortexes is 6, * there needs to be *three* versions of the accumulate operation used * for the remaining 2 lanes. * * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap * nearly perfectly. */ XXH_FORCE_INLINE void XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); { /* GCC for darwin arm64 does not like aliasing here */ xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ uint8_t const* xinput = (const uint8_t *) input; uint8_t const* xsecret = (const uint8_t *) secret; size_t i; #ifdef __wasm_simd128__ /* * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret * is constant propagated, which results in it converting it to this * inside the loop: * * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) * ... * * This requires a full 32-bit address immediate (and therefore a 6 byte * instruction) as well as an add for each offset. * * Putting an asm guard prevents it from folding (at the cost of losing * the alignment hint), and uses the free offset in `v128.load` instead * of adding secret_offset each time which overall reduces code size by * about a kilobyte and improves performance. */ XXH_COMPILER_GUARD(xsecret); #endif /* Scalar lanes use the normal scalarRound routine */ for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { XXH3_scalarRound(acc, input, secret, i); } i = 0; /* 4 NEON lanes at a time. */ for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { /* data_vec = xinput[i]; */ uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); /* key_vec = xsecret[i]; */ uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); /* data_swap = swap(data_vec) */ uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); /* data_key = data_vec ^ key_vec; */ uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); /* * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to * get one vector with the low 32 bits of each lane, and one vector * with the high 32 bits of each lane. * * The intrinsic returns a double vector because the original ARMv7-a * instruction modified both arguments in place. AArch64 and SIMD128 emit * two instructions from this intrinsic. * * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] */ uint32x4x2_t unzipped = vuzpq_u32( vreinterpretq_u32_u64(data_key_1), vreinterpretq_u32_u64(data_key_2) ); /* data_key_lo = data_key & 0xFFFFFFFF */ uint32x4_t data_key_lo = unzipped.val[0]; /* data_key_hi = data_key >> 32 */ uint32x4_t data_key_hi = unzipped.val[1]; /* * Then, we can split the vectors horizontally and multiply which, as for most * widening intrinsics, have a variant that works on both high half vectors * for free on AArch64. A similar instruction is available on SIMD128. * * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi */ uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); /* * Clang reorders * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s * c += a; // add acc.2d, acc.2d, swap.2d * to * c += a; // add acc.2d, acc.2d, swap.2d * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s * * While it would make sense in theory since the addition is faster, * for reasons likely related to umlal being limited to certain NEON * pipelines, this is worse. A compiler guard fixes this. */ XXH_COMPILER_GUARD_CLANG_NEON(sum_1); XXH_COMPILER_GUARD_CLANG_NEON(sum_2); /* xacc[i] = acc_vec + sum; */ xacc[i] = vaddq_u64(xacc[i], sum_1); xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); } /* Operate on the remaining NEON lanes 2 at a time. */ for (; i < XXH3_NEON_LANES / 2; i++) { /* data_vec = xinput[i]; */ uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); /* key_vec = xsecret[i]; */ uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); /* acc_vec_2 = swap(data_vec) */ uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); /* data_key = data_vec ^ key_vec; */ uint64x2_t data_key = veorq_u64(data_vec, key_vec); /* For two lanes, just use VMOVN and VSHRN. */ /* data_key_lo = data_key & 0xFFFFFFFF; */ uint32x2_t data_key_lo = vmovn_u64(data_key); /* data_key_hi = data_key >> 32; */ uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); /* Same Clang workaround as before */ XXH_COMPILER_GUARD_CLANG_NEON(sum); /* xacc[i] = acc_vec + sum; */ xacc[i] = vaddq_u64 (xacc[i], sum); } } } XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; uint8_t const* xsecret = (uint8_t const*) secret; size_t i; /* WASM uses operator overloads and doesn't need these. */ #ifndef __wasm_simd128__ /* { prime32_1, prime32_1 } */ uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); /* { 0, prime32_1, 0, prime32_1 } */ uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); #endif /* AArch64 uses both scalar and neon at the same time */ for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { XXH3_scalarScrambleRound(acc, secret, i); } for (i=0; i < XXH3_NEON_LANES / 2; i++) { /* xacc[i] ^= (xacc[i] >> 47); */ uint64x2_t acc_vec = xacc[i]; uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); uint64x2_t data_vec = veorq_u64(acc_vec, shifted); /* xacc[i] ^= xsecret[i]; */ uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); uint64x2_t data_key = veorq_u64(data_vec, key_vec); /* xacc[i] *= XXH_PRIME32_1 */ #ifdef __wasm_simd128__ /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ xacc[i] = data_key * XXH_PRIME32_1; #else /* * Expanded version with portable NEON intrinsics * * lo(x) * lo(y) + (hi(x) * lo(y) << 32) * * prod_hi = hi(data_key) * lo(prime) << 32 * * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits * and avoid the shift. */ uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); /* Extract low bits for vmlal_u32 */ uint32x2_t data_key_lo = vmovn_u64(data_key); /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); #endif } } } #endif #if (XXH_VECTOR == XXH_VSX) XXH_FORCE_INLINE void XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { /* presumed aligned */ xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ xxh_u64x2 const v32 = { 32, 32 }; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { /* data_vec = xinput[i]; */ xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); /* key_vec = xsecret[i]; */ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); xxh_u64x2 const data_key = data_vec ^ key_vec; /* shuffled = (data_key << 32) | (data_key >> 32); */ xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); /* acc_vec = xacc[i]; */ xxh_u64x2 acc_vec = xacc[i]; acc_vec += product; /* swap high and low halves */ #ifdef __s390x__ acc_vec += vec_permi(data_vec, data_vec, 2); #else acc_vec += vec_xxpermdi(data_vec, data_vec, 2); #endif xacc[i] = acc_vec; } } XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { XXH_ASSERT((((size_t)acc) & 15) == 0); { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; const xxh_u8* const xsecret = (const xxh_u8*) secret; /* constants */ xxh_u64x2 const v32 = { 32, 32 }; xxh_u64x2 const v47 = { 47, 47 }; xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; size_t i; for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { /* xacc[i] ^= (xacc[i] >> 47); */ xxh_u64x2 const acc_vec = xacc[i]; xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); /* xacc[i] ^= xsecret[i]; */ xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); xxh_u64x2 const data_key = data_vec ^ key_vec; /* xacc[i] *= XXH_PRIME32_1 */ /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); xacc[i] = prod_odd + (prod_even << v32); } } } #endif #if (XXH_VECTOR == XXH_SVE) XXH_FORCE_INLINE void XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { uint64_t *xacc = (uint64_t *)acc; const uint64_t *xinput = (const uint64_t *)(const void *)input; const uint64_t *xsecret = (const uint64_t *)(const void *)secret; svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); uint64_t element_count = svcntd(); if (element_count >= 8) { svbool_t mask = svptrue_pat_b64(SV_VL8); svuint64_t vacc = svld1_u64(mask, xacc); ACCRND(vacc, 0); svst1_u64(mask, xacc, vacc); } else if (element_count == 2) { /* sve128 */ svbool_t mask = svptrue_pat_b64(SV_VL2); svuint64_t acc0 = svld1_u64(mask, xacc + 0); svuint64_t acc1 = svld1_u64(mask, xacc + 2); svuint64_t acc2 = svld1_u64(mask, xacc + 4); svuint64_t acc3 = svld1_u64(mask, xacc + 6); ACCRND(acc0, 0); ACCRND(acc1, 2); ACCRND(acc2, 4); ACCRND(acc3, 6); svst1_u64(mask, xacc + 0, acc0); svst1_u64(mask, xacc + 2, acc1); svst1_u64(mask, xacc + 4, acc2); svst1_u64(mask, xacc + 6, acc3); } else { svbool_t mask = svptrue_pat_b64(SV_VL4); svuint64_t acc0 = svld1_u64(mask, xacc + 0); svuint64_t acc1 = svld1_u64(mask, xacc + 4); ACCRND(acc0, 0); ACCRND(acc1, 4); svst1_u64(mask, xacc + 0, acc0); svst1_u64(mask, xacc + 4, acc1); } } XXH_FORCE_INLINE void XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT input, const xxh_u8* XXH_RESTRICT secret, size_t nbStripes) { if (nbStripes != 0) { uint64_t *xacc = (uint64_t *)acc; const uint64_t *xinput = (const uint64_t *)(const void *)input; const uint64_t *xsecret = (const uint64_t *)(const void *)secret; svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); uint64_t element_count = svcntd(); if (element_count >= 8) { svbool_t mask = svptrue_pat_b64(SV_VL8); svuint64_t vacc = svld1_u64(mask, xacc + 0); do { /* svprfd(svbool_t, void *, enum svfprop); */ svprfd(mask, xinput + 128, SV_PLDL1STRM); ACCRND(vacc, 0); xinput += 8; xsecret += 1; nbStripes--; } while (nbStripes != 0); svst1_u64(mask, xacc + 0, vacc); } else if (element_count == 2) { /* sve128 */ svbool_t mask = svptrue_pat_b64(SV_VL2); svuint64_t acc0 = svld1_u64(mask, xacc + 0); svuint64_t acc1 = svld1_u64(mask, xacc + 2); svuint64_t acc2 = svld1_u64(mask, xacc + 4); svuint64_t acc3 = svld1_u64(mask, xacc + 6); do { svprfd(mask, xinput + 128, SV_PLDL1STRM); ACCRND(acc0, 0); ACCRND(acc1, 2); ACCRND(acc2, 4); ACCRND(acc3, 6); xinput += 8; xsecret += 1; nbStripes--; } while (nbStripes != 0); svst1_u64(mask, xacc + 0, acc0); svst1_u64(mask, xacc + 2, acc1); svst1_u64(mask, xacc + 4, acc2); svst1_u64(mask, xacc + 6, acc3); } else { svbool_t mask = svptrue_pat_b64(SV_VL4); svuint64_t acc0 = svld1_u64(mask, xacc + 0); svuint64_t acc1 = svld1_u64(mask, xacc + 4); do { svprfd(mask, xinput + 128, SV_PLDL1STRM); ACCRND(acc0, 0); ACCRND(acc1, 4); xinput += 8; xsecret += 1; nbStripes--; } while (nbStripes != 0); svst1_u64(mask, xacc + 0, acc0); svst1_u64(mask, xacc + 4, acc1); } } } #endif /* scalar variants - universal */ #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) /* * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they * emit an excess mask and a full 64-bit multiply-add (MADD X-form). * * While this might not seem like much, as AArch64 is a 64-bit architecture, only * big Cortex designs have a full 64-bit multiplier. * * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit * multiplies expand to 2-3 multiplies in microcode. This has a major penalty * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. * * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does * not have this penalty and does the mask automatically. */ XXH_FORCE_INLINE xxh_u64 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) { xxh_u64 ret; /* note: %x = 64-bit register, %w = 32-bit register */ __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); return ret; } #else XXH_FORCE_INLINE xxh_u64 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) { return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; } #endif /*! * @internal * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). * * This is extracted to its own function because the NEON path uses a combination * of NEON and scalar. */ XXH_FORCE_INLINE void XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, void const* XXH_RESTRICT secret, size_t lane) { xxh_u64* xacc = (xxh_u64*) acc; xxh_u8 const* xinput = (xxh_u8 const*) input; xxh_u8 const* xsecret = (xxh_u8 const*) secret; XXH_ASSERT(lane < XXH_ACC_NB); XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); { xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); } } /*! * @internal * @brief Processes a 64 byte block of data using the scalar path. */ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT input, const void* XXH_RESTRICT secret) { size_t i; /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ #if defined(__GNUC__) && !defined(__clang__) \ && (defined(__arm__) || defined(__thumb2__)) \ && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ && XXH_SIZE_OPT <= 0 # pragma GCC unroll 8 #endif for (i=0; i < XXH_ACC_NB; i++) { XXH3_scalarRound(acc, input, secret, i); } } XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) /*! * @internal * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). * * This is extracted to its own function because the NEON path uses a combination * of NEON and scalar. */ XXH_FORCE_INLINE void XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT secret, size_t lane) { xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); XXH_ASSERT(lane < XXH_ACC_NB); { xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); xxh_u64 acc64 = xacc[lane]; acc64 = XXH_xorshift64(acc64, 47); acc64 ^= key64; acc64 *= XXH_PRIME32_1; xacc[lane] = acc64; } } /*! * @internal * @brief Scrambles the accumulators after a large chunk has been read */ XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) { size_t i; for (i=0; i < XXH_ACC_NB; i++) { XXH3_scalarScrambleRound(acc, secret, i); } } XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) { /* * We need a separate pointer for the hack below, * which requires a non-const pointer. * Any decent compiler will optimize this out otherwise. */ const xxh_u8* kSecretPtr = XXH3_kSecret; XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); #if defined(__GNUC__) && defined(__aarch64__) /* * UGLY HACK: * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are * placed sequentially, in order, at the top of the unrolled loop. * * While MOVK is great for generating constants (2 cycles for a 64-bit * constant compared to 4 cycles for LDR), it fights for bandwidth with * the arithmetic instructions. * * I L S * MOVK * MOVK * MOVK * MOVK * ADD * SUB STR * STR * By forcing loads from memory (as the asm line causes the compiler to assume * that XXH3_kSecretPtr has been changed), the pipelines are used more * efficiently: * I L S * LDR * ADD LDR * SUB STR * STR * * See XXH3_NEON_LANES for details on the pipsline. * * XXH3_64bits_withSeed, len == 256, Snapdragon 835 * without hack: 2654.4 MB/s * with hack: 3202.9 MB/s */ XXH_COMPILER_GUARD(kSecretPtr); #endif { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; int i; for (i=0; i < nbRounds; i++) { /* * The asm hack causes the compiler to assume that kSecretPtr aliases with * customSecret, and on aarch64, this prevented LDP from merging two * loads together for free. Putting the loads together before the stores * properly generates LDP. */ xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); } } } typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); #if (XXH_VECTOR == XXH_AVX512) #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 #define XXH3_accumulate XXH3_accumulate_avx512 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 #elif (XXH_VECTOR == XXH_AVX2) #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 #define XXH3_accumulate XXH3_accumulate_avx2 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 #elif (XXH_VECTOR == XXH_SSE2) #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 #define XXH3_accumulate XXH3_accumulate_sse2 #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 #elif (XXH_VECTOR == XXH_NEON) #define XXH3_accumulate_512 XXH3_accumulate_512_neon #define XXH3_accumulate XXH3_accumulate_neon #define XXH3_scrambleAcc XXH3_scrambleAcc_neon #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #elif (XXH_VECTOR == XXH_VSX) #define XXH3_accumulate_512 XXH3_accumulate_512_vsx #define XXH3_accumulate XXH3_accumulate_vsx #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #elif (XXH_VECTOR == XXH_SVE) #define XXH3_accumulate_512 XXH3_accumulate_512_sve #define XXH3_accumulate XXH3_accumulate_sve #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #else /* scalar */ #define XXH3_accumulate_512 XXH3_accumulate_512_scalar #define XXH3_accumulate XXH3_accumulate_scalar #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #endif #if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ # undef XXH3_initCustomSecret # define XXH3_initCustomSecret XXH3_initCustomSecret_scalar #endif XXH_FORCE_INLINE void XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; size_t const nb_blocks = (len - 1) / block_len; size_t n; XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); for (n = 0; n < nb_blocks; n++) { f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); } /* last partial block */ XXH_ASSERT(len > XXH_STRIPE_LEN); { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); /* last stripe */ { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; #define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); } } } XXH_FORCE_INLINE xxh_u64 XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) { return XXH3_mul128_fold64( acc[0] ^ XXH_readLE64(secret), acc[1] ^ XXH_readLE64(secret+8) ); } static XXH64_hash_t XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) { xxh_u64 result64 = start; size_t i = 0; for (i = 0; i < 4; i++) { result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); #if defined(__clang__) /* Clang */ \ && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Prevent autovectorization on Clang ARMv7-a. Exact same problem as * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. * XXH3_64bits, len == 256, Snapdragon 835: * without hack: 2063.7 MB/s * with hack: 2560.7 MB/s */ XXH_COMPILER_GUARD(result64); #endif } return XXH3_avalanche(result64); } #define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, const void* XXH_RESTRICT secret, size_t secretSize, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); /* do not align on 8, so that the secret is different from the accumulator */ #define XXH_SECRET_MERGEACCS_START 11 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); } /* * It's important for performance to transmit secret's size (when it's static) * so that the compiler can properly optimize the vectorized loop. * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE * breaks -Og, this is XXH_NO_INLINE. */ XXH3_WITH_SECRET_INLINE XXH64_hash_t XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); } /* * It's preferable for performance that XXH3_hashLong is not inlined, * as it results in a smaller function for small data, easier to the instruction cache. * Note that inside this no_inline function, we do inline the internal loop, * and provide a statically defined secret size to allow optimization of vector loop. */ XXH_NO_INLINE XXH_PUREF XXH64_hash_t XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; (void)secret; (void)secretLen; return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); } /* * XXH3_hashLong_64b_withSeed(): * Generate a custom key based on alteration of default XXH3_kSecret with the seed, * and then use this key for long mode hashing. * * This operation is decently fast but nonetheless costs a little bit of time. * Try to avoid it whenever possible (typically when seed==0). * * It's important for performance that XXH3_hashLong is not inlined. Not sure * why (uop cache maybe?), but the difference is large and easily measurable. */ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, XXH64_hash_t seed, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { #if XXH_SIZE_OPT <= 0 if (seed == 0) return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc, f_scramble); #endif { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; f_initSec(secret, seed); return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), f_acc, f_scramble); } } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH64_hash_t XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) { (void)secret; (void)secretLen; return XXH3_hashLong_64b_withSeed_internal(input, len, seed, XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); } typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); XXH_FORCE_INLINE XXH64_hash_t XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, XXH3_hashLong64_f f_hashLong) { XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); /* * If an action is to be taken if `secretLen` condition is not respected, * it should be done here. * For now, it's a contract pre-condition. * Adding a check and a branch here would cost performance at every hash. * Also, note that function signature doesn't offer room to return an error. */ if (len <= 16) return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); if (len <= 128) return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); } /* === Public entry point === */ /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) { return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) { return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) { return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); } XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) { if (length <= XXH3_MIDSIZE_MAX) return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); } /* === XXH3 streaming === */ #ifndef XXH_NO_STREAM /* * Malloc's a pointer that is always aligned to align. * * This must be freed with `XXH_alignedFree()`. * * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. * * This underalignment previously caused a rather obvious crash which went * completely unnoticed due to XXH3_createState() not actually being tested. * Credit to RedSpah for noticing this bug. * * The alignment is done manually: Functions like posix_memalign or _mm_malloc * are avoided: To maintain portability, we would have to write a fallback * like this anyways, and besides, testing for the existence of library * functions without relying on external build tools is impossible. * * The method is simple: Overallocate, manually align, and store the offset * to the original behind the returned pointer. * * Align must be a power of 2 and 8 <= align <= 128. */ static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) { XXH_ASSERT(align <= 128 && align >= 8); /* range check */ XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ { /* Overallocate to make room for manual realignment and an offset byte */ xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); if (base != NULL) { /* * Get the offset needed to align this pointer. * * Even if the returned pointer is aligned, there will always be * at least one byte to store the offset to the original pointer. */ size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ /* Add the offset for the now-aligned pointer */ xxh_u8* ptr = base + offset; XXH_ASSERT((size_t)ptr % align == 0); /* Store the offset immediately before the returned pointer. */ ptr[-1] = (xxh_u8)offset; return ptr; } return NULL; } } /* * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. */ static void XXH_alignedFree(void* p) { if (p != NULL) { xxh_u8* ptr = (xxh_u8*)p; /* Get the offset byte we added in XXH_malloc. */ xxh_u8 offset = ptr[-1]; /* Free the original malloc'd pointer */ xxh_u8* base = ptr - offset; XXH_free(base); } } /*! @ingroup XXH3_family */ /*! * @brief Allocate an @ref XXH3_state_t. * * @return An allocated pointer of @ref XXH3_state_t on success. * @return `NULL` on failure. * * @note Must be freed with XXH3_freeState(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) { XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); if (state==NULL) return NULL; XXH3_INITSTATE(state); return state; } /*! @ingroup XXH3_family */ /*! * @brief Frees an @ref XXH3_state_t. * * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). * * @return @ref XXH_OK. * * @note Must be allocated with XXH3_createState(). * * @see @ref streaming_example "Streaming Example" */ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) { XXH_alignedFree(statePtr); return XXH_OK; } /*! @ingroup XXH3_family */ XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) { XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); } static void XXH3_reset_internal(XXH3_state_t* statePtr, XXH64_hash_t seed, const void* secret, size_t secretSize) { size_t const initStart = offsetof(XXH3_state_t, bufferedSize); size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); XXH_ASSERT(statePtr != NULL); /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ memset((char*)statePtr + initStart, 0, initLength); statePtr->acc[0] = XXH_PRIME32_3; statePtr->acc[1] = XXH_PRIME64_1; statePtr->acc[2] = XXH_PRIME64_2; statePtr->acc[3] = XXH_PRIME64_3; statePtr->acc[4] = XXH_PRIME64_4; statePtr->acc[5] = XXH_PRIME32_2; statePtr->acc[6] = XXH_PRIME64_5; statePtr->acc[7] = XXH_PRIME32_1; statePtr->seed = seed; statePtr->useSeed = (seed != 0); statePtr->extSecret = (const unsigned char*)secret; XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) { if (statePtr == NULL) return XXH_ERROR; if (seed==0) return XXH3_64bits_reset(statePtr); if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) XXH3_initCustomSecret(statePtr->customSecret, seed); XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) { if (statePtr == NULL) return XXH_ERROR; if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; XXH3_reset_internal(statePtr, seed64, secret, secretSize); statePtr->useSeed = 1; /* always, even if seed64==0 */ return XXH_OK; } /*! * @internal * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). * * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. * * @param acc Pointer to the 8 accumulator lanes * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* * @param nbStripesPerBlock Number of stripes in a block * @param input Input pointer * @param nbStripes Number of stripes to process * @param secret Secret pointer * @param secretLimit Offset of the last block in @p secret * @param f_acc Pointer to an XXH3_accumulate implementation * @param f_scramble Pointer to an XXH3_scrambleAcc implementation * @return Pointer past the end of @p input after processing */ XXH_FORCE_INLINE const xxh_u8 * XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, const xxh_u8* XXH_RESTRICT input, size_t nbStripes, const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; /* Process full blocks */ if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { /* Process the initial partial block... */ size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; do { /* Accumulate and scramble */ f_acc(acc, input, initialSecret, nbStripesThisIter); f_scramble(acc, secret + secretLimit); input += nbStripesThisIter * XXH_STRIPE_LEN; nbStripes -= nbStripesThisIter; /* Then continue the loop with the full block size */ nbStripesThisIter = nbStripesPerBlock; initialSecret = secret; } while (nbStripes >= nbStripesPerBlock); *nbStripesSoFarPtr = 0; } /* Process a partial block */ if (nbStripes > 0) { f_acc(acc, input, initialSecret, nbStripes); input += nbStripes * XXH_STRIPE_LEN; *nbStripesSoFarPtr += nbStripes; } /* Return end pointer */ return input; } #ifndef XXH3_STREAM_USE_STACK # if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ # define XXH3_STREAM_USE_STACK 1 # endif #endif /* * Both XXH3_64bits_update and XXH3_128bits_update use this routine. */ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t* XXH_RESTRICT const state, const xxh_u8* XXH_RESTRICT input, size_t len, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { if (input==NULL) { XXH_ASSERT(len == 0); return XXH_OK; } XXH_ASSERT(state != NULL); { const xxh_u8* const bEnd = input + len; const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 /* For some reason, gcc and MSVC seem to suffer greatly * when operating accumulators directly into state. * Operating into stack space seems to enable proper optimization. * clang, on the other hand, doesn't seem to need this trick */ XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; XXH_memcpy(acc, state->acc, sizeof(acc)); #else xxh_u64* XXH_RESTRICT const acc = state->acc; #endif state->totalLen += len; XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); /* small input : just fill in tmp buffer */ if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { XXH_memcpy(state->buffer + state->bufferedSize, input, len); state->bufferedSize += (XXH32_hash_t)len; return XXH_OK; } /* total input is now > XXH3_INTERNALBUFFER_SIZE */ #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ /* * Internal buffer is partially filled (always, except at beginning) * Complete it, then consume it. */ if (state->bufferedSize) { size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); input += loadSize; XXH3_consumeStripes(acc, &state->nbStripesSoFar, state->nbStripesPerBlock, state->buffer, XXH3_INTERNALBUFFER_STRIPES, secret, state->secretLimit, f_acc, f_scramble); state->bufferedSize = 0; } XXH_ASSERT(input < bEnd); if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; input = XXH3_consumeStripes(acc, &state->nbStripesSoFar, state->nbStripesPerBlock, input, nbStripes, secret, state->secretLimit, f_acc, f_scramble); XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); } /* Some remaining input (always) : buffer it */ XXH_ASSERT(input < bEnd); XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); XXH_ASSERT(state->bufferedSize == 0); XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); state->bufferedSize = (XXH32_hash_t)(bEnd-input); #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 /* save stack accumulators into state */ XXH_memcpy(state->acc, acc, sizeof(acc)); #endif } return XXH_OK; } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) { return XXH3_update(state, (const xxh_u8*)input, len, XXH3_accumulate, XXH3_scrambleAcc); } XXH_FORCE_INLINE void XXH3_digest_long (XXH64_hash_t* acc, const XXH3_state_t* state, const unsigned char* secret) { xxh_u8 lastStripe[XXH_STRIPE_LEN]; const xxh_u8* lastStripePtr; /* * Digest on a local copy. This way, the state remains unaltered, and it can * continue ingesting more input afterwards. */ XXH_memcpy(acc, state->acc, sizeof(state->acc)); if (state->bufferedSize >= XXH_STRIPE_LEN) { /* Consume remaining stripes then point to remaining data in buffer */ size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; size_t nbStripesSoFar = state->nbStripesSoFar; XXH3_consumeStripes(acc, &nbStripesSoFar, state->nbStripesPerBlock, state->buffer, nbStripes, secret, state->secretLimit, XXH3_accumulate, XXH3_scrambleAcc); lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; } else { /* bufferedSize < XXH_STRIPE_LEN */ /* Copy to temp buffer */ size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); lastStripePtr = lastStripe; } /* Last stripe */ XXH3_accumulate_512(acc, lastStripePtr, secret + state->secretLimit - XXH_SECRET_LASTACC_START); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) { const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; if (state->totalLen > XXH3_MIDSIZE_MAX) { XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; XXH3_digest_long(acc, state, secret); return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)state->totalLen * XXH_PRIME64_1); } /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ if (state->useSeed) return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), secret, state->secretLimit + XXH_STRIPE_LEN); } #endif /* !XXH_NO_STREAM */ /* ========================================== * XXH3 128 bits (a.k.a XXH128) * ========================================== * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, * even without counting the significantly larger output size. * * For example, extra steps are taken to avoid the seed-dependent collisions * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). * * This strength naturally comes at the cost of some speed, especially on short * lengths. Note that longer hashes are about as fast as the 64-bit version * due to it using only a slight modification of the 64-bit loop. * * XXH128 is also more oriented towards 64-bit machines. It is still extremely * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). */ XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { /* A doubled version of 1to3_64b with different constants. */ XXH_ASSERT(input != NULL); XXH_ASSERT(1 <= len && len <= 3); XXH_ASSERT(secret != NULL); /* * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } */ { xxh_u8 const c1 = input[0]; xxh_u8 const c2 = input[len >> 1]; xxh_u8 const c3 = input[len - 1]; xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; XXH128_hash_t h128; h128.low64 = XXH64_avalanche(keyed_lo); h128.high64 = XXH64_avalanche(keyed_hi); return h128; } } XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(4 <= len && len <= 8); seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; { xxh_u32 const input_lo = XXH_readLE32(input); xxh_u32 const input_hi = XXH_readLE32(input + len - 4); xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; xxh_u64 const keyed = input_64 ^ bitflip; /* Shift len to the left to ensure it is even, this avoids even multiplies. */ XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); m128.high64 += (m128.low64 << 1); m128.low64 ^= (m128.high64 >> 3); m128.low64 = XXH_xorshift64(m128.low64, 35); m128.low64 *= PRIME_MX2; m128.low64 = XXH_xorshift64(m128.low64, 28); m128.high64 = XXH3_avalanche(m128.high64); return m128; } } XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); XXH_ASSERT(9 <= len && len <= 16); { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; xxh_u64 const input_lo = XXH_readLE64(input); xxh_u64 input_hi = XXH_readLE64(input + len - 8); XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); /* * Put len in the middle of m128 to ensure that the length gets mixed to * both the low and high bits in the 128x64 multiply below. */ m128.low64 += (xxh_u64)(len - 1) << 54; input_hi ^= bitfliph; /* * Add the high 32 bits of input_hi to the high 32 bits of m128, then * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to * the high 64 bits of m128. * * The best approach to this operation is different on 32-bit and 64-bit. */ if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ /* * 32-bit optimized version, which is more readable. * * On 32-bit, it removes an ADC and delays a dependency between the two * halves of m128.high64, but it generates an extra mask on 64-bit. */ m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); } else { /* * 64-bit optimized (albeit more confusing) version. * * Uses some properties of addition and multiplication to remove the mask: * * Let: * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) * c = XXH_PRIME32_2 * * a + (b * c) * Inverse Property: x + y - x == y * a + (b * (1 + c - 1)) * Distributive Property: x * (y + z) == (x * y) + (x * z) * a + (b * 1) + (b * (c - 1)) * Identity Property: x * 1 == x * a + b + (b * (c - 1)) * * Substitute a, b, and c: * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) * * Since input_hi.hi + input_hi.lo == input_hi, we get this: * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) */ m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); } /* m128 ^= XXH_swap64(m128 >> 64); */ m128.low64 ^= XXH_swap64(m128.high64); { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); h128.high64 += m128.high64 * XXH_PRIME64_2; h128.low64 = XXH3_avalanche(h128.low64); h128.high64 = XXH3_avalanche(h128.high64); return h128; } } } /* * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN */ XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) { XXH_ASSERT(len <= 16); { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); if (len) return XXH3_len_1to3_128b(input, len, secret, seed); { XXH128_hash_t h128; xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); h128.low64 = XXH64_avalanche(seed ^ bitflipl); h128.high64 = XXH64_avalanche( seed ^ bitfliph); return h128; } } } /* * A bit slower than XXH3_mix16B, but handles multiply by zero better. */ XXH_FORCE_INLINE XXH128_hash_t XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, const xxh_u8* secret, XXH64_hash_t seed) { acc.low64 += XXH3_mix16B (input_1, secret+0, seed); acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); acc.high64 += XXH3_mix16B (input_2, secret+16, seed); acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); return acc; } XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(16 < len && len <= 128); { XXH128_hash_t acc; acc.low64 = len * XXH_PRIME64_1; acc.high64 = 0; #if XXH_SIZE_OPT >= 1 { /* Smaller, but slightly slower. */ unsigned int i = (unsigned int)(len - 1) / 32; do { acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); } while (i-- != 0); } #else if (len > 32) { if (len > 64) { if (len > 96) { acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); } acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); } acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); } acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); #endif { XXH128_hash_t h128; h128.low64 = acc.low64 + acc.high64; h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2); h128.low64 = XXH3_avalanche(h128.low64); h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); return h128; } } } XXH_NO_INLINE XXH_PUREF XXH128_hash_t XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH64_hash_t seed) { XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); { XXH128_hash_t acc; unsigned i; acc.low64 = len * XXH_PRIME64_1; acc.high64 = 0; /* * We set as `i` as offset + 32. We do this so that unchanged * `len` can be used as upper bound. This reaches a sweet spot * where both x86 and aarch64 get simple agen and good codegen * for the loop. */ for (i = 32; i < 160; i += 32) { acc = XXH128_mix32B(acc, input + i - 32, input + i - 16, secret + i - 32, seed); } acc.low64 = XXH3_avalanche(acc.low64); acc.high64 = XXH3_avalanche(acc.high64); /* * NB: `i <= len` will duplicate the last 32-bytes if * len % 32 was zero. This is an unfortunate necessity to keep * the hash result stable. */ for (i=160; i <= len; i += 32) { acc = XXH128_mix32B(acc, input + i - 32, input + i - 16, secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, seed); } /* last bytes */ acc = XXH128_mix32B(acc, input + len - 16, input + len - 32, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, (XXH64_hash_t)0 - seed); { XXH128_hash_t h128; h128.low64 = acc.low64 + acc.high64; h128.high64 = (acc.low64 * XXH_PRIME64_1) + (acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2); h128.low64 = XXH3_avalanche(h128.low64); h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); return h128; } } } XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble) { XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); { XXH128_hash_t h128; h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); h128.high64 = XXH3_mergeAccs(acc, secret + secretSize - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((xxh_u64)len * XXH_PRIME64_2)); return h128; } } /* * It's important for performance that XXH3_hashLong() is not inlined. */ XXH_NO_INLINE XXH_PUREF XXH128_hash_t XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; (void)secret; (void)secretLen; return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); } /* * It's important for performance to pass @p secretLen (when it's static) * to the compiler, so that it can properly optimize the vectorized loop. * * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE * breaks -Og, this is XXH_NO_INLINE. */ XXH3_WITH_SECRET_INLINE XXH128_hash_t XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { (void)seed64; return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); } XXH_FORCE_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, XXH64_hash_t seed64, XXH3_f_accumulate f_acc, XXH3_f_scrambleAcc f_scramble, XXH3_f_initCustomSecret f_initSec) { if (seed64 == 0) return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), f_acc, f_scramble); { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; f_initSec(secret, seed64); return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), f_acc, f_scramble); } } /* * It's important for performance that XXH3_hashLong is not inlined. */ XXH_NO_INLINE XXH128_hash_t XXH3_hashLong_128b_withSeed(const void* input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) { (void)secret; (void)secretLen; return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); } typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, XXH64_hash_t, const void* XXH_RESTRICT, size_t); XXH_FORCE_INLINE XXH128_hash_t XXH3_128bits_internal(const void* input, size_t len, XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, XXH3_hashLong128_f f_hl128) { XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); /* * If an action is to be taken if `secret` conditions are not respected, * it should be done here. * For now, it's a contract pre-condition. * Adding a check and a branch here would cost performance at every hash. */ if (len <= 16) return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); if (len <= 128) return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); if (len <= XXH3_MIDSIZE_MAX) return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); return f_hl128(input, len, seed64, secret, secretLen); } /* === Public XXH128 API === */ /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) { return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_default); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) { return XXH3_128bits_internal(input, len, 0, (const xxh_u8*)secret, secretSize, XXH3_hashLong_128b_withSecret); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) { return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) { if (len <= XXH3_MIDSIZE_MAX) return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) { return XXH3_128bits_withSeed(input, len, seed); } /* === XXH3 128-bit streaming === */ #ifndef XXH_NO_STREAM /* * All initialization and update functions are identical to 64-bit streaming variant. * The only difference is the finalization routine. */ /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) { return XXH3_64bits_reset(statePtr); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) { return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) { return XXH3_64bits_reset_withSeed(statePtr, seed); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) { return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) { return XXH3_64bits_update(state, input, len); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) { const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; if (state->totalLen > XXH3_MIDSIZE_MAX) { XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; XXH3_digest_long(acc, state, secret); XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); { XXH128_hash_t h128; h128.low64 = XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)state->totalLen * XXH_PRIME64_1); h128.high64 = XXH3_mergeAccs(acc, secret + state->secretLimit + XXH_STRIPE_LEN - sizeof(acc) - XXH_SECRET_MERGEACCS_START, ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); return h128; } } /* len <= XXH3_MIDSIZE_MAX : short code */ if (state->seed) return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), secret, state->secretLimit + XXH_STRIPE_LEN); } #endif /* !XXH_NO_STREAM */ /* 128-bit utility functions */ #include /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ /*! @ingroup XXH3_family */ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { /* note : XXH128_hash_t is compact, it has no padding byte */ return !(memcmp(&h1, &h2, sizeof(h1))); } /* This prototype is compatible with stdlib's qsort(). * @return : >0 if *h128_1 > *h128_2 * <0 if *h128_1 < *h128_2 * =0 if *h128_1 == *h128_2 */ /*! @ingroup XXH3_family */ XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) { XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); /* note : bets that, in most cases, hash values are different */ if (hcmp) return hcmp; return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); } /*====== Canonical representation ======*/ /*! @ingroup XXH3_family */ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) { XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) { hash.high64 = XXH_swap64(hash.high64); hash.low64 = XXH_swap64(hash.low64); } XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) { XXH128_hash_t h; h.high64 = XXH_readBE64(src); h.low64 = XXH_readBE64(src->digest + 8); return h; } /* ========================================== * Secret generators * ========================================== */ #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) { XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); } /*! @ingroup XXH3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) { #if (XXH_DEBUGLEVEL >= 1) XXH_ASSERT(secretBuffer != NULL); XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); #else /* production mode, assert() are disabled */ if (secretBuffer == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; #endif if (customSeedSize == 0) { customSeed = XXH3_kSecret; customSeedSize = XXH_SECRET_DEFAULT_SIZE; } #if (XXH_DEBUGLEVEL >= 1) XXH_ASSERT(customSeed != NULL); #else if (customSeed == NULL) return XXH_ERROR; #endif /* Fill secretBuffer with a copy of customSeed - repeat as needed */ { size_t pos = 0; while (pos < secretSize) { size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); memcpy((char*)secretBuffer + pos, customSeed, toCopy); pos += toCopy; } } { size_t const nbSeg16 = secretSize / 16; size_t n; XXH128_canonical_t scrambler; XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); for (n=0; n #include #define BLAKE3_NO_AVX512 #define BLAKE3_NO_SSE41 #define BLAKE3_NO_AVX2 #ifdef __cplusplus extern "C" { #endif #define BLAKE3_KEY_LEN 32 #define BLAKE3_OUT_LEN 32 #define BLAKE3_BLOCK_LEN 64 #define BLAKE3_CHUNK_LEN 1024 #define BLAKE3_MAX_DEPTH 54 #define BLAKE3_MAX_SIMD_DEGREE 16 // This struct is a private implementation detail. It has to be here because // it's part of blake3_hasher below. typedef struct { uint32_t cv[8]; uint64_t chunk_counter; uint8_t buf[BLAKE3_BLOCK_LEN]; uint8_t buf_len; uint8_t blocks_compressed; uint8_t flags; } blake3_chunk_state; typedef struct { uint32_t key[8]; blake3_chunk_state chunk; uint8_t cv_stack_len; // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk // requires a 4th entry, rather than merging everything down to 1, because we // don't know whether more input is coming. This is different from how the // reference implementation does things. uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; } blake3_hasher; void blake3_hasher_init(blake3_hasher *self); void blake3_hasher_init_keyed(blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); void blake3_hasher_update(blake3_hasher *self, const void *input, size_t input_len); void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, size_t out_len); void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len); #ifdef __cplusplus } #endif #endif /* BLAKE3_H */ digest/src/crc32c.cpp0000644000176200001440000000247114423472174014117 0ustar liggesusers// Copyright 2017 The CRC32C Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. //#include "crc32c/crc32c.h" #include "crc32c.h" #include #include // #include "./crc32c_arm64.h" // #include "./crc32c_arm64_check.h" // #include "./crc32c_internal.h" // #include "./crc32c_sse42.h" // #include "./crc32c_sse42_check.h" #include "crc32c/crc32c_internal.h" namespace crc32c { uint32_t Extend(uint32_t crc, const uint8_t* data, size_t count) { // #if HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__)) // static bool can_use_sse42 = CanUseSse42(); // if (can_use_sse42) return ExtendSse42(crc, data, count); // #elif HAVE_ARM64_CRC32C // static bool can_use_arm64_crc32 = CanUseArm64Crc32(); // if (can_use_arm64_crc32) return ExtendArm64(crc, data, count); // #endif // HAVE_SSE42 && (defined(_M_X64) || defined(__x86_64__)) return ExtendPortable(crc, data, count); } extern "C" uint32_t crc32c_extend(uint32_t crc, const uint8_t* data, size_t count) { return crc32c::Extend(crc, data, count); } extern "C" uint32_t crc32c_value(const uint8_t* data, size_t count) { return crc32c::Crc32c(data, count); } } // namespace crc32c digest/src/pmurhash.h0000644000176200001440000000367612444156327014344 0ustar liggesusers/*----------------------------------------------------------------------------- * MurmurHash3 was written by Austin Appleby, and is placed in the public * domain. * * This implementation was written by Shane Day, and is also public domain. * * This is a portable ANSI C implementation of MurmurHash3_x86_32 (Murmur3A) * with support for progressive processing. */ /* ------------------------------------------------------------------------- */ /* Determine what native type to use for uint32_t */ /* We can't use the name 'uint32_t' here because it will conflict with * any version provided by the system headers or application. */ /* First look for special cases */ #if defined(_MSC_VER) #define MH_UINT32 unsigned long #endif /* If the compiler says it's C99 then take its word for it */ #if !defined(MH_UINT32) && ( \ defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L ) #include #define MH_UINT32 uint32_t #endif /* Otherwise try testing against max value macros from limit.h */ #if !defined(MH_UINT32) #include #if (USHRT_MAX == 0xffffffffUL) #define MH_UINT32 unsigned short #elif (UINT_MAX == 0xffffffffUL) #define MH_UINT32 unsigned int #elif (ULONG_MAX == 0xffffffffUL) #define MH_UINT32 unsigned long #endif #endif #if !defined(MH_UINT32) #error Unable to determine type name for unsigned 32-bit int #endif /* I'm yet to work on a platform where 'unsigned char' is not 8 bits */ #define MH_UINT8 unsigned char /* ------------------------------------------------------------------------- */ /* Prototypes */ #ifdef __cplusplus extern "C" { #endif void PMurHash32_Process(MH_UINT32 *ph1, MH_UINT32 *pcarry, const void *key, int len); MH_UINT32 PMurHash32_Result(MH_UINT32 h1, MH_UINT32 carry, MH_UINT32 total_length); MH_UINT32 PMurHash32(MH_UINT32 seed, const void *key, int len); void PMurHash32_test(const void *key, int len, MH_UINT32 seed, void *out); #ifdef __cplusplus } #endif digest/src/sha2.h0000644000176200001440000001606513343737703013350 0ustar liggesusers/* * FILE: sha2.h * AUTHOR: Aaron D. Gifford - http://www.aarongifford.com/ * * Copyright (c) 2000-2001, Aaron D. Gifford * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTOR(S) ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTOR(S) BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Id: sha2.h,v 1.1 2001/11/08 00:02:01 adg Exp adg $ */ #ifndef __SHA2_H__ #define __SHA2_H__ #ifdef __cplusplus extern "C" { #endif #include /* configuration settings learned when R was compiled */ /* * Import u_intXX_t size_t type definitions from system headers. You * may need to change this, or define these things yourself in this * file. */ #include #ifdef SHA2_USE_INTTYPES_H #include #endif /* SHA2_USE_INTTYPES_H */ /*** SHA-256/384/512 Various Length Definitions ***********************/ #define SHA256_BLOCK_LENGTH 64 #define SHA256_DIGEST_LENGTH 32 #define SHA256_DIGEST_STRING_LENGTH (SHA256_DIGEST_LENGTH * 2 + 1) #define SHA384_BLOCK_LENGTH 128 #define SHA384_DIGEST_LENGTH 48 #define SHA384_DIGEST_STRING_LENGTH (SHA384_DIGEST_LENGTH * 2 + 1) #define SHA512_BLOCK_LENGTH 128 #define SHA512_DIGEST_LENGTH 64 #define SHA512_DIGEST_STRING_LENGTH (SHA512_DIGEST_LENGTH * 2 + 1) /*** SHA-256/384/512 Context Structures *******************************/ /* NOTE: If your architecture does not define either u_intXX_t types or * uintXX_t (from inttypes.h), you may need to define things by hand * for your system: */ // edd 24 Nov 2012 -- gcc on Windows does not know u_int_{8,32,64}_t #define SHA2_USE_INTTYPES_H 1 #include #ifdef WIN32 #ifndef BYTE_ORDER #define LITTLE_ENDIAN 1234 #define BIG_ENDIAN 4321 #define BYTE_ORDER LITTLE_ENDIAN #endif #endif // edd 02 Dec 2013 use Rconfig.h to define BYTE_ORDER, unless already defined #ifndef BYTE_ORDER // see sha2.c comments, and on the internet at large #define LITTLE_ENDIAN 1234 #define BIG_ENDIAN 4321 #ifdef WORDS_BIGENDIAN #define BYTE_ORDER BIG_ENDIAN #else #define BYTE_ORDER LITTLE_ENDIAN #endif #endif #if 0 typedef unsigned char u_int8_t; /* 1-byte (8-bits) */ typedef unsigned int u_int32_t; /* 4-bytes (32-bits) */ typedef unsigned long long u_int64_t; /* 8-bytes (64-bits) */ #endif /* * Most BSD systems already define u_intXX_t types, as does Linux. * Some systems, however, like Compaq's Tru64 Unix instead can use * uintXX_t types defined by very recent ANSI C standards and included * in the file: * * #include * * If you choose to use then please define: * * #define SHA2_USE_INTTYPES_H * * Or on the command line during compile: * * cc -DSHA2_USE_INTTYPES_H ... */ #ifdef SHA2_USE_INTTYPES_H typedef struct _SHA256_CTX { uint32_t state[8]; uint64_t bitcount; uint8_t buffer[SHA256_BLOCK_LENGTH]; /* must be 32-bit aligned */ } SHA256_CTX; typedef struct _SHA512_CTX { uint64_t state[8]; uint64_t bitcount[2]; uint8_t buffer[SHA512_BLOCK_LENGTH]; /* must be 64-bit aligned */ } SHA512_CTX; #else /* SHA2_USE_INTTYPES_H */ typedef struct _SHA256_CTX { u_int32_t state[8]; u_int64_t bitcount; u_int8_t buffer[SHA256_BLOCK_LENGTH]; /* must be 32-bit aligned */ } SHA256_CTX; typedef struct _SHA512_CTX { u_int64_t state[8]; u_int64_t bitcount[2]; u_int8_t buffer[SHA512_BLOCK_LENGTH]; /* must be 64-bit aligned */ } SHA512_CTX; #endif /* SHA2_USE_INTTYPES_H */ typedef SHA512_CTX SHA384_CTX; /*** SHA-256/384/512 Function Prototypes ******************************/ #ifndef NOPROTO #ifdef SHA2_USE_INTTYPES_H void SHA256_Init(SHA256_CTX *); void SHA256_Update(SHA256_CTX*, const uint8_t*, size_t); void SHA256_Final(uint8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*); char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]); char* SHA256_Data(const uint8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]); void SHA384_Init(SHA384_CTX*); void SHA384_Update(SHA384_CTX*, const uint8_t*, size_t); void SHA384_Final(uint8_t[SHA384_DIGEST_LENGTH], SHA384_CTX*); char* SHA384_End(SHA384_CTX*, char[SHA384_DIGEST_STRING_LENGTH]); char* SHA384_Data(const uint8_t*, size_t, char[SHA384_DIGEST_STRING_LENGTH]); void SHA512_Init(SHA512_CTX*); void SHA512_Update(SHA512_CTX*, const uint8_t*, size_t); void SHA512_Final(uint8_t[SHA512_DIGEST_LENGTH], SHA512_CTX*); char* SHA512_End(SHA512_CTX*, char[SHA512_DIGEST_STRING_LENGTH]); char* SHA512_Data(const uint8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]); #else /* SHA2_USE_INTTYPES_H */ void SHA256_Init(SHA256_CTX *); void SHA256_Update(SHA256_CTX*, const u_int8_t*, size_t); void SHA256_Final(u_int8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*); char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]); char* SHA256_Data(const u_int8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]); void SHA384_Init(SHA384_CTX*); void SHA384_Update(SHA384_CTX*, const u_int8_t*, size_t); void SHA384_Final(u_int8_t[SHA384_DIGEST_LENGTH], SHA384_CTX*); char* SHA384_End(SHA384_CTX*, char[SHA384_DIGEST_STRING_LENGTH]); char* SHA384_Data(const u_int8_t*, size_t, char[SHA384_DIGEST_STRING_LENGTH]); void SHA512_Init(SHA512_CTX*); void SHA512_Update(SHA512_CTX*, const u_int8_t*, size_t); void SHA512_Final(u_int8_t[SHA512_DIGEST_LENGTH], SHA512_CTX*); char* SHA512_End(SHA512_CTX*, char[SHA512_DIGEST_STRING_LENGTH]); char* SHA512_Data(const u_int8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]); #endif /* SHA2_USE_INTTYPES_H */ #else /* NOPROTO */ void SHA256_Init(); void SHA256_Update(); void SHA256_Final(); char* SHA256_End(); char* SHA256_Data(); void SHA384_Init(); void SHA384_Update(); void SHA384_Final(); char* SHA384_End(); char* SHA384_Data(); void SHA512_Init(); void SHA512_Update(); void SHA512_Final(); char* SHA512_End(); char* SHA512_Data(); #endif /* NOPROTO */ #ifdef __cplusplus } #endif /* __cplusplus */ #endif /* __SHA2_H__ */ digest/src/SpookyV2.cpp0000644000176200001440000001702314621373425014532 0ustar liggesusers// Spooky Hash // A 128-bit noncryptographic hash, for checksums and table lookup // By Bob Jenkins. Public domain. // Oct 31 2010: published framework, disclaimer ShortHash isn't right // Nov 7 2010: disabled ShortHash // Oct 31 2011: replace End, ShortMix, ShortEnd, enable ShortHash again // April 10 2012: buffer overflow on platforms without unaligned reads // July 12 2012: was passing out variables in final to in/out in short // July 30 2012: I reintroduced the buffer overflow // August 5 2012: SpookyV2: d = should be d += in short hash, and remove extra mix from long hash // Kendon Bell April 30 2019: Added counters to help with skipping first n bytes // Kendon Bell May 04 2019: Deleted original source code that doesn't get used #include #include "SpookyV2.h" // DE 2021-01-16 This defaults to 1 but is seen to trigger SAN warnings (c.f. GitHub issue #167) #define ALLOW_UNALIGNED_READS 0 // // short hash ... it could be used on any message, // but it's used by Spooky just for short messages. // void SpookyHash::Short( const void *message, size_t length, uint64 *hash1, uint64 *hash2) { uint64 buf[2*sc_numVars]; union { const uint8 *p8; uint32 *p32; uint64 *p64; size_t i; } u; u.p8 = (const uint8 *)message; if (!ALLOW_UNALIGNED_READS && (u.i & 0x7)) { memcpy(buf, message, length); u.p64 = buf; } size_t remainder = length%32; uint64 a=*hash1; uint64 b=*hash2; uint64 c=sc_const; uint64 d=sc_const; if (length > 15) { const uint64 *end = u.p64 + (length/32)*4; // handle all complete sets of 32 bytes for (; u.p64 < end; u.p64 += 4) { c += u.p64[0]; d += u.p64[1]; ShortMix(a,b,c,d); a += u.p64[2]; b += u.p64[3]; } //Handle the case of 16+ remaining bytes. if (remainder >= 16) { c += u.p64[0]; d += u.p64[1]; ShortMix(a,b,c,d); u.p64 += 2; remainder -= 16; } } // Handle the last 0..15 bytes, and its length d += ((uint64)length) << 56; switch (remainder) { case 15: d += ((uint64)u.p8[14]) << 48; // #nocov case 14: d += ((uint64)u.p8[13]) << 40; case 13: d += ((uint64)u.p8[12]) << 32; case 12: d += u.p32[2]; c += u.p64[0]; break; case 11: d += ((uint64)u.p8[10]) << 16; // #nocov case 10: d += ((uint64)u.p8[9]) << 8; case 9: d += (uint64)u.p8[8]; case 8: c += u.p64[0]; break; case 7: c += ((uint64)u.p8[6]) << 48; // #nocov case 6: c += ((uint64)u.p8[5]) << 40; // #nocov case 5: c += ((uint64)u.p8[4]) << 32; // #nocov case 4: c += u.p32[0]; // #nocov break; // #nocov case 3: c += ((uint64)u.p8[2]) << 16; case 2: c += ((uint64)u.p8[1]) << 8; case 1: c += (uint64)u.p8[0]; break; case 0: c += sc_const; d += sc_const; } ShortEnd(a,b,c,d); *hash1 = a; *hash2 = b; } // init spooky state void SpookyHash::Init(uint64 seed1, uint64 seed2, uint8 to_skip) { m_length = 0; m_remainder = 0; m_state[0] = seed1; m_state[1] = seed2; m_skipped = 0; m_to_skip = to_skip; } // add a message fragment to the state void SpookyHash::Update(const void *message, size_t length) { uint64 h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11; size_t newLength = length + m_remainder; uint8 remainder; union { const uint8 *p8; uint64 *p64; size_t i; } u; const uint64 *end; // Is this message fragment too short? If it is, stuff it away. if (newLength < sc_bufSize) { memcpy(&((uint8 *)m_data)[m_remainder], message, length); m_length = length + m_length; m_remainder = (uint8)newLength; return; } // init the variables if (m_length < sc_bufSize) { h0=h3=h6=h9 = m_state[0]; h1=h4=h7=h10 = m_state[1]; h2=h5=h8=h11 = sc_const; } else { h0 = m_state[0]; h1 = m_state[1]; h2 = m_state[2]; h3 = m_state[3]; h4 = m_state[4]; h5 = m_state[5]; h6 = m_state[6]; h7 = m_state[7]; h8 = m_state[8]; h9 = m_state[9]; h10 = m_state[10]; h11 = m_state[11]; } m_length = length + m_length; // if we've got anything stuffed away, use it now if (m_remainder) { uint8 prefix = sc_bufSize-m_remainder; memcpy(&(((uint8 *)m_data)[m_remainder]), message, prefix); u.p64 = m_data; Mix(u.p64, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); Mix(&u.p64[sc_numVars], h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); u.p8 = ((const uint8 *)message) + prefix; length -= prefix; } else { u.p8 = (const uint8 *)message; // #nocov } // handle all whole blocks of sc_blockSize bytes end = u.p64 + (length/sc_blockSize)*sc_numVars; remainder = (uint8)(length-((const uint8 *)end-u.p8)); if (ALLOW_UNALIGNED_READS || (u.i & 0x7) == 0) { while (u.p64 < end) { Mix(u.p64, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); u.p64 += sc_numVars; } } else { while (u.p64 < end) { memcpy(m_data, u.p8, sc_blockSize); Mix(m_data, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); u.p64 += sc_numVars; } } // stuff away the last few bytes m_remainder = remainder; memcpy(m_data, end, remainder); // stuff away the variables m_state[0] = h0; m_state[1] = h1; m_state[2] = h2; m_state[3] = h3; m_state[4] = h4; m_state[5] = h5; m_state[6] = h6; m_state[7] = h7; m_state[8] = h8; m_state[9] = h9; m_state[10] = h10; m_state[11] = h11; } // report the hash for the concatenation of all message fragments so far void SpookyHash::Final(uint64 *hash1, uint64 *hash2) { // init the variables if (m_length < sc_bufSize) { *hash1 = m_state[0]; *hash2 = m_state[1]; Short( m_data, m_length, hash1, hash2); return; } const uint64 *data = (const uint64 *)m_data; uint8 remainder = m_remainder; uint64 h0 = m_state[0]; uint64 h1 = m_state[1]; uint64 h2 = m_state[2]; uint64 h3 = m_state[3]; uint64 h4 = m_state[4]; uint64 h5 = m_state[5]; uint64 h6 = m_state[6]; uint64 h7 = m_state[7]; uint64 h8 = m_state[8]; uint64 h9 = m_state[9]; uint64 h10 = m_state[10]; uint64 h11 = m_state[11]; if (remainder >= sc_blockSize) { // m_data can contain two blocks; handle any whole first block Mix(data, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); // #nocov start data += sc_numVars; remainder -= sc_blockSize; // #nocov end } // mix in the last partial block, and the length mod sc_blockSize memset(&((uint8 *)data)[remainder], 0, (sc_blockSize-remainder)); ((uint8 *)data)[sc_blockSize-1] = remainder; // do some final mixing End(data, h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); *hash1 = h0; *hash2 = h1; } void SpookyHash::UpdateSkipCounter(size_t length) { m_skipped = static_cast(m_skipped + length); } void SpookyHash::GetSkipCounter(uint8 *count) { *count = m_skipped; } void SpookyHash::GetToSkip(uint8 *to_skip) { *to_skip = m_to_skip; } digest/src/SpookyV2.h0000644000176200001440000002644413463366440014210 0ustar liggesusers// // SpookyHash: a 128-bit noncryptographic hash function // By Bob Jenkins, public domain // Oct 31 2010: alpha, framework + SpookyHash::Mix appears right // Oct 31 2011: alpha again, Mix only good to 2^^69 but rest appears right // Dec 31 2011: beta, improved Mix, tested it for 2-bit deltas // Feb 2 2012: production, same bits as beta // Feb 5 2012: adjusted definitions of uint* to be more portable // Mar 30 2012: 3 bytes/cycle, not 4. Alpha was 4 but wasn't thorough enough. // August 5 2012: SpookyV2 (different results) // Kendon Bell April 30 2019: Added counters to help with skipping first n bytes // Kendon Bell May 04 2019: Deleted original source code that doesn't get used // // Up to 3 bytes/cycle for long messages. Reasonably fast for short messages. // All 1 or 2 bit deltas achieve avalanche within 1% bias per output bit. // // This was developed for and tested on 64-bit x86-compatible processors. // It assumes the processor is little-endian. There is a macro // controlling whether unaligned reads are allowed (by default they are). // This should be an equally good hash on big-endian machines, but it will // compute different results on them than on little-endian machines. // // Google's CityHash has similar specs to SpookyHash, and CityHash is faster // on new Intel boxes. MD4 and MD5 also have similar specs, but they are orders // of magnitude slower. CRCs are two or more times slower, but unlike // SpookyHash, they have nice math for combining the CRCs of pieces to form // the CRCs of wholes. There are also cryptographic hashes, but those are even // slower than MD5. // #include #ifdef _MSC_VER # define INLINE __forceinline typedef unsigned __int64 uint64; typedef unsigned __int32 uint32; typedef unsigned __int16 uint16; typedef unsigned __int8 uint8; #else # include # define INLINE inline typedef uint64_t uint64; typedef uint32_t uint32; typedef uint16_t uint16; typedef uint8_t uint8; #endif class SpookyHash { public: // // Init: initialize the context of a SpookyHash // void Init( uint64 seed1, // any 64-bit value will do, including 0 uint64 seed2, uint8 to_skip); // different seeds produce independent hashes // // Update: add a piece of a message to a SpookyHash state // void Update( const void *message, // message fragment size_t length); // length of message fragment in bytes // // Final: compute the hash for the current SpookyHash state // // This does not modify the state; you can keep updating it afterward // // The result is the same as if SpookyHash() had been called with // all the pieces concatenated into one message. // void Final( uint64 *hash1, // out only: first 64 bits of hash value. uint64 *hash2); // out only: second 64 bits of hash value. // // left rotate a 64-bit value by k bytes // static INLINE uint64 Rot64(uint64 x, int k) { return (x << k) | (x >> (64 - k)); } // // This is used if the input is 96 bytes long or longer. // // The internal state is fully overwritten every 96 bytes. // Every input bit appears to cause at least 128 bits of entropy // before 96 other bytes are combined, when run forward or backward // For every input bit, // Two inputs differing in just that input bit // Where "differ" means xor or subtraction // And the base value is random // When run forward or backwards one Mix // I tried 3 pairs of each; they all differed by at least 212 bits. // static INLINE void Mix( const uint64 *data, uint64 &s0, uint64 &s1, uint64 &s2, uint64 &s3, uint64 &s4, uint64 &s5, uint64 &s6, uint64 &s7, uint64 &s8, uint64 &s9, uint64 &s10,uint64 &s11) { s0 += data[0]; s2 ^= s10; s11 ^= s0; s0 = Rot64(s0,11); s11 += s1; s1 += data[1]; s3 ^= s11; s0 ^= s1; s1 = Rot64(s1,32); s0 += s2; s2 += data[2]; s4 ^= s0; s1 ^= s2; s2 = Rot64(s2,43); s1 += s3; s3 += data[3]; s5 ^= s1; s2 ^= s3; s3 = Rot64(s3,31); s2 += s4; s4 += data[4]; s6 ^= s2; s3 ^= s4; s4 = Rot64(s4,17); s3 += s5; s5 += data[5]; s7 ^= s3; s4 ^= s5; s5 = Rot64(s5,28); s4 += s6; s6 += data[6]; s8 ^= s4; s5 ^= s6; s6 = Rot64(s6,39); s5 += s7; s7 += data[7]; s9 ^= s5; s6 ^= s7; s7 = Rot64(s7,57); s6 += s8; s8 += data[8]; s10 ^= s6; s7 ^= s8; s8 = Rot64(s8,55); s7 += s9; s9 += data[9]; s11 ^= s7; s8 ^= s9; s9 = Rot64(s9,54); s8 += s10; s10 += data[10]; s0 ^= s8; s9 ^= s10; s10 = Rot64(s10,22); s9 += s11; s11 += data[11]; s1 ^= s9; s10 ^= s11; s11 = Rot64(s11,46); s10 += s0; } // // Mix all 12 inputs together so that h0, h1 are a hash of them all. // // For two inputs differing in just the input bits // Where "differ" means xor or subtraction // And the base value is random, or a counting value starting at that bit // The final result will have each bit of h0, h1 flip // For every input bit, // with probability 50 +- .3% // For every pair of input bits, // with probability 50 +- 3% // // This does not rely on the last Mix() call having already mixed some. // Two iterations was almost good enough for a 64-bit result, but a // 128-bit result is reported, so End() does three iterations. // static INLINE void EndPartial( uint64 &h0, uint64 &h1, uint64 &h2, uint64 &h3, uint64 &h4, uint64 &h5, uint64 &h6, uint64 &h7, uint64 &h8, uint64 &h9, uint64 &h10,uint64 &h11) { h11+= h1; h2 ^= h11; h1 = Rot64(h1,44); h0 += h2; h3 ^= h0; h2 = Rot64(h2,15); h1 += h3; h4 ^= h1; h3 = Rot64(h3,34); h2 += h4; h5 ^= h2; h4 = Rot64(h4,21); h3 += h5; h6 ^= h3; h5 = Rot64(h5,38); h4 += h6; h7 ^= h4; h6 = Rot64(h6,33); h5 += h7; h8 ^= h5; h7 = Rot64(h7,10); h6 += h8; h9 ^= h6; h8 = Rot64(h8,13); h7 += h9; h10^= h7; h9 = Rot64(h9,38); h8 += h10; h11^= h8; h10= Rot64(h10,53); h9 += h11; h0 ^= h9; h11= Rot64(h11,42); h10+= h0; h1 ^= h10; h0 = Rot64(h0,54); } static INLINE void End( const uint64 *data, uint64 &h0, uint64 &h1, uint64 &h2, uint64 &h3, uint64 &h4, uint64 &h5, uint64 &h6, uint64 &h7, uint64 &h8, uint64 &h9, uint64 &h10,uint64 &h11) { h0 += data[0]; h1 += data[1]; h2 += data[2]; h3 += data[3]; h4 += data[4]; h5 += data[5]; h6 += data[6]; h7 += data[7]; h8 += data[8]; h9 += data[9]; h10 += data[10]; h11 += data[11]; EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); EndPartial(h0,h1,h2,h3,h4,h5,h6,h7,h8,h9,h10,h11); } // // The goal is for each bit of the input to expand into 128 bits of // apparent entropy before it is fully overwritten. // n trials both set and cleared at least m bits of h0 h1 h2 h3 // n: 2 m: 29 // n: 3 m: 46 // n: 4 m: 57 // n: 5 m: 107 // n: 6 m: 146 // n: 7 m: 152 // when run forwards or backwards // for all 1-bit and 2-bit diffs // with diffs defined by either xor or subtraction // with a base of all zeros plus a counter, or plus another bit, or random // static INLINE void ShortMix(uint64 &h0, uint64 &h1, uint64 &h2, uint64 &h3) { h2 = Rot64(h2,50); h2 += h3; h0 ^= h2; h3 = Rot64(h3,52); h3 += h0; h1 ^= h3; h0 = Rot64(h0,30); h0 += h1; h2 ^= h0; h1 = Rot64(h1,41); h1 += h2; h3 ^= h1; h2 = Rot64(h2,54); h2 += h3; h0 ^= h2; h3 = Rot64(h3,48); h3 += h0; h1 ^= h3; h0 = Rot64(h0,38); h0 += h1; h2 ^= h0; h1 = Rot64(h1,37); h1 += h2; h3 ^= h1; h2 = Rot64(h2,62); h2 += h3; h0 ^= h2; h3 = Rot64(h3,34); h3 += h0; h1 ^= h3; h0 = Rot64(h0,5); h0 += h1; h2 ^= h0; h1 = Rot64(h1,36); h1 += h2; h3 ^= h1; } // // Mix all 4 inputs together so that h0, h1 are a hash of them all. // // For two inputs differing in just the input bits // Where "differ" means xor or subtraction // And the base value is random, or a counting value starting at that bit // The final result will have each bit of h0, h1 flip // For every input bit, // with probability 50 +- .3% (it is probably better than that) // For every pair of input bits, // with probability 50 +- .75% (the worst case is approximately that) // static INLINE void ShortEnd(uint64 &h0, uint64 &h1, uint64 &h2, uint64 &h3) { h3 ^= h2; h2 = Rot64(h2,15); h3 += h2; h0 ^= h3; h3 = Rot64(h3,52); h0 += h3; h1 ^= h0; h0 = Rot64(h0,26); h1 += h0; h2 ^= h1; h1 = Rot64(h1,51); h2 += h1; h3 ^= h2; h2 = Rot64(h2,28); h3 += h2; h0 ^= h3; h3 = Rot64(h3,9); h0 += h3; h1 ^= h0; h0 = Rot64(h0,47); h1 += h0; h2 ^= h1; h1 = Rot64(h1,54); h2 += h1; h3 ^= h2; h2 = Rot64(h2,32); h3 += h2; h0 ^= h3; h3 = Rot64(h3,25); h0 += h3; h1 ^= h0; h0 = Rot64(h0,63); h1 += h0; } // Added by Kendon Bell 2019-04-30 // UpdateSkipCounter: update the skip counter // GetSkipCounter: get the skip counter // void UpdateSkipCounter( size_t length); // length of message fragment skipped void GetSkipCounter( uint8 *count); // length of message fragment skipped void GetToSkip( uint8 *to_skip); // value of skip parameter private: // // Short is used for messages under 192 bytes in length // Short has a low startup cost, the normal mode is good for long // keys, the cost crossover is at about 192 bytes. The two modes were // held to the same quality bar. // static void Short( const void *message, // message (array of bytes, not necessarily aligned) size_t length, // length of message (in bytes) uint64 *hash1, // in/out: in the seed, out the hash value uint64 *hash2); // in/out: in the seed, out the hash value // number of uint64's in internal state static const size_t sc_numVars = 12; // size of the internal state static const size_t sc_blockSize = sc_numVars*8; // size of buffer of unhashed data, in bytes static const size_t sc_bufSize = 2*sc_blockSize; // // sc_const: a constant which: // * is not zero // * is odd // * is a not-very-regular mix of 1's and 0's // * does not need any other special mathematical properties // static const uint64 sc_const = 0xdeadbeefdeadbeefLL; uint64 m_data[2*sc_numVars]; // unhashed data, for partial messages uint64 m_state[sc_numVars]; // internal state of the hash size_t m_length; // total length of the input so far uint8 m_remainder; // length of unhashed data stashed in m_data // Added by Kendon Bell 2019-04-30 uint8 m_skipped; // length of original message already skipped uint8 m_to_skip; // parameter for how many bytes to skip at the front of the object }; digest/src/md5.h0000644000176200001440000000063512247504516013170 0ustar liggesusers#ifndef _MD5_H #define _MD5_H #ifndef uint8 #define uint8 unsigned char #endif #ifndef uint32 #define uint32 unsigned long int #endif typedef struct { uint32 total[2]; uint32 state[4]; uint8 buffer[64]; } md5_context; void md5_starts( md5_context *ctx ); void md5_update( md5_context *ctx, uint8 *input, uint32 length ); void md5_finish( md5_context *ctx, uint8 digest[16] ); #endif /* md5.h */ digest/src/sha256.h0000644000176200001440000000067312247504516013515 0ustar liggesusers#ifndef _SHA256_H #define _SHA256_H #ifndef uint8 #define uint8 unsigned char #endif #ifndef uint32 #define uint32 unsigned long int #endif typedef struct { uint32 total[2]; uint32 state[8]; uint8 buffer[64]; } sha256_context; void sha256_starts( sha256_context *ctx ); void sha256_update( sha256_context *ctx, uint8 *input, uint32 length ); void sha256_finish( sha256_context *ctx, uint8 digest[32] ); #endif /* sha256.h */ digest/src/blake3_portable.c0000755000176200001440000001407314635552730015537 0ustar liggesusers/* blake3 license: * The C code is copyright Samuel Neves and Jack O'Connor, 2019-2020. * This work is released into the public domain with CC0 1.0. * Alternatively, it is licensed under the Apache License 2.0. * For full license see * https://raw.githubusercontent.com/BLAKE3-team/BLAKE3/master/LICENSE */ #include "blake3_impl.h" #include INLINE uint32_t rotr32(uint32_t w, uint32_t c) { return (w >> c) | (w << (32 - c)); } INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, uint32_t x, uint32_t y) { state[a] = state[a] + state[b] + x; state[d] = rotr32(state[d] ^ state[a], 16); state[c] = state[c] + state[d]; state[b] = rotr32(state[b] ^ state[c], 12); state[a] = state[a] + state[b] + y; state[d] = rotr32(state[d] ^ state[a], 8); state[c] = state[c] + state[d]; state[b] = rotr32(state[b] ^ state[c], 7); } INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { // Select the message schedule based on the round. const uint8_t *schedule = MSG_SCHEDULE[round]; // Mix the columns. g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); // Mix the rows. g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); } INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags) { uint32_t block_words[16]; block_words[0] = load32(block + 4 * 0); block_words[1] = load32(block + 4 * 1); block_words[2] = load32(block + 4 * 2); block_words[3] = load32(block + 4 * 3); block_words[4] = load32(block + 4 * 4); block_words[5] = load32(block + 4 * 5); block_words[6] = load32(block + 4 * 6); block_words[7] = load32(block + 4 * 7); block_words[8] = load32(block + 4 * 8); block_words[9] = load32(block + 4 * 9); block_words[10] = load32(block + 4 * 10); block_words[11] = load32(block + 4 * 11); block_words[12] = load32(block + 4 * 12); block_words[13] = load32(block + 4 * 13); block_words[14] = load32(block + 4 * 14); block_words[15] = load32(block + 4 * 15); state[0] = cv[0]; state[1] = cv[1]; state[2] = cv[2]; state[3] = cv[3]; state[4] = cv[4]; state[5] = cv[5]; state[6] = cv[6]; state[7] = cv[7]; state[8] = IV[0]; state[9] = IV[1]; state[10] = IV[2]; state[11] = IV[3]; state[12] = counter_low(counter); state[13] = counter_high(counter); state[14] = (uint32_t)block_len; state[15] = (uint32_t)flags; round_fn(state, &block_words[0], 0); round_fn(state, &block_words[0], 1); round_fn(state, &block_words[0], 2); round_fn(state, &block_words[0], 3); round_fn(state, &block_words[0], 4); round_fn(state, &block_words[0], 5); round_fn(state, &block_words[0], 6); } void blake3_compress_in_place_portable(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags) { uint32_t state[16]; compress_pre(state, cv, block, block_len, counter, flags); cv[0] = state[0] ^ state[8]; cv[1] = state[1] ^ state[9]; cv[2] = state[2] ^ state[10]; cv[3] = state[3] ^ state[11]; cv[4] = state[4] ^ state[12]; cv[5] = state[5] ^ state[13]; cv[6] = state[6] ^ state[14]; cv[7] = state[7] ^ state[15]; } void blake3_compress_xof_portable(const uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags, uint8_t out[64]) { uint32_t state[16]; compress_pre(state, cv, block, block_len, counter, flags); store32(&out[0 * 4], state[0] ^ state[8]); store32(&out[1 * 4], state[1] ^ state[9]); store32(&out[2 * 4], state[2] ^ state[10]); store32(&out[3 * 4], state[3] ^ state[11]); store32(&out[4 * 4], state[4] ^ state[12]); store32(&out[5 * 4], state[5] ^ state[13]); store32(&out[6 * 4], state[6] ^ state[14]); store32(&out[7 * 4], state[7] ^ state[15]); store32(&out[8 * 4], state[8] ^ cv[0]); store32(&out[9 * 4], state[9] ^ cv[1]); store32(&out[10 * 4], state[10] ^ cv[2]); store32(&out[11 * 4], state[11] ^ cv[3]); store32(&out[12 * 4], state[12] ^ cv[4]); store32(&out[13 * 4], state[13] ^ cv[5]); store32(&out[14 * 4], state[14] ^ cv[6]); store32(&out[15 * 4], state[15] ^ cv[7]); } INLINE void hash_one_portable(const uint8_t *input, size_t blocks, const uint32_t key[8], uint64_t counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { uint32_t cv[8]; memcpy(cv, key, BLAKE3_KEY_LEN); uint8_t block_flags = flags | flags_start; while (blocks > 0) { if (blocks == 1) { block_flags |= flags_end; } blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, block_flags); input = &input[BLAKE3_BLOCK_LEN]; blocks -= 1; block_flags = flags; } store_cv_words(out, cv); } void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, bool increment_counter, uint8_t flags, uint8_t flags_start, uint8_t flags_end, uint8_t *out) { while (num_inputs > 0) { hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, flags_end, out); if (increment_counter) { counter += 1; } inputs += 1; num_inputs -= 1; out = &out[BLAKE3_OUT_LEN]; } } digest/ChangeLog0000644000176200001440000013233114660627452013322 0ustar liggesusers2024-08-19 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.37 2024-08-18 Dirk Eddelbuettel * DESCRIPTION (Authors@R): Added 2024-08-15 Kevin Ushey * src/raes.c: Calloc -> R_Calloc; Free -> R_Free 2024-07-15 Dirk Eddelbuettel * README.md: Switch some URLs from http to https 2024-06-23 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.36 * src/digest.c (is_little_endian): Define alternate helper too * man/digest.Rd: Compare to spookyhash ref only on little endian 2024-06-22 Dirk Eddelbuettel * src/digest.c (is_big_endian): Use endian-ness definition from Rconfig.h and define one-line helper * inst/tinytest/test_digest.R: Skip spookyhash test on big endian 2024-06-21 Sergey Fedorov * src/blake3.c: Upstream patch for big endian systems * src/blake3_impl.h: Idem * src/blake3_portable.c: Idem 2024-06-15 Dirk Eddelbuettel * .github/workflows/ci.yaml (jobs): Update to r-ci-setup action 2024-05-16 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * README.md: Use tinyverse.netlify.app for dependency badge * src/spooky_serialize.cpp: Several small updates to please clang++ * src/SpookyV2.cpp: Idem * .github/workflows/ci.yaml: Show logs in case of failure 2024-03-12 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * src/spooky_serialize.cpp: Use R_NO_REMAP, add three Rf_ prefixes 2024-03-10 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.35 * src/xxhash.h: Update to current release 0.8.2 * src/xxhash.c: Idem * src/digest.c: Support added xxhash algorithms xxh3_64 and xxh3_128 * R/digest.R: Idem * R/vdigest.R: Support xxh3_64 and xxh3_128 in vectorized mode * inst/tinytest/test_digest.R: Add tests for xxh3_64 and xxh3_128 * man/digest.Rd: Documentation * man/vdigest.Rd: Idem * DESCRIPTION (Description): Mention xxh3_64 and xxh3_128 * README.md: Idem * src/digest.c (digest): Consistently print uint64_t via PRIx 2024-01-11 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.34 2024-01-05 Dirk Eddelbuettel * .github/workflows/ci.yaml (jobs): Update to actions/checkout@v4 2024-01-04 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * src/digest.c (open_with_widechar_on_windows): Correct format specification in error() call on Windows 2023-08-23 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * DESCRIPTION: Add Michael contributors * README.md: Idem 2023-08-03 Michael Chirico * tests/tinytest.R: Define `expect_length()` if needed (it is only available from tinytest 1.4.1 released February 2023) * src/digest.c: `#include ` for `uint32_t` 2023-06-28 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.33 * src/crc32c/crc32c_config.h: Minor comment edit 2023-06-27 Sergey Fedorov * src/crc32c/crc32c_config.h: Use endian macros to set endianness 2023-06-27 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * src/crc32c/crc32c_config.h: Undefine HAVE_BUILTIN_PREFETCH and HAVE_MM_PREFETCH for maximum portability and builds on M1/M2/Arm64 2023-06-26 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.32 * README.md: Add r-universe badge * README.md: Add crc32c, and link to docs site, to Overview 2023-06-25 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * DESCRIPTION (Author): Add Dean Attali 2023-06-25 Dean Attali * R/AES.R: Add `padding` parameter to `AES()` to allow CBC mode to use PKCS#7 padding * inst/tinytest/test_aes.R: Add tests for new `padding` parameter * man/AES.Rd: Document the `raw` argument of `AES()$decrypt()` and the new `padding` parameter 2023-04-30 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll micro version and date * R/vdigest.R: Enable vectorised operation for blake3 and crc32v * man/vdigest.Rd: Update documentation 2023-04-29 Dirk Eddelbuettel * src/digest.c (digest): Support crc33c * man/digest.Rd: Document crc32c option, add two examples * src/crc32c.h: Adding new CRC32C implementation * src/crc32c.cpp: Idem * src/crc32c_portable.cpp: Idem; also added codecov nocov * src/crc32c/*: Idem * src/Makevars: Added for PKG_CPPFLAGS * src/Makevars.win: Idem * R/init.R: Added some codecov nocov * .codecov.yml (coverage): Added coverage diff settings 2023-02-05 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * src/Makevars: No longer require CXX_STD 2022-12-10 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.31 * src/digest.c: Replace sprintf with snprintf * .editorconfig: Added * .Rbuildignore: Updated 2022-11-05 Dirk Eddelbuettel * .github/workflows/ci.yaml (jobs): Update to actions/checkout@v3 2022-10-17 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.30 * src/crc32.c: Modernize three function signatures to current C standards to appease clang-15 now used by CRAN * src/sha2.c (SHA512_End): Align two function signatures to their prototypes to appease gcc-12 2022-10-02 Dirk Eddelbuettel * docs/mkdmt-src/src/index.md: Several small updates with upgrade to Material for MkDocs 8.5.5 2021-11-30 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.29 * src/digest.c (digest): Accomodate Windows UCRT build * man/AES.Rd: Remove one URL that upsets the URL checker * README.md: Update one URL that upsets the URL checker 2021-11-20 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * vignettes/sha1.md: Renamed from .Rmd, corrected code block syntax 2021-11-19 Dirk Eddelbuettel * DESCRIPTION (VignetteBuilder): Converted to simplermarkdown engine * vignettes/sha1.Rmd: Idem * vignettes/water.css: Added 2021-11-05 Dirk Eddelbuettel * README.md: Remove Travis badge * .travis.yml: Remove Travis YAML config 2021-09-23 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.28 2021-09-22 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * vignettes/sha1.Rmd (date): Remove knitr date calculation in YAML header which GitHub does not like and mistakes for Jekyll code * docs/mkdmt-src/src/vignette.md (date): Idem * README.md: Add total download badge, label other as monthly 2021-09-22 András Svraka * R/vdigest.R (non_streaming_digest): Ensure UTF-8 encoded file paths on Windows * inst/tinytest/test_encoding.R: Expand test coverage for path name encodings on Windows 2021-03-28 Dirk Eddelbuettel * docs/mkdmt-src/: Moved mkdocs-material input 2021-03-26 Dirk Eddelbuettel * README.md: Add 'last commit' badge 2021-03-23 Dirk Eddelbuettel * DESCRIPTION (URL): Add link to repo 2021-03-22 Dirk Eddelbuettel * docs-src/mkdocs.yml (theme): Add (draft, incomplete) vignette * docs-src/src/vignette.md (title): Idem 2021-03-22 Floris Vanderhaeghe * man/digest.Rd: Reworded to also highlight file mode 2021-01-16 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * src/SpookyV2.cpp: Disallow unaligned reads which have been seen to trigger SAN issues as reported by Winston in #167 2020-12-23 Dirk Eddelbuettel * .github/workflows/ci.yaml: Add CI runner using r-ci * README.md: Add new CI badge 2020-10-20 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.27 2020-10-19 Dirk Eddelbuettel * src/blake3_dispatch.c: Comment-out include of intrinsics to also comment-out erroring on insufficient architectures such as Solaris 2020-10-17 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.26 * docs/: Added package website * docs-src/: Added package website inputs * .travis.yml (dist): Change to focal 2020-10-15 Dirk Eddelbuettel * README.md: Update URLs * man/digest.Rd: Ditto * man/hmac.Rd: Ditto 2020-10-14 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * inst/tinytest/test_blake3.R: Skip file-based blake3 tests on Windows * vignettes/sha1.Rmd: Switch to minidown and 'framework: water' * DESCRIPTION (Suggests): Add minidown 2020-10-07 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version 2020-10-06 Winston Chang * R/digest.R: Speedup via explicit alternative values for match.arg() and a direct call to serialize() * R/vdigest.R: Idem * R/init.R: Idem (for serialize()) * inst/tinytest/test_digest.R: New test ensuring all variants hit 2020-09-21 Dirk Eddelbuettel * .travis.yml: Updated to 'bionic', R 4.0, and BSPM 2020-08-03 Dirk Eddelbuettel * DESCRIPTION (Depends): Roll Depends: on to R (>= 3.3.0) due to use of startsWith(), with thanks to Florian Pein 2020-05-20 Dirk Eddelbuettel * .travis.yml: Switch to bionic and R 4.0.0 2020-05-19 Dirk Schumacher * src/digest.c: Support file operation * inst/tinytest/test_blake3.R: Update test 2020-05-19 Dirk Eddelbuettel * DESCRIPTION (Version): Roll minor version and date 2020-05-19 Dirk Schumacher * src/blake3.c: Add blake3 implementation * src/blake3.h: Idem * src/blake3_dispatch.c: Idem * src/blake3_impl.h: Idem * src/blake3_portable.c: Idem * src/digest.c: Support blake3 call * R/digest.R: Idem * inst/tinytest/test_blake3.R: Add tests * man/digest.Rd: Add documentation 2020-03-05 Harris McGehee * man/sha1.Rd: Correct typo 2020-02-26 Dirk Eddelbuettel * README.md: Add Debian badge 2020-02-22 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.25 2020-02-15 Kendon Bell * R/digest.R: Complete call to spookyhash 2020-02-12 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.24 2019-12-12 Thierry Onkelinx * DESCRIPTION (Version, Date): Roll minor version * R/sha1.R: ignore the "srcref" attribute if set * inst/tinytest/test_new_matrix_behaviour.R: update unit test * inst/tinytest/test_sha1.R: update unit test * man/sha1.Rd: update documentation 2019-12-09 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * src/init.c: Do not register a .Call method for PMurHash 2019-12-05 Will Landau * R/init.R (.onLoad): Cache nosharing state in base::serialize (.hasNoSharing): Return cached value * R/digest.R (digest): Use cached value accessor * R/vdigest.R (non_streaming_digest): Use cached value accessor 2019-12-04 Dirk Eddelbuettel * R/init.R (.onLoad): Cache isWindows state (.isWindows): Return cached value * R/digest.R (digest): Use cached value accessor 2019-11-23 Dirk Eddelbuettel * inst/tinytest/test_new_matrix_behaviour.R: Split off tests affected by new matrix/array behavior in R-devel (i.e. future R 4.0.0) 2019-11-22 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.23 * inst/tinytest/test_sha1.R: Uncomment several tests which failed under r-devel on Linux at CRAN 2019-11-13 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version 2019-11-11 Bill Denney * R/sha1.R: Increase efficiency of num2hex() 2019-11-07 Bill Denney * NAMESPACE: add sha1_digest() and sha1_attr_digest() functions * R/sha1.R: Idem * man/sha1.Rd: Idem * Also, renamed sha1.rd to sha1.Rd 2019-11-03 Bill Denney * R/sha1.R: Add support for the "(" class used in some formulae * NAMESPACE: Idem * man/sha1.Rd: Idem; also corrected typo for old version behavior * inst.tinytest/test_sha1.R: Add tests for the "(" class 2019-10-27 Thierry Onkelinx * DESCRIPTION (Version, Date): Roll minor version * R/sha1.R: consider object attributes when calculating the hash * man/sha1.Rd: update documentation * R/init.R: add "sha1PackageVersion" option * inst.tinytest/test_sha1.R: add units tests for new functionality * NAMESPACE: import utils::packageVersion 2019-10-26 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * R/digest.R (digest): Call enc2utf8() only on Windows; add internal one-liner function to test for being on Windows 2019-10-23 Dirk Eddelbuettel * inst/tinytest/test_encoding.R: Skip test unless on Windows 2019-10-21 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.22 2019-10-20 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * inst/tinytest/test_misc.R: File split off test_digest.R 2019-10-19 Dirk Eddelbuettel * DESCRIPTION: Add Ion and Bill to Authors * README.md: Idem 2019-10-13 Jim Hester * src/digest.c: Add support for UTF-8 file paths on Windows * R/digest.R: Idem * inst/tinytest/test_encoding.R: Add tests for UTF-8 file paths. 2019-10-13 Bill Denney * R/sha1.R: Add sha1.formula() * NAMESPACE: Idem * man/sha1.rd: Idem * inst/tinytest/test_sha1.R: Add tests for sha1.formula() 2019-10-12 Bill Denney * R/sha1.R: Fix issue with num2hex() and input of Inf * inst/tinytest/test_sha1.R: Add Inf to the tests 2019-10-05 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * man/digest.Rd: Change three URL references to https * man/hmac.Rd: Ditto * R/utils.R: Move back functions used by digest() + makeVDigest() * R/AES.R: Added copyright header * R/digest.R: Idem * R/hmac.R: Idem * R/sha1.R: Idem * R/vdigest.R: Idem 2019-10-02 Ion Suruceanu * R/AES.R: Add support for CFB cipher mode * man/AES.Rd: Add documentation * inst/tinytest/test_aes.R: Add tests 2019-09-20 Matthew de Queljoe * R/digest.R: refactor digest function * R/vdigest.R: remove helper functions from file * R/utils.R: new file to hold helper functions 2019-09-19 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.21 2019-09-18 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * DESCRIPTION (Author): Add Matthew * README.md (Author): Ditto * demo/vectorised.R: Add demo (based on Matthew's PR) * R/vdigest.R: Added a handful of #nocov tags * man/vdigest.Rd: Break up two long lines 2019-09-17 Matthew de Queljoe * src/digest.c: Add vectorised digest function * R/vdigest.R: Add vectorised digest function factory * man/vdigest.Rd: Add documentation * NAMESPACE: Export new function getVDigest * inst/tinytest/test_digest.R: Add tests 2019-07-04 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.20 2019-06-29 Dirk Eddelbuettel * README.md: Add installation and continues testing sections * .travis.yml (install): Use r-cran-tinytest from PPA 2019-06-23 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * inst/tinytest/test_num2hex.R: Quieter with sapply * test/tinytest/test_aes.R: renamed from testAES.R * test/tinytest/test_crc32.R: renamed from testCRC32.R * test/tinytest/test_digest.R: renamed from testDigest.R * test/tinytest/test_digest2int.R: from testDigest2Int.R * test/tinytest/test_hmac.R: renamed from testHMAC.R * test/tinytest/test_num2hex.R: renamed from testNum2Hex.R * test/tinytest/test_raw.R: renamed from testRaw.R * test/tinytest/test_sha1.R: renamed from testSHA1.R 2019-06-10 Dirk Eddelbuettel * inst/tinytest/testCRC32.R: Use expcect_* functions * inst/tinytest/testDigest2Int.R: Idem * inst/tinytest/testNum2Hex.R: Idem * inst/tinytest/testRaw.R: Idem 2019-05-26 Dirk Eddelbuettel * tests/tinytest.R: New test driver using tinytest * DESCRIPTION (Suggests): Added tinytest * .travis.yml (install): Install tinytest * inst/tinytest/testRaw.R: Converted from tests/raw.R * inst/tinytest/testCRC32.R: Converted from tests/crc32.R * inst/tinytest/testDigest2Int.R: From tests/digest2int.R * inst/tinytest/testNum2Hex.R: From tests/num2hexTest.R * inst/tinytest/testSHA1.R: Converted from tests/sha1Test.R * inst/tinytest/testHMAC.R: Converted from tests/hmacTest.R * inst/tinytest/testAES.R: Converted from tests/aesTest.R * inst/tinytest/testDigest.R: From tests/digestTest.R * R/AES.R: Additional #nocov tags * R/digest.R: Idem * R/hmac.R: Idem * R/sha1.R: Idem 2019-05-19 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.19 * src/SpookyV2.cpp: Add some #nocov tags 2019-05-18 Dirk Eddelbuettel * DESCRIPTION (Version, Date): New minor version * tests/digestTest.Rout.save: Updated reference output 2019-05-12 Kendon Bell * R/digest.R: Account for 'skip' bytes for streaming algos * tests/digestTest.R: Adjust test for streaming_algos 2019-05-09 Dirk Eddelbuettel * src/Makevars: Added to ensure C++11 compilation standard * cleanup: Do not delete src/Makevars now that we have one * tests/digestTest.Rout.save: Updated reference output 2019-05-06 Dirk Eddelbuettel * README.md: Mention spookyhash and Kendon * DESCRIPTION (Description): Mention spookyhash 2019-05-04 Kendon Bell * tests/digestTest.R: Added more tests * src/spooky_serialize.cpp: Added copyright header 2019-05-04 Dirk Eddelbuettel * R/digest.R: Add some #nocov tags * src/SpookyV2.h: Idem * src/SpookyV2.cpp: Idem * src/spooky_serialize.cpp: Idem 2019-05-03 Dirk Eddelbuettel * tests/digestTest.Rout.save: Updated reference output 2019-04-30 Kendon Bell * digest.R: Support spookyhash * src/SpookyV2.h: Idem * src/SpookyV2.cpp: Idem * src/spooky_serialize.cpp: Idem * man/digest.Rd: Document spookyhash 2019-04-23 Kendon Bell * src/digest.c: Switch length counter to R_xlen_t 2018-12-23 Dirk Eddelbuettel * NAMESPACE: Add .registration=TRUE to useDynLib() 2018-11-10 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version 2018-11-08 Dmitriy Selivanov * src/digest2int.c: Removed (bad) redefinition of uint32_t * tests/digest2intTest.R: Additional test 2018-10-29 Dmitriy Selivanov * DESCRIPTION (Version, Date): minor version * digest2int.R (digest2int): added digest2int * src/digest2int.c (digest2int): added Bob Jenkins `one_at_a_time` function for hashing arbitrary character vectors to integer vectors 2018-10-18 Dirk Eddelbuettel * README.md: Added dependencies badge 2018-10-10 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Release 0.6.18 2018-09-16 Dirk Eddelbuettel * src/digest.c (digest): Six more #nocov tags * README.md: Small edits 2018-09-14 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * src/sha2.c: Explicitly #undef SHA256_ and SHA384_ functions not used from this file 2018-09-13 Radford Neal * src/pmurhash.c (DOBYTES): Explicit cast to uint32_t avoids UBSAN 2018-09-13 Jim Hester * src/xxhash.c: Updated to use xxHash v0.6.5 * src/xxhash.h: Ditto * src/digest.c: Ditto 2018-09-11 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Release 0.6.17 2018-09-02 Dirk Eddelbuettel * .travis.yml: Simplified as covr can be installed from c2d4u 2018-08-30 Radford Neal * src/sha2.c: Memory alignment changes motivated by 32bit sparc * src/sha2.h: Ditto 2018-08-21 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Release 0.6.16 2018-08-20 Dirk Eddelbuettel * tests/digestTest.Rout: Skip one test which creates different results across versions and operating systems * tests/digestTest.Rout.save: Ditto 2018-08-19 Dirk Eddelbuettel * tests/hmacTest.R: Added raw test * tests/hmacTest.Rout.save: Updated reference output accordingly * tests/raw.R: Added test for 'raw' vector input to sha1() * R/sha1.R: Removed no longer needed #nocov tags 2018-07-21 Henrik Bengtsson * DESCRIPTION: digest (>= 0.6.14) requires R (>= 3.0.3) 2018-06-30 Dirk Eddelbuettel * tests/crc32.R: Added test for crc32 and old versus new formats * tests/raw.R: Added small test file for raw tests 2018-06-29 Dirk Eddelbuettel * R/init.R: Provide option 'digestOldCRC32Format' which if TRUE returns without zero padding which can be shorter than eight bytes * R/digest.R: If crc32 selected and option set, return in old format * man/digest.Rd: Document new option 2018-06-28 Dirk Eddelbuettel * src/digest.c (digest): Change print format for crc32 to always return eight characters and no longer drop leading zeros (thanks to Henrik Bengtsson for the heads-up) 2018-06-21 Dirk Eddelbuettel * tests/sha1Test.R: Do not run test using serialize() as its output always reflects the R version used (thanks, Radford Neal) 2018-01-29 Dirk Eddelbuettel * src/sha2.c: Comment-out three unused 'const static' variables 2018-01-28 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Release 0.6.15 * R/sha1.R (sha1.POSIXlt): Unclass POSIXlt objects as suggested by Kurt Hornik to accomodate R-devel changes * tests/sha1Test.R: Condition one sha1 test seemingly affected by R-devel serialization changes to run only with R < 3.5.0 2018-01-21 Thierry Onkelinx * sha1() gains an `algo` argument * sha1() handles raw class 2018-01-14 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Release 0.6.14 * man/AES.Rd: Use https for nist.gov reference * man/digest.Rd: Idem * man/hmac.Rd: Idem * man/digest.Rd: Use Wikiepedia page as reference for SHA1 as the reference page at NIST has (long) vanished. * man/hmac.Rd: Idem 2018-01-12 Thierry Onkelinx * sha1() handles empty matrices 2017-12-20 Dirk Eddelbuettel * src/raes.c (AESencryptECB,AESdecryptECD): Replace two uses of NAMED with MAYBE_REFERENCED 2017-12-13 Dirk Eddelbuettel * DESCRIPTION (Date, Version): Release 0.6.13 2017-12-12 Dirk Eddelbuettel * DESCRIPTION (Version, Date): New minor version * R/digest.R: Support serializeVersion format * man/digest.Rd: Document new option * R/init.R: Set a default version, and internal getter function 2017-10-12 Chris Muir * vignette/sha1.Rmd: Correct simple typo 2017-11-16 Moritz Beller * man/digest.Rd: Replace old Google Code URLs with GitHub ones 2017-02-20 Dirk Eddelbuettel * man/digest.Rd: Expand example section with a Vectorize() use 2017-01-29 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Roll minor version * man/digest.Rd: Note that support for 'raw' is not available for all hashing algorithms * .travis.yml (before_install): Use https for curl fetch 2017-01-26 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.12 2017-01-23 Thierry Onkelinx * NAMESPACE: export sha1.function() and sha1.call() * R/sha1.R: - sha1() gains methods for the class "function" and "call" - sha1() gains a ... argument, currently only relevant for "function" - sha1() takes arguments into account for hash for complex, Date and array. Note that this will lead to different hasheS for these classes and for objects containing these classes * man/sha1.rd: update helppage for sha1() * tests/sha1Test.R: update unit tests for sha1() 2017-01-01 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Release 0.6.11 * R/sha1.R (sha1.anova): Added more #nocov marks * src/sha2.c (SHA256_Transform): Idem * tests/AESTest.R (hextextToRaw): Print AES object * tests/AESTest.Rout.save: Updated 2016-12-08 Dirk Eddelbuettel * NAMESPACE: Register (and exported) makeRaw S3 methods * man/makeRaw.Rd: New manual page * tests/hmacTest.R: Direct call to makeRaw() * tests/hmacTest.Rout.save: Ditto * src/digest.c: Additional #nocov tags * src/xxhash.c: Ditto 2016-12-07 Dirk Eddelbuettel * DESCRIPTION (Version, Date): Rolled minor version * README.md: Use shields.io badge for codecov * R/digest.R: Additional #nocov tags * src/sha2.c: Ditto * src/raes.c: Ditto * tests/hmacTest.R: Additional tests * tests/hmacTest.Rout.save: Ditto 2016-11-30 Dirk Eddelbuettel * .travis.yml (before_install): Activate PPA as we (currently) need an updated version of (r-cran)-covr to run coverage * tests/load-unload.R: Comment-out for now as it upsets coverage * tests/digestTest.R: Test two more algorithms * tests/digestTest.Rout.save: Updated reference output * R/digest.R: Added #nocov tags * R/zzz.R (.onUnload): Ditto * src/crc32.c: Ditto * src/pmurhash.c: Ditto * src/raes.c: Ditto * src/sha2.c: Ditto * src/xxhash.c: Ditto 2016-11-26 Dirk Eddelbuettel * .travis.yml (after_success): Integrated Jim Hester's suggestion of activating code coverage sent many moons ago (in PR #12) * .codecov.yml (comment): Added * .Rbuildignore: Exclude .codecov.yml * README.md: Added code coverage badge 2016-10-16 Dirk Eddelbuettel * R/digest.R (digest): Support 'nosharing' option of base::serialize as suggested by Radford Neal whose pqR uses this 2016-08-02 Dirk Eddelbuettel * DESCRIPTION (License): Now GPL (>= 2), cf issue 36 on GH * README.md: Updated badge accordingly 2016-08-02 Dirk Eddelbuettel * DESCRIPTION (Version): Release 0.6.10 * DESCRIPTION (Description): Shortened to one paragraph * DESCRIPTION (BugReports): URL to GH issues added * .travis.yml: Rewritten for run.sh from forked r-travis 2016-07-12 Henrik Bengtsson * src/digest.c: Correct bug with skip and file parameter interaction * tests/digestTest.R: Test code * tests/digestTest.Rout.save: Test reference output * R/zzz.R: Allow for unloading of shared library * tests/load-unload.R: Test code * DESCRIPTION: Rolled minor Version and Date 2016-05-25 Thierry Onkelinx * R/sha1.R: Support for pairlist and name * tests/sha1Test.R: Support for pairlist and name * man/sha1.Rd: Support for pairlist, name, complex, array and Date * NAMESPACE: Support for pairlist, name and array * DESCRIPTION: bump version number and date 2016-05-01 Viliam Simko * R/sha1.R: Support for complex, Date and array * tests/sha1Test.R: Ditto * NAMESPACE: Ditto 2016-04-27 Dirk Eddelbuettel * DESCRIPTION (Author): Add Qiang Kou to Authors * README.md: Ditto 2016-01-25 Dirk Eddelbuettel * src/digest.c (digest): Use XLENGTH if R >= 3.0.0 (issue #29) 2016-01-11 Thierry Onkelinx * R/sha1.R: handle empty list and empty dataframe (#issue 27); take the object class, digits and zapsmall into account (#PR 28) * vignettes/sha1.Rmd: Small edits to reflect changes is sha1() 2016-01-09 Michel Lang * R/sha1.R: Add a length check to sha1(), use vapply() 2016-01-07 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.9 * DESCRIPTION (Date): Bumped Date: to current date 2016-01-06 Dirk Eddelbuettel * vignettes/sha1.Rmd: Small edits 2016-01-06 Thierry Onkelinx * R/sha1.R: Add functions to calculate stable SHA1 with floating points * man/sha1.Rd: Add helpfile for sha1() * tests/num2hexTest.R: unit tests for num2hex() (non exported function) * tests/sha1Test.R: unit tests for sha1() * NAMESPACE: Export sha1 and its methods * DESCRIPTION: Add Thierry Onkelinx as contributor, bump Version and Date * README.md: Add Thierry Onkelinx as contributor * vignette/sha1.Rmd: Added * .travis.yml: Added 'sudo: required' per recent Travis changes 2015-10-14 Dirk Eddelbuettel * man/digest.Rd: Remove references to inaccessible web pages * man/hmac.Rd: Ditto 2015-10-13 Dirk Eddelbuettel * src/digest.c: Use uint32_t instead of int for nchar 2015-10-12 Qiang Kou * src/digest.c: Use XLENGTH instead of LENGTH (PR #17, issue #16) 2015-08-06 Dirk Eddelbuettel * DESCRIPTION (Title): Updated now stressing 'compact' over 'crypto' 2014-12-30 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.8 * DESCRIPTION (Date): Bumped Date: to current date 2014-12-29 Dirk Eddelbuettel * inst/include/pmurhashAPI.h: Added HOWTO comment to top of file 2014-12-26 Dirk Eddelbuettel * src/pmurhash.c: Protect against _BIG_ENDIAN defined but empty * inst/include/pmurhash.h: Consistent four space indentation 2014-12-25 Dirk Eddelbuettel * DESCRIPTION: Bump Date: and Version: * src/init.c: Minor edit and removal of unused headers 2014-12-25 Wush Wu * inst/include/pmurhash.h: Export function * src/init.c: Register function for use by other packages 2014-12-20 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.7 * DESCRIPTION (Date): Bumped Date: to current date 2014-12-19 Dirk Eddelbuettel * cleanup: Also remove src/symbols.rds * src/sha2.c: Apply (slightly edited) patch from https://www.nlnetlabs.nl/bugs-script/attachment.cgi?id=220&action=diff to overcome the strict-aliasing warning * src/digest.c: Use inttypes.h macro PRIx64 only on Windows 2014-12-16 Dirk Eddelbuettel * src/xxhash.c: Remove two semicolons to make gcc -pedantic happy * tests/digestTest.Rout.save: Updated reflecting murmurHash test * src/pmurhash.c: Renamed from PMurHash.c for naming consistency * src/pmurhash.h: Renamed from PMurHash.h for naming consistency 2014-12-16 Jim Hester * src/digest.c: murmurHash implementation * tests/digestTest.R: murmurHash implementation * R/digest.R: murmurHash implementation * src/PMurHash.c: murmurHash implementation * src/PMurHash.h: murmurHash implementation 2014-12-10 Dirk Eddelbuettel * src/xxhash.c: Applied pull request #6 by Jim Hester with updated upstream code and already corrected UBSAN issue identified by CRAN 2014-12-09 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.6 * DESCRIPTION (Date): Bumped Date: to current date * src/digest.c: Applied pull request #5 by Jim Hester providing portable integer printing inttypes.h header 2014-12-08 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.5 * DESCRIPTION (Date): Bumped Date: to current date * NAMESPACE: Expanded useDynLib() declaring C level symbols, in particular using digest_impl to for the C-level digest * R/AES.R: Use R symbols from NAMESPACE declaration in .Call() * R/digest.R: Use R symbol digest_impl to load C level digest 2014-12-07 Dirk Eddelbuettel * DESCRIPTION: Edited Title and Description * R/digest.R: Added GPL copyright header, reindented to four spaces * src/digest.c: Reindented to four spaces * R/AES.R: Reindented to four spaces * R/hmac.R: Reindented to four spaces 2014-12-06 Dirk Eddelbuettel * src/digest.c: Updated GPL copyright header * src/xxhash.c: Removed two spurious ';' * man/digest.Rd: Document 'seed' argument in \usage * tests/digest.Rout.save: Updated for expanded tests * DESCRIPTION: Add Jim Hester to list of Authors 2014-12-05 Dirk Eddelbuettel * R/digest.R: Applied pull request #3 by Jim Hester with support for xxHash (https://code.google.com/p/xxhash/) * src/digest.c: Ditto * src/xxhash.c: xxHash implementation supplied as part of #3 * src/xxhash.h: xxHash implementation supplied as part of #3 * R/digest.R: Applied pull request #4 by Jim Hester with expanded support for xxHash providing xxhash32 and xxhash64 * src/digest.c: Ditto * man/digest.Rd: Added documentation for xxHash, corrected typos * src/digest.R: New support for a seed parameter used by xxHash * tests/digestTest.R: Added tests for xxHash 2014-08-15 Dirk Eddelbuettel * R/hmac.R: Applied (slightly edited) patch for crc32 computation of hmac kindly supplied by Suchen Jin 2013-12-02 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.4 * src/sha2.h (BYTE_ORDER): Define BYTE_ORDER unless already defined, rely on Rconfig.h which itself goes back to an R compile-time test 2013-02-16 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.3 2013-02-06 Hannes Mühleisen * R/hmac.R: Fixed hmac for sha512 hashes * tests/hmacTest.R: Added test cases for hmac with sha512 * tests/hmacTest.Rout.save: Updated accordingly 2013-01-25 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.2 * man/AES.Rd: Switch from paste0() to paste() to permit use on R-oldrelease as per email by Uwe Ligges * tests/AESTest.R: Idem * tests/AESTest.Rout.save: Updated accordingly 2013-01-21 Dirk Eddelbuettel * DESCRIPTION (Version): Version 0.6.1 * inst/GPL-2: Added as a fixed reference input for regression tests via the scripts in tests/ as the versions installed by R differ across OS and installations * tests/digestTest.R: No longer rely on file.path(R.home(),"COPYING") but rather use our own copy of GPL-2; ensure final test does not print * tests/digestTest.Rout.save: Updated accordingly 2013-01-19 Dirk Eddelbuettel * DESCRIPTION (Authors): Added Duncan Murdoch 2013-01-19 Duncan Murdoch * src/aes.c: Devine's AES implementation added * src/aes.h: header for AES implementation * src/raes.c: interface to it * R/AES.R: Add AES object to do AES encryption * tests/AESTest.R: tests from the standards documents * man/AES.Rd: document AES object * NAMESPACE: export AES constructor and print method 2012-11-25 Dirk Eddelbuettel * Release 0.6.0 * src/sha2.h: In order to build on Windows: - Include the newer header file stdint.h - Enforce standard integer types by defining SHA2_USE_INTTYPES_H - Define BYTE_ORDER and default to LITTLE_ENDIAN * src/digest.c: Use uint8_t from stdint.h for sha256 2012-11-24 Dirk Eddelbuettel * src/digest.c: Allow for extra null character in output[] vector * tests/digestTest.Rout.save: Update/Revert a change by Hannes; Naturally we do not get a single set that works for R-release and R-devel. Sigh. 2012-11-24 Hannes Mühleisen * src/digest.c: sha-512 integration * src/sha2.h: Header file for Aaron Gifford's SHA2 implementation * src/sha2.c: Aaron Gifford's sha2 implementation * R/digest.R: enabled new sha-512 algorithm parameter * R/hmac.R: enabled new sha-512 algorithm parameter * man/digest.Rd: documented new sha-512 algorithm parameter * man/hmac.Rd: documented new sha-512 algorithm parameter * tests/digestTest.R: Added sha-512 test cases 2012-03-14 Dirk Eddelbuettel * Release 0.5.2 * Applied patch by Murray Stokely: - R/digest.R: Additional test for file accessibility; segfault can happen if inaccesible file passed down - tests/digestTest.R: New test which leads to segfault in unpatched package, and passes with these changes - src/digest.c: Simpler use of error() via format string * DESCRIPTION: Fixed one typo in extended description * tests/digest.Rout.save: Updated to current output; version 0.5.1 created the same difference so it is presumably once again something that changed in R's serialization. Oh, and R-devel changes it again. 2011-09-20 Dirk Eddelbuettel * Release 0.5.1 * tests/digestTest.Rout.save: Updated reference output too 2011-09-18 Bryan Lewis * tests/digestTest.R: Added basic raw output md5 and sha1 tests 2011-09-14 Dirk Eddelbuettel * src/digest.c: Applied patch contributed by Bryan Lewis which supports output of unmodified raw binary output if a new function parameter 'raw' (which defaults to FALSE) is set to TRUE * R/digest.R: Support new parameter 'raw' * man/digest.Rd: Document new parameter 'raw' 2011-05-29 Dirk Eddelbuettel * Release 0.5.0 2011-05-26 Dirk Eddelbuettel * R/hmac.R: Switched to camelCase identifiers after discussion with Henrik and Mario 2011-05-25 Dirk Eddelbuettel * R/hmac.R: New hmac() function contributed by Mario Frasca * man/hmac.Rd: Documentation contributed by Mario Frasca * tests/hmacTest.R: Added a new regression test file for hmac() * tests/hmacTest.Rout.save: Added new reference output * tests/digestTest.Rout.save: Updated reference digest output for simple R structure to match what R 2.13.0 yields * inst/ChangeLog: moved to ChangeLog (in top-level directory) * INDEX: removed, as no longer needed 2009-12-03 Henrik Bengtsson * Release 0.4.2 * R/digest.R: Bug fix - digest(object, file=TRUE) where object is a directory would cause R to crash. Now it gives an error. 2009-10-06 Dirk Eddelbuettel * DESCRIPTION: set SVN properties Date and Id, fix indentation 2009-09-24 Dirk Eddelbuettel * Release 0.4.1 * src/Makefile.win: Removed as package builds on Windows without it but not with it being present * src/digest.c: Updated Copyright years, set SVN properties Date and Id * R/digest.R: Idem * man/digest.Rd: Idem 2009-09-23 Dirk Eddelbuettel * Release 0.4.0 * src/sha256.c: Added sha-256 implementation by Christophe Devine as found (via Google Code Search) in a number of Open Source projects such as mushclient, aescrypt2, scrypt, and ipanon * src/sha256.h: Idem * src/digest.c: Modified to support SHA-256 * R/digest.R: Idem * man/digest.Rd: Idem, also added more references * src/Makefile.win: Updated for SHA-256, and generally spruced up 2007-09-28 Dirk Eddelbuettel * Release 0.3.1 * DESCRIPTION: Switched to standardised form 'GPL-2' for License: * src/digest.c: Added one explicit (char *) cast 2007-04-27 Dirk Eddelbuettel * Release 0.3.0 * R/digest.R: Adopted two more changes suggested by Henrik 2007-03-12 Dirk Eddelbuettel * R/digest.R: Adopted a few changes suggested by Henrik 2007-03-09 Dirk Eddelbuettel - R/digest.R, man/digest.Rd, Applied two more patches by Simon Urbanek that clean object mangling (for better comparison across R versions, adds an ascii flag, adds skip="auto" support to by pass the version header info, and clean the file option interface. This effectively replaces Henrik's patch relative to the 0.2.3 release. Thanks for the patches, Simon! - tests/digestTest.Rout.save: New reference output; one line changed 2007-03-08 Dirk Eddelbuettel - R/digest.R, man/digest.Rd, src/digest.C: Applied two patches by Simon Urbanek to help improve consistence of digest output across different R versions by allowing digest to 'skip' a certain number of bytes; and by adding support for 'raw' serialization input 2007-01-08 Dirk Eddelbuettel - R/digest.R: Added improvement suggested by Henrik 2006-12-30 Dirk Eddelbuettel * Release 0.2.3 - R/digest.R: Added file.expand() around filename 2006-07-28 Dirk Eddelbuettel * Release 0.2.2 - R/digest.R: Added patch for R 2.4.0 by Henrik Bengtsson - tests/: Added simple unit tests 2005-11-02 Dirk Eddelbuettel * Release 0.2.1 - R/digest.R, src/digest.c, man/digest.Rd: add support for file mode based on a complete set of patches by Jarek Tuszynski 2005-04-06 Dirk Eddelbuettel * Release 0.2.0 - R/digest.R, src/digest.c, man/digest.Rd: add support for crc32 digests based on a complete set of patches by Antoine Lucas - src/{crc32.c,crc32.h,zlib.h,zutil.h,zconf.h}: From zlib 2004-05-26 Dirk Eddelbuettel * Release 0.1.2 - R/zzz.R: remove test for R < 1.8.0 and load of serialize package (as serialize has been removed from CRAN with serialize() in R) 2003-11-29 Dirk Eddelbuettel * Release 0.1.1 - DESCRIPTION: added to RCS, $Date$ is now filled - DESCRIPTION: small rewording in Description field - corrected minor packaging error by removing spurious tarball 2003-11-28 Dirk Eddelbuettel * Release 0.1.0 - DESCRIPTION: added extended Description - inst/ChangeLog: added - man/digest.Rd: added complete test vectors for md5 and sha-1 in example code, values taken from the examples of the C code 2003-10-23 Dirk Eddelbuettel * Release 0.0.1 - initial version digest/NAMESPACE0000644000176200001440000000212314636020207012746 0ustar liggesusers## package has a dynamic library useDynLib(digest, digest_impl=digest, vdigest_impl=vdigest, digest2int_impl=digest2int, AESinit, AESencryptECB, AESdecryptECB, spookydigest_impl, is_little_endian, is_big_endian, .registration=TRUE) importFrom(utils, packageVersion) ## and exported functions export(AES, digest, digest2int, getVDigest, sha1, sha1_attr_digest, sha1_digest, hmac, makeRaw) S3method(print, AES) S3method(sha1, anova) S3method(sha1, array) S3method(sha1, call) S3method(sha1, character) S3method(sha1, complex) S3method(sha1, data.frame) S3method(sha1, Date) S3method(sha1, default) S3method(sha1, integer) S3method(sha1, factor) S3method(sha1, formula) S3method(sha1, "function") S3method(sha1, logical) S3method(sha1, list) S3method(sha1, matrix) S3method(sha1, name) S3method(sha1, NULL) S3method(sha1, numeric) S3method(sha1, pairlist) S3method(sha1, POSIXct) S3method(sha1, POSIXlt) S3method(sha1, raw) S3method(sha1, "(") S3method(makeRaw, default) S3method(makeRaw, digest) S3method(makeRaw, character) S3method(makeRaw, raw) digest/inst/0000755000176200001440000000000014660633605012517 5ustar liggesusersdigest/inst/include/0000755000176200001440000000000014423316130014126 5ustar liggesusersdigest/inst/include/pmurhashAPI.h0000644000176200001440000000613512450374665016504 0ustar liggesusers/* -*- mode: c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- pmurhashAPI.h -- interface file for external clients Copyright (C) 2014 Wush Wu and Dirk Eddelbuettel This file is part of digest. digest is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. digest is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with digest. If not, see . */ /* Purpose: Provide MurmurHash3A for use by C / C++ code of other R packages Usage: 1) In your package, add 'LinkingTo: digest' to the DESCRIPTION file. This ensure that R will know the path to this file. 2) In your source code, add '#include ' This ensures the compiler knows the declaration of the function interface. 3) In your package's source code, call 'res = PMurHash32(seed, key, len);' where res and seed are of type MH_UINT32 (which is defined appropriately below). 4) The local function here sets a static pointer to the actual MurmurHash32 implementation in this package. R takes care of the function registration, export and import --- meaning that no build-time linking of your package is needed resulting in simpler and more robust build steps. */ #ifndef __PMURHASH_H__ #define __PMURHASH_H__ #include #include #ifdef HAVE_VISIBILITY_ATTRIBUTE # define attribute_hidden __attribute__ ((visibility ("hidden"))) #else # define attribute_hidden #endif #ifdef __cplusplus extern "C" { #endif /* First look for special cases */ #if defined(_MSC_VER) #define MH_UINT32 unsigned long #endif /* If the compiler says it's C99 then take its word for it */ #if !defined(MH_UINT32) && ( \ defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L ) #include #define MH_UINT32 uint32_t #endif /* Otherwise try testing against max value macros from limit.h */ #if !defined(MH_UINT32) #include #if (USHRT_MAX == 0xffffffffUL) #define MH_UINT32 unsigned short #elif (UINT_MAX == 0xffffffffUL) #define MH_UINT32 unsigned int #elif (ULONG_MAX == 0xffffffffUL) #define MH_UINT32 unsigned long #endif #endif #if !defined(MH_UINT32) #error Unable to determine type name for unsigned 32-bit int #endif MH_UINT32 attribute_hidden PMurHash32(MH_UINT32 seed, const void *key, int len) { static MH_UINT32(*f)(MH_UINT32, const void*, int) = NULL; if (!f) { f = (MH_UINT32(*)(MH_UINT32, const void*, int)) R_GetCCallable("digest", "PMurHash32"); } return f(seed, key, len); } #ifdef __cplusplus } #endif #endif /* __PMURHASH_H__ */ digest/inst/doc/0000755000176200001440000000000014660633605013264 5ustar liggesusersdigest/inst/doc/sha1.R0000644000176200001440000000537014660633605014250 0ustar liggesusers# faq7_31 # FAQ 7.31 a0 <- 2 b <- sqrt(a0) a1 <- b ^ 2 identical(a0, a1) a0 - a1 a <- c(a0, a1) # hexadecimal representation sprintf("%a", a) # faq7_31digest library(digest) # different hashes with digest sapply(a, digest, algo = "sha1") # same hash with sha1 with default digits (14) sapply(a, sha1) # larger digits can lead to different hashes sapply(a, sha1, digits = 15) # decreasing the number of digits gives a stronger truncation # the hash will change when then truncation gives a different result # case where truncating gives same hexadecimal value sapply(a, sha1, digits = 13) sapply(a, sha1, digits = 10) # case where truncating gives different hexadecimal value c(sha1(pi), sha1(pi, digits = 13), sha1(pi, digits = 10)) # sha1_lm_sum # taken from the help file of lm.influence lm_SR <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LifeCycleSavings) lm_sum <- summary(lm_SR) class(lm_sum) # str() gives the structure of the lm object str(lm_sum) # extract the coefficients and their standard error coef_sum <- coef(lm_sum)[, c("Estimate", "Std. Error")] # extract sigma sigma <- lm_sum$sigma # check the class of each component class(coef_sum) class(sigma) # sha1() has methods for both matrix and numeric # because the values originate from floating point arithmetic it is better to use a low number of digits sha1(coef_sum, digits = 4) sha1(sigma, digits = 4) # we want a single hash # combining the components in a list is a solution that works sha1(list(coef_sum, sigma), digits = 4) # now turn everything into an S3 method # - a function with name "sha1.classname" # - must have the same arguments as sha1() sha1.summary.lm <- function(x, digits = 4, zapsmall = 7){ coef_sum <- coef(x)[, c("Estimate", "Std. Error")] sigma <- x$sigma combined <- list(coef_sum, sigma) sha1(combined, digits = digits, zapsmall = zapsmall) } sha1(lm_sum) # try an altered dataset LCS2 <- LifeCycleSavings[rownames(LifeCycleSavings) != "Zambia", ] lm_SR2 <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LCS2) sha1(summary(lm_SR2)) # sha1_lm class(lm_SR) # str() gives the structure of the lm object str(lm_SR) # extract the model and the terms lm_model <- lm_SR$model lm_terms <- lm_SR$terms # check their class class(lm_model) # handled by sha1() class(lm_terms) # not handled by sha1() # define a method for formula sha1.formula <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){ sha1(as.character(x), digits = digits, zapsmall = zapsmall, algo = algo) } sha1(lm_terms) sha1(lm_model) # define a method for lm sha1.lm <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){ lm_model <- x$model lm_terms <- x$terms combined <- list(lm_model, lm_terms) sha1(combined, digits = digits, zapsmall = zapsmall, ..., algo = algo) } sha1(lm_SR) sha1(lm_SR2) digest/inst/doc/sha1.html0000644000176200001440000021265714660633605015023 0ustar liggesusers Calculating SHA1 hashes with digest() and sha1()

Calculating SHA1 hashes with digest() and sha1()

Thierry Onkelinx and Dirk Eddelbuettel

Written Jan 2016, updated Jan 2018 and Oct 2020

NB: This vignette is (still) work-in-progress and not yet complete.

Short intro on hashes

TBD

Difference between digest() and sha1()

R FAQ 7.31 illustrates potential problems with floating point arithmetic. Mathematically the equality $x = \sqrt{x}^2$ should hold. But the precision of floating points numbers is finite. Hence some rounding is done, leading to numbers which are no longer identical.

An illustration:

> # FAQ 7.31
> a0 <- 2
> b <- sqrt(a0)
> a1 <- b ^ 2
> identical(a0, a1)
[1] FALSE
> a0 - a1
[1] -4.440892e-16
> a <- c(a0, a1)
> # hexadecimal representation
> sprintf("%a", a)
[1] "0x1p+1"               "0x1.0000000000001p+1"

Although the difference is small, any difference will result in different hash when using the digest() function. However, the sha1() function tackles this problem by using the hexadecimal representation of the numbers and truncates that representation to a certain number of digits prior to calculating the hash function.

> library(digest)
> # different hashes with digest
> sapply(a, digest, algo = "sha1")
[1] "315a5aa84aa6cfa4f3fb4b652a596770be0365e8"
[2] "5e3999bf79c230f7430e97d7f30070a7eff5ec92"
> # same hash with sha1 with default digits (14)
> sapply(a, sha1)
[1] "8a938d8f5fb9b8ccb6893aa1068babcc517f32d4"
[2] "8a938d8f5fb9b8ccb6893aa1068babcc517f32d4"
> # larger digits can lead to different hashes
> sapply(a, sha1, digits = 15)
[1] "98eb1dc9fada00b945d3ef01c77049ee5a4b7b9c"
[2] "5a173e2445df1134908037f69ac005fbd8afee74"
> # decreasing the number of digits gives a stronger truncation
> # the hash will change when then truncation gives a different result
> # case where truncating gives same hexadecimal value
> sapply(a, sha1, digits = 13)
[1] "43b3b465c975af322c85473190a9214b79b79bf6"
[2] "43b3b465c975af322c85473190a9214b79b79bf6"
> sapply(a, sha1, digits = 10)
[1] "6b537a9693c750ed535ca90527152f06e358aa3a"
[2] "6b537a9693c750ed535ca90527152f06e358aa3a"
> # case where truncating gives different hexadecimal value
> c(sha1(pi), sha1(pi, digits = 13), sha1(pi, digits = 10))
[1] "169388cf1ce60dc4e9904a22bc934c5db33d975b"
[2] "20dc38536b6689d5f2d053f30efb09c44af78378"
[3] "3a727417bd1807af2f0148cf3de69abff32c23ec"

The result of floating point arithematic on 32-bit and 64-bit can be slightly different. E.g. print(pi ^ 11, 22) returns 294204.01797389047 on 32-bit and 294204.01797389053 on 64-bit. Note that only the last 2 digits are different.

command 32-bit 64-bit
print(pi ^ 11, 22) 294204.01797389047 294204.01797389053
sprintf("%a", pi ^ 11) "0x1.1f4f01267bf5fp+18" "0x1.1f4f01267bf6p+18"
digest(pi ^ 11, algo = "sha1") "c5efc7f167df1bb402b27cf9b405d7cebfba339a" "b61f6fea5e2a7952692cefe8bba86a00af3de713"
sha1(pi ^ 11, digits = 14) "5c7740500b8f78ec2354ea6af58ea69634d9b7b1" "4f3e296b9922a7ddece2183b1478d0685609a359"
sha1(pi ^ 11, digits = 13) "372289f87396b0877ccb4790cf40bcb5e658cad7" "372289f87396b0877ccb4790cf40bcb5e658cad7"
sha1(pi ^ 11, digits = 10) "c05965af43f9566bfb5622f335817f674abfc9e4" "c05965af43f9566bfb5622f335817f674abfc9e4"

Choosing digest() or sha1()

TBD

Creating a sha1 method for other classes

How to

  1. Identify the relevant components for the hash.
  2. Determine the class of each relevant component and check if they are handled by sha1().
    • Write a method for each component class not yet handled by sha1.
  3. Extract the relevant components.
  4. Combine the relevant components into a list. Not required in case of a single component.
  5. Apply sha1() on the (list of) relevant component(s).
  6. Turn this into a function with name sha1._classname_.
  7. sha1._classname_ needs exactly the same arguments as sha1()
  8. Choose sensible defaults for the arguments
    • zapsmall = 7 is recommended.
    • digits = 14 is recommended in case all numerics are data.
    • digits = 4 is recommended in case some numerics stem from floating point arithmetic.

summary.lm

Let’s illustrate this using the summary of a simple linear regression. Suppose that we want a hash that takes into account the coefficients, their standard error and sigma.

> # taken from the help file of lm.influence
> lm_SR <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LifeCycleSavings)
> lm_sum <- summary(lm_SR)
> class(lm_sum)
[1] "summary.lm"
> # str() gives the structure of the lm object
> str(lm_sum)
List of 11
 $ call         : language lm(formula = sr ~ pop15 + pop75 + dpi + ddpi, data = LifeCycleSavings)
 $ terms        :Classes 'terms', 'formula'  language sr ~ pop15 + pop75 + dpi + ddpi
  .. ..- attr(*, "variables")= language list(sr, pop15, pop75, dpi, ddpi)
  .. ..- attr(*, "factors")= int [1:5, 1:4] 0 1 0 0 0 0 0 1 0 0 ...
  .. .. ..- attr(*, "dimnames")=List of 2
  .. .. .. ..$ : chr [1:5] "sr" "pop15" "pop75" "dpi" ...
  .. .. .. ..$ : chr [1:4] "pop15" "pop75" "dpi" "ddpi"
  .. ..- attr(*, "term.labels")= chr [1:4] "pop15" "pop75" "dpi" "ddpi"
  .. ..- attr(*, "order")= int [1:4] 1 1 1 1
  .. ..- attr(*, "intercept")= int 1
  .. ..- attr(*, "response")= int 1
  .. ..- attr(*, ".Environment")=<environment: R_GlobalEnv> 
  .. ..- attr(*, "predvars")= language list(sr, pop15, pop75, dpi, ddpi)
  .. ..- attr(*, "dataClasses")= Named chr [1:5] "numeric" "numeric" "numeric" "numeric" ...
  .. .. ..- attr(*, "names")= chr [1:5] "sr" "pop15" "pop75" "dpi" ...
 $ residuals    : Named num [1:50] 0.864 0.616 2.219 -0.698 3.553 ...
  ..- attr(*, "names")= chr [1:50] "Australia" "Austria" "Belgium" "Bolivia" ...
 $ coefficients : num [1:5, 1:4] 28.566087 -0.461193 -1.691498 -0.000337 0.409695 ...
  ..- attr(*, "dimnames")=List of 2
  .. ..$ : chr [1:5] "(Intercept)" "pop15" "pop75" "dpi" ...
  .. ..$ : chr [1:4] "Estimate" "Std. Error" "t value" "Pr(>|t|)"
 $ aliased      : Named logi [1:5] FALSE FALSE FALSE FALSE FALSE
  ..- attr(*, "names")= chr [1:5] "(Intercept)" "pop15" "pop75" "dpi" ...
 $ sigma        : num 3.8
 $ df           : int [1:3] 5 45 5
 $ r.squared    : num 0.338
 $ adj.r.squared: num 0.28
 $ fstatistic   : Named num [1:3] 5.76 4 45
  ..- attr(*, "names")= chr [1:3] "value" "numdf" "dendf"
 $ cov.unscaled : num [1:5, 1:5] 3.74 -7.24e-02 -4.46e-01 -7.86e-05 -1.88e-02 ...
  ..- attr(*, "dimnames")=List of 2
  .. ..$ : chr [1:5] "(Intercept)" "pop15" "pop75" "dpi" ...
  .. ..$ : chr [1:5] "(Intercept)" "pop15" "pop75" "dpi" ...
 - attr(*, "class")= chr "summary.lm"
> # extract the coefficients and their standard error
> coef_sum <- coef(lm_sum)[, c("Estimate", "Std. Error")]
> # extract sigma
> sigma <- lm_sum$sigma
> # check the class of each component
> class(coef_sum)
[1] "matrix" "array" 
> class(sigma)
[1] "numeric"
> # sha1() has methods for both matrix and numeric
> # because the values originate from floating point arithmetic it is better to use a low number of digits
> sha1(coef_sum, digits = 4)
[1] "3f0b0c552f94d753fcc8deb4d3e9fc11a83197af"
> sha1(sigma, digits = 4)
[1] "cbc83d1791ef1eeadd824ea9a038891b5889056b"
> # we want a single hash
> # combining the components in a list is a solution that works
> sha1(list(coef_sum, sigma), digits = 4)
[1] "476d27265365cd41662eedf059b335d38a221cc2"
> # now turn everything into an S3 method
> #   - a function with name "sha1.classname"
> #   - must have the same arguments as sha1()
> sha1.summary.lm <- function(x, digits = 4, zapsmall = 7){
+     coef_sum <- coef(x)[, c("Estimate", "Std. Error")]
+     sigma <- x$sigma
+     combined <- list(coef_sum, sigma)
+     sha1(combined, digits = digits, zapsmall = zapsmall)
+ }
> sha1(lm_sum)
[1] "476d27265365cd41662eedf059b335d38a221cc2"
> # try an altered dataset
> LCS2 <- LifeCycleSavings[rownames(LifeCycleSavings) != "Zambia", ]
> lm_SR2 <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LCS2)
> sha1(summary(lm_SR2))
[1] "90beb028833bf0542997fde7c3f19e5b9fdfeef4"

lm

Let’s illustrate this using the summary of a simple linear regression. Suppose that we want a hash that takes into account the coefficients, their standard error and sigma.

> class(lm_SR)
[1] "lm"
> # str() gives the structure of the lm object
> str(lm_SR)
List of 12
 $ coefficients : Named num [1:5] 28.566087 -0.461193 -1.691498 -0.000337 0.409695
  ..- attr(*, "names")= chr [1:5] "(Intercept)" "pop15" "pop75" "dpi" ...
 $ residuals    : Named num [1:50] 0.864 0.616 2.219 -0.698 3.553 ...
  ..- attr(*, "names")= chr [1:50] "Australia" "Austria" "Belgium" "Bolivia" ...
 $ effects      : Named num [1:50] -68.38 -14.29 7.3 -3.52 -7.94 ...
  ..- attr(*, "names")= chr [1:50] "(Intercept)" "pop15" "pop75" "dpi" ...
 $ rank         : int 5
 $ fitted.values: Named num [1:50] 10.57 11.45 10.95 6.45 9.33 ...
  ..- attr(*, "names")= chr [1:50] "Australia" "Austria" "Belgium" "Bolivia" ...
 $ assign       : int [1:5] 0 1 2 3 4
 $ qr           :List of 5
  ..$ qr   : num [1:50, 1:5] -7.071 0.141 0.141 0.141 0.141 ...
  .. ..- attr(*, "dimnames")=List of 2
  .. .. ..$ : chr [1:50] "Australia" "Austria" "Belgium" "Bolivia" ...
  .. .. ..$ : chr [1:5] "(Intercept)" "pop15" "pop75" "dpi" ...
  .. ..- attr(*, "assign")= int [1:5] 0 1 2 3 4
  ..$ qraux: num [1:5] 1.14 1.17 1.16 1.15 1.05
  ..$ pivot: int [1:5] 1 2 3 4 5
  ..$ tol  : num 1e-07
  ..$ rank : int 5
  ..- attr(*, "class")= chr "qr"
 $ df.residual  : int 45
 $ xlevels      : Named list()
 $ call         : language lm(formula = sr ~ pop15 + pop75 + dpi + ddpi, data = LifeCycleSavings)
 $ terms        :Classes 'terms', 'formula'  language sr ~ pop15 + pop75 + dpi + ddpi
  .. ..- attr(*, "variables")= language list(sr, pop15, pop75, dpi, ddpi)
  .. ..- attr(*, "factors")= int [1:5, 1:4] 0 1 0 0 0 0 0 1 0 0 ...
  .. .. ..- attr(*, "dimnames")=List of 2
  .. .. .. ..$ : chr [1:5] "sr" "pop15" "pop75" "dpi" ...
  .. .. .. ..$ : chr [1:4] "pop15" "pop75" "dpi" "ddpi"
  .. ..- attr(*, "term.labels")= chr [1:4] "pop15" "pop75" "dpi" "ddpi"
  .. ..- attr(*, "order")= int [1:4] 1 1 1 1
  .. ..- attr(*, "intercept")= int 1
  .. ..- attr(*, "response")= int 1
  .. ..- attr(*, ".Environment")=<environment: R_GlobalEnv> 
  .. ..- attr(*, "predvars")= language list(sr, pop15, pop75, dpi, ddpi)
  .. ..- attr(*, "dataClasses")= Named chr [1:5] "numeric" "numeric" "numeric" "numeric" ...
  .. .. ..- attr(*, "names")= chr [1:5] "sr" "pop15" "pop75" "dpi" ...
 $ model        :'data.frame':  50 obs. of  5 variables:
  ..$ sr   : num [1:50] 11.43 12.07 13.17 5.75 12.88 ...
  ..$ pop15: num [1:50] 29.4 23.3 23.8 41.9 42.2 ...
  ..$ pop75: num [1:50] 2.87 4.41 4.43 1.67 0.83 2.85 1.34 0.67 1.06 1.14 ...
  ..$ dpi  : num [1:50] 2330 1508 2108 189 728 ...
  ..$ ddpi : num [1:50] 2.87 3.93 3.82 0.22 4.56 2.43 2.67 6.51 3.08 2.8 ...
  ..- attr(*, "terms")=Classes 'terms', 'formula'  language sr ~ pop15 + pop75 + dpi + ddpi
  .. .. ..- attr(*, "variables")= language list(sr, pop15, pop75, dpi, ddpi)
  .. .. ..- attr(*, "factors")= int [1:5, 1:4] 0 1 0 0 0 0 0 1 0 0 ...
  .. .. .. ..- attr(*, "dimnames")=List of 2
  .. .. .. .. ..$ : chr [1:5] "sr" "pop15" "pop75" "dpi" ...
  .. .. .. .. ..$ : chr [1:4] "pop15" "pop75" "dpi" "ddpi"
  .. .. ..- attr(*, "term.labels")= chr [1:4] "pop15" "pop75" "dpi" "ddpi"
  .. .. ..- attr(*, "order")= int [1:4] 1 1 1 1
  .. .. ..- attr(*, "intercept")= int 1
  .. .. ..- attr(*, "response")= int 1
  .. .. ..- attr(*, ".Environment")=<environment: R_GlobalEnv> 
  .. .. ..- attr(*, "predvars")= language list(sr, pop15, pop75, dpi, ddpi)
  .. .. ..- attr(*, "dataClasses")= Named chr [1:5] "numeric" "numeric" "numeric" "numeric" ...
  .. .. .. ..- attr(*, "names")= chr [1:5] "sr" "pop15" "pop75" "dpi" ...
 - attr(*, "class")= chr "lm"
> # extract the model and the terms
> lm_model <- lm_SR$model
> lm_terms <- lm_SR$terms
> # check their class
> class(lm_model) # handled by sha1()
[1] "data.frame"
> class(lm_terms) # not handled by sha1()
[1] "terms"   "formula"
> # define a method for formula
> sha1.formula <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){
+     sha1(as.character(x), digits = digits, zapsmall = zapsmall, algo = algo)
+ }
> sha1(lm_terms)
[1] "2737d209720aa7d1c0555050ad06ebe89f3850cd"
> sha1(lm_model)
[1] "27b7dd9e3e09b9577da6947b8473b63a1d0b6eb4"
> # define a method for lm
> sha1.lm <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){
+     lm_model <- x$model
+     lm_terms <- x$terms
+     combined <- list(lm_model, lm_terms)
+     sha1(combined, digits = digits, zapsmall = zapsmall, ..., algo = algo)
+ }
> sha1(lm_SR)
[1] "7eda2a9d58e458c8e782e40ce140d62b836b2a2f"
> sha1(lm_SR2)
[1] "4d3abdb1f17bd12fdf9d9b91a2ad04c07824fe4a"

Using hashes to track changes in analysis

Use case

  • automated analysis

  • update frequency of the data might be lower than the frequency of automated analysis

  • similar analyses on many datasets (e.g. many species in ecology)

  • analyses that require a lot of computing time

    • not rerunning an analysis because nothing has changed saves enough resources to compensate the overhead of tracking changes

  • Bundle all relevant information on an analysis in a class

    • data
    • method
    • formula
    • other metadata
    • resulting model

  • calculate sha1()

    file fingerprint ~ sha1() on the stable parts

    status fingerprint ~ sha1() on the parts that result for the model

  1. Prepare analysis objects
  2. Store each analysis object in a rda file which uses the file fingerprint as filename
    • File will already exist when no change in analysis
    • Don’t overwrite existing files
  3. Loop over all rda files
    • Do nothing if the analysis was run
    • Otherwise run the analysis and update the status and status fingerprint
digest/inst/doc/sha1.md0000644000176200001440000001620314146242237014440 0ustar liggesusers --- title: "Calculating SHA1 hashes with digest() and sha1()" author: "Thierry Onkelinx and Dirk Eddelbuettel" date: "Written Jan 2016, updated Jan 2018 and Oct 2020" css: "water.css" --- NB: This vignette is (still) work-in-progress and not yet complete. ## Short intro on hashes TBD ## Difference between `digest()` and `sha1()` R [FAQ 7.31](https://cran.r-project.org/doc/FAQ/R-FAQ.html#Why-doesn_0027t-R-think-these-numbers-are-equal_003f) illustrates potential problems with floating point arithmetic. Mathematically the equality $x = \sqrt{x}^2$ should hold. But the precision of floating points numbers is finite. Hence some rounding is done, leading to numbers which are no longer identical. An illustration: ```{#faq7_31 .R} # FAQ 7.31 a0 <- 2 b <- sqrt(a0) a1 <- b ^ 2 identical(a0, a1) a0 - a1 a <- c(a0, a1) # hexadecimal representation sprintf("%a", a) ``` Although the difference is small, any difference will result in different hash when using the `digest()` function. However, the `sha1()` function tackles this problem by using the hexadecimal representation of the numbers and truncates that representation to a certain number of digits prior to calculating the hash function. ```{#faq7_31digest .R} library(digest) # different hashes with digest sapply(a, digest, algo = "sha1") # same hash with sha1 with default digits (14) sapply(a, sha1) # larger digits can lead to different hashes sapply(a, sha1, digits = 15) # decreasing the number of digits gives a stronger truncation # the hash will change when then truncation gives a different result # case where truncating gives same hexadecimal value sapply(a, sha1, digits = 13) sapply(a, sha1, digits = 10) # case where truncating gives different hexadecimal value c(sha1(pi), sha1(pi, digits = 13), sha1(pi, digits = 10)) ``` The result of floating point arithematic on 32-bit and 64-bit can be slightly different. E.g. `print(pi ^ 11, 22)` returns `294204.01797389047` on 32-bit and `294204.01797389053` on 64-bit. Note that only the last 2 digits are different. | command | 32-bit | 64-bit| | - | - | - | | `print(pi ^ 11, 22)` | `294204.01797389047` | `294204.01797389053` | | `sprintf("%a", pi ^ 11)`| `"0x1.1f4f01267bf5fp+18"` | `"0x1.1f4f01267bf6p+18"` | | `digest(pi ^ 11, algo = "sha1")` | `"c5efc7f167df1bb402b27cf9b405d7cebfba339a"` | `"b61f6fea5e2a7952692cefe8bba86a00af3de713"`| | `sha1(pi ^ 11, digits = 14)` | `"5c7740500b8f78ec2354ea6af58ea69634d9b7b1"` | `"4f3e296b9922a7ddece2183b1478d0685609a359"` | | `sha1(pi ^ 11, digits = 13)` | `"372289f87396b0877ccb4790cf40bcb5e658cad7"` | `"372289f87396b0877ccb4790cf40bcb5e658cad7"` | | `sha1(pi ^ 11, digits = 10)` | `"c05965af43f9566bfb5622f335817f674abfc9e4"` | `"c05965af43f9566bfb5622f335817f674abfc9e4"` | ## Choosing `digest()` or `sha1()` TBD ## Creating a sha1 method for other classes ### How to 1. Identify the relevant components for the hash. 1. Determine the class of each relevant component and check if they are handled by `sha1()`. - Write a method for each component class not yet handled by `sha1`. 1. Extract the relevant components. 1. Combine the relevant components into a list. Not required in case of a single component. 1. Apply `sha1()` on the (list of) relevant component(s). 1. Turn this into a function with name sha1._classname_. 1. sha1._classname_ needs exactly the same arguments as `sha1()` 1. Choose sensible defaults for the arguments - `zapsmall = 7` is recommended. - `digits = 14` is recommended in case all numerics are data. - `digits = 4` is recommended in case some numerics stem from floating point arithmetic. ### summary.lm Let's illustrate this using the summary of a simple linear regression. Suppose that we want a hash that takes into account the coefficients, their standard error and sigma. ```{#sha1_lm_sum .R} # taken from the help file of lm.influence lm_SR <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LifeCycleSavings) lm_sum <- summary(lm_SR) class(lm_sum) # str() gives the structure of the lm object str(lm_sum) # extract the coefficients and their standard error coef_sum <- coef(lm_sum)[, c("Estimate", "Std. Error")] # extract sigma sigma <- lm_sum$sigma # check the class of each component class(coef_sum) class(sigma) # sha1() has methods for both matrix and numeric # because the values originate from floating point arithmetic it is better to use a low number of digits sha1(coef_sum, digits = 4) sha1(sigma, digits = 4) # we want a single hash # combining the components in a list is a solution that works sha1(list(coef_sum, sigma), digits = 4) # now turn everything into an S3 method # - a function with name "sha1.classname" # - must have the same arguments as sha1() sha1.summary.lm <- function(x, digits = 4, zapsmall = 7){ coef_sum <- coef(x)[, c("Estimate", "Std. Error")] sigma <- x$sigma combined <- list(coef_sum, sigma) sha1(combined, digits = digits, zapsmall = zapsmall) } sha1(lm_sum) # try an altered dataset LCS2 <- LifeCycleSavings[rownames(LifeCycleSavings) != "Zambia", ] lm_SR2 <- lm(sr ~ pop15 + pop75 + dpi + ddpi, data = LCS2) sha1(summary(lm_SR2)) ``` ### lm Let's illustrate this using the summary of a simple linear regression. Suppose that we want a hash that takes into account the coefficients, their standard error and sigma. ```{#sha1_lm .R} class(lm_SR) # str() gives the structure of the lm object str(lm_SR) # extract the model and the terms lm_model <- lm_SR$model lm_terms <- lm_SR$terms # check their class class(lm_model) # handled by sha1() class(lm_terms) # not handled by sha1() # define a method for formula sha1.formula <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){ sha1(as.character(x), digits = digits, zapsmall = zapsmall, algo = algo) } sha1(lm_terms) sha1(lm_model) # define a method for lm sha1.lm <- function(x, digits = 14, zapsmall = 7, ..., algo = "sha1"){ lm_model <- x$model lm_terms <- x$terms combined <- list(lm_model, lm_terms) sha1(combined, digits = digits, zapsmall = zapsmall, ..., algo = algo) } sha1(lm_SR) sha1(lm_SR2) ``` ## Using hashes to track changes in analysis Use case - automated analysis - update frequency of the data might be lower than the frequency of automated analysis - similar analyses on many datasets (e.g. many species in ecology) - analyses that require a lot of computing time - not rerunning an analysis because nothing has changed saves enough resources to compensate the overhead of tracking changes - Bundle all relevant information on an analysis in a class - data - method - formula - other metadata - resulting model - calculate `sha1()` file fingerprint ~ `sha1()` on the stable parts status fingerprint ~ `sha1()` on the parts that result for the model 1. Prepare analysis objects 1. Store each analysis object in a rda file which uses the file fingerprint as filename - File will already exist when no change in analysis - Don't overwrite existing files 1. Loop over all rda files - Do nothing if the analysis was run - Otherwise run the analysis and update the status and status fingerprint digest/inst/tinytest/0000755000176200001440000000000014636020207014371 5ustar liggesusersdigest/inst/tinytest/test_blake3.R0000644000176200001440000000303013741612225016713 0ustar liggesuserssuppressMessages(library(digest)) isWindows <- Sys.info()[["sysname"]] == "Windows" res <- digest(charToRaw("test"), "blake3", serialize = FALSE) # generated using the blake3 rust implementation expected <- "4878ca0425c739fa427f7eda20fe845f6b2e46ba5fe2a14df5b1e32f50603215" expect_equal( res, expected ) res <- digest(charToRaw("test"), "blake3", serialize = FALSE, raw = TRUE) expect_equal( paste0(res, collapse = ""), expected ) ## blake3 example blake3Input <- c( "abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "" ) blake3Output <- c( "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85", "c19012cc2aaf0dc3d8e5c45a1b79114d2df42abb2a410bf54be09e891af06ff8", "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262" ) for (i in seq(along = blake3Input)) { blake3 <- digest(blake3Input[i], algo = "blake3", serialize = FALSE) expect_equal( blake3, blake3Output[i] ) } if (isWindows) exit_file("Skipping remainder on Windows platform") # test file # used the b3sum rust crate file <- tempfile() writeLines("test", file) res <- digest(file = file, algo = "blake3") try(file.remove(file)) expect_equal( res, "dea2b412aa90f1b43a06ca5e8b8feafec45ae1357971322749480f4e1572eaa2" ) # try another long text file <- tempfile() text <- paste0(rep(LETTERS, 1000), collapse = "-") writeLines(text, file) res <- digest(file = file, algo = "blake3") try(file.remove(file)) expect_equal( res, "a84aef38d9a7ad55fa458d9d1857eb895832e358385e8b6466c6af54ea46eeaa" ) digest/inst/tinytest/test_sha1.R0000644000176200001440000003243513574673675016445 0ustar liggesusers ## tests for digest, taken from the examples in the manual page suppressMessages(library(digest)) # calculate sha1 fingerprints x.numeric <- c(seq(0, 1, length = 4 ^ 3), -Inf, Inf, NA, NaN) x.list <- list(letters, x.numeric) x.dataframe <- data.frame(X = letters, Y = x.numeric[2], Z = factor(letters), stringsAsFactors = FALSE) x.matrix.num <- as.matrix(x.numeric) x.matrix.letter <- as.matrix(letters) x.dataframe.round <- x.dataframe x.dataframe.round$Y <- signif(x.dataframe.round$Y, 14) x.factor <- factor(letters) x.array.num <- as.array(x.numeric) x.formula <- a~b+c|d x.paren_formula <- a~(b+c) x.no_paren_formula <- a~b+c # tests using detailed numbers expect_false(identical(x.numeric, signif(x.numeric, 14))) expect_false(identical(x.matrix.num, signif(x.matrix.num, 14))) # returns the correct SHA1 expect_true( identical( sha1(x.numeric), { z <- digest:::num2hex(x.numeric) attr(z, "digest::sha1") <- list( class = class(x.numeric), digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) # Verify that all numeric values (especially +-Inf and NA/NaN) return unique # SHA1 hashes expect_false( any(duplicated(sapply(x.numeric, sha1))) ) expect_true( identical( sha1(letters), { z <- letters attr(z, "digest::sha1") <- list( class = "character", digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) expect_true( identical( sha1(x.list), { z <- sapply(x.list, sha1) attr(z, "digest::sha1") <- list( class = "list", digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) options(sha1PackageVersion = "0.6.22.1") expect_true( identical( sha1(x.dataframe), { z <- sapply(x.dataframe, sha1) attr(z, "digest::sha1") <- list( class = "data.frame", digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) ## expect_true( ## identical( ## sha1(x.matrix.num), ## { ## z <- matrix( ## apply(x.matrix.num, 2, digest:::num2hex), ## ncol = ncol(x.matrix.num) ## ) ## attr(z, "digest::sha1") <- list( ## class = "matrix", ## digits = 14L, ## zapsmall = 7L ## ) ## digest(z, algo = "sha1") ## } ## ) ## ) ## expect_true( ## identical( ## sha1(x.matrix.letter), ## { ## z <- x.matrix.letter ## attr(z, "digest::sha1") <- list( ## class = "matrix", ## digits = 14L, ## zapsmall = 7L ## ) ## digest(z, algo = "sha1") ## } ## ) ## ) stopifnot( identical( sha1(x.factor), { z <- x.factor attr(z, "digest::sha1") <- list( class = "factor", digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) # a matrix and a vector should have a different hash expect_true( !identical( sha1(x.numeric), sha1(matrix(x.numeric, nrow = 1)) ) ) expect_true( !identical( sha1(x.numeric), sha1(matrix(x.numeric, ncol = 1)) ) ) expect_true( !identical( sha1(letters), sha1(matrix(letters, nrow = 1)) ) ) expect_true( !identical( sha1(letters), sha1(matrix(letters, ncol = 1)) ) ) # character(0) and numeric(0) should have a different hash expect_true(!identical(sha1(character(0)), sha1(numeric(0)))) # a POSIXct and a POSIXlt should give a different hash z <- as.POSIXct("2015-01-02 03:04:06.07", tz = "UTC") expect_true( !identical( sha1(z), sha1(as.POSIXlt(z)) ) ) lm.model.0 <- lm(weight ~ Time, data = ChickWeight) lm.model.1 <- lm(weight ~ 1, data = ChickWeight) glm.model.0 <- glm(weight ~ Time, data = ChickWeight, family = poisson) glm.model.1 <- glm(weight ~ 1, data = ChickWeight, family = poisson) anova.list <- list( lm = anova(lm.model.0, lm.model.1), glm = anova(glm.model.0, glm.model.1), glm.test = anova(glm.model.0, glm.model.1, test = "Chisq") ) # works with lm anova" expect_true( identical( sha1(anova.list[["lm"]]), { y <- apply( anova.list[["lm"]], 1, digest:::num2hex, digits = 4, zapsmall = 7 ) attr(y, "digest::sha1") <- list( class = c("anova", "data.frame"), digits = 4L, zapsmall = 7L ) digest(y, algo = "sha1") } ) ) # works with glm anova" expect_true( identical( sha1(anova.list[["glm"]]), { y <- apply( anova.list[["glm"]], 1, digest:::num2hex, digits = 4, zapsmall = 7 ) attr(y, "digest::sha1") <- list( class = c("anova", "data.frame"), digits = 4L, zapsmall = 7L ) digest(y, algo = "sha1") } ) ) expect_true( identical( sha1(anova.list[["glm.test"]]), { y <- apply( anova.list[["glm.test"]], 1, digest:::num2hex, digits = 4, zapsmall = 7 ) attr(y, "digest::sha1") <- list( class = c("anova", "data.frame"), digits = 4L, zapsmall = 7L ) digest(y, algo = "sha1") } ) ) expect_true( identical( sha1(x.formula, environment=FALSE), { y <- sapply( X=x.formula, FUN=sha1, digits=14L, zapsmall=7L, ...=list(environment=FALSE), algo="sha1" ) attr(y, "digest::sha1") <- list( class="formula", digits=14L, zapsmall=7L, environment=FALSE ) digest(y, algo="sha1") } ) ) expect_true( identical( sha1(x.formula), { y <- c( sapply( X=x.formula, FUN=sha1, digits=14L, zapsmall=7L, ...=list(environment=TRUE), algo="sha1" ), digest(environment(x.formula), algo="sha1") ) attr(y, "digest::sha1") <- list( class="formula", digits=14L, zapsmall=7L ) digest(y, algo="sha1") } ) ) expect_true( sha1(x.paren_formula) != sha1(x.no_paren_formula) ) test.element <- list( # NULL NULL, # empty classes logical(0), integer(0), numeric(0), character(0), list(), data.frame(), # scalar TRUE, FALSE, 1L, 1, "a", # date. Make sure to add the time zone. Otherwise the test might fail as.POSIXct("2015-01-02 03:04:06.07", tz = "UTC"), # vector c(TRUE, FALSE), 1:3, seq(0, 10, length = 4), letters[1:3], factor(letters[4:6]), as.POSIXct(c("2015-01-02 03:04:06.07", "1960-12-31 23:59:59"), tz = "UTC") ) select.vector <- which(sapply(test.element, length) > 1) test.element <- c( test.element, # add a data.frame list(expand.grid(test.element[select.vector])), # add a list list(test.element[select.vector]), # add matrices list(matrix(1:10)), list(matrix(seq(0, 10, length = 4))), list(matrix(letters)) ) # different values for digits or zapsmall gives different hashes # expect for NULL expect_true( identical( sha1(NULL, digits = 14), sha1(NULL, digits = 13) ) ) expect_true( identical( sha1(NULL, zapsmall = 14), sha1(NULL, zapsmall = 13) ) ) for (i in tail(seq_along(test.element), -1)) { expect_true( !identical( sha1(test.element[[i]], digits = 14), sha1(test.element[[i]], digits = 13) ) ) expect_true( !identical( sha1(test.element[[i]], zapsmall = 7), sha1(test.element[[i]], zapsmall = 6) ) ) } test.element <- c(test.element, anova.list) #cat("\ncorrect <- c(\n") #cat( # sprintf(" \"%s\"", sapply(test.element, sha1)), # sep = ",\n" #) #cat(")\n") correct <- c( "8d9c05ec7ae28b219c4c56edbce6a721bd68af82", "d61eeea290dd09c5a3eba41c2b3174b6e4e2366d", "af23305d27f0409c91bdb86ba7c0cdb2e09a5dc6", "0c9ca70ce773deb0d9c0b0579c3b94856edf15cc", "095886422ad26e315c0960ef6b09842a1f9cc0ce", "6cc04c6c432bb91e210efe0b25c6ca809e6df2e3", "c1113ba008a349de64da2a7a724e501c1eb3929b", "6e12370bdc6fc457cc154f0525e22c6aa6439f4d", "1c1b5393c68a643bc79c277c6d7374d0b30cd985", "b48c17a2ac82601ff38df374f87d76005fb61cbd", "35280c99aa6a48bfc2810b72b763ccac0f632207", "f757cc017308d217f35ed8f0c001a57b97308fb7", "cfcf101b8449af67d34cdc1bcb0432fe9e4de08e", "a14384d1997440bad13b97b3ccfb3b8c0392e79a", "555f6bea49e58a2c2541060a21c2d4f9078c3086", "631d18dec342e2cb87614864ba525ebb9ad6a124", "b6c04f16b6fdacc794ea75c8c8dd210f99fafa65", "25485ba7e315956267b3fdc521b421bbb046325d", "6def3ca353dfc1a904bddd00e6a410d41ac7ab01", "cf220bcf84c3d0ab1b01f8f764396941d15ff20f", "2af8021b838f613aee7670bed19d0ddf1d6bc0c1", "270ed85d46524a59e3274d89a1bbf693521cb6af", "60e09482f12fda20f7d4a70e379c969c5a73f512", "10380001af2a541b5feefc7aab9f719b67330a42", "4580ff07f27eb8321421efac1676a80d9239572a", "d3022c5a223caaf77e9c564e024199e5d6f51bd5", "f54742ac61edd8c3980354620816c762b524dfc7" ) # each object should yield a different hash expect_true(!any(duplicated(correct))) # returns the same SHA1 on both 32-bit and 64-bit OS" ## for (i in seq_along(test.element)) { ## expect_true( ## identical( ## sha1(test.element[[i]]), ## correct[i] ## ) ## ) ## } # does work with empty lists and data.frames expect_true(is.character(sha1(list()))) expect_true(is.character(sha1(data.frame()))) expect_true(is.character(sha1(list(a = 1, b = list(), c = data.frame())))) # does work with complex type expect_true(is.character(sha1(2 + 5i))) # single complex number expect_true(is.character(sha1(1:10 + 5i))) # vector of complex numbers # complex number with only the real part should be different from real number expect_true(sha1(2) != sha1(2 + 0i)) # does work with Date type expect_true(is.character(sha1(Sys.Date()))) expect_true(sha1(as.Date("1980-01-01")) != sha1(as.Date("1990-01-01"))) # different hashes for differently shaped arrays that contain the same data data <- 1:8 a <- array(data, dim = c(2,2,2)) # cube 2x2x2 b <- array(data, dim = c(2,4,1)) # matrix 2x4 expect_true(sha1(a) != sha1(b)) # test error message junk <- pi class(junk) <- c("A", "B") #error.message <- try(sha1(junk)) #expect_true(grepl("sha1\\(\\) has no method for the 'A', 'B' class", error.message)) junk <- function( x, y = list(...), test = TRUE, start = NULL, text = "abc", family = poisson, ... ){ sha1(x) } #expect_true(sha1(junk) == "be194e8cdae926c13fd4e2c65bf6cb7a28dd0505") expect_true(sha1(junk) == sha1(junk, environment = TRUE)) expect_true(sha1(junk) != sha1(junk, environment = FALSE)) #expect_true(sha1(matrix(integer(0))) == "e13485e1b995f3e36d43674dcbfedea08ce237bc") expect_true( !identical( sha1(matrix(integer(0))), sha1(matrix(character(0))) ) ) expect_true( !identical( sha1(matrix(integer(0))), sha1(matrix(numeric(0))) ) ) ## if (getRversion() < "3.5.0") { ## expect_true( ## identical( ## sha1(serialize("e13485e1b995f3e36d43674dcbfedea08ce237bc", NULL)), ## "93ab6a61f1a2ad50d4bf58396dc38cd3821b2eaf" ## ) ## ) ## } x <- letters for (algo in c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32")) { y <- x attr(y, "digest::sha1") <- digest:::attr_sha1(x, 14L, 7L, algo = algo) expect_true( identical( sha1(x, algo = algo), digest(y, algo = algo) ) ) } expect_true(is.character(sha1(sessionInfo()))) # check the effect of attributes from version 0.6.22.2 options(sha1PackageVersion = utils::packageVersion("digest")) check_attribute_effect <- function(x) { y <- x attr(y, "test") <- "junk" expect_false(sha1(x) == sha1(y)) } test.element <- list(2 + 5i, x.array.num, Sys.Date(), Sys.time(), y ~ x) test.element <- c(test.element, list(x.dataframe), anova.list, function(x){x}) for (z in test.element) { check_attribute_effect(z) } # check that sha1() on contributed functions maintain there hash after storing f <- tempfile(fileext = ".rds") x <- digest::sha1 saveRDS(x, f) y <- readRDS(f) expect_identical(sha1(x), sha1(y)) expect_identical(sha1(x, environment = FALSE), sha1(y, environment = FALSE)) digest/inst/tinytest/test_new_matrix_behaviour.R0000644000176200001440000000673513574673675022036 0ustar liggesusers if (getRversion() >= '4.0.0') exit_file("Skip tests for R 4.0.0 or later") library(digest) x.numeric <- c(seq(0, 1, length = 4 ^ 3), -Inf, Inf, NA, NaN) x.list <- list(letters, x.numeric) x.dataframe <- data.frame(X = letters, Y = x.numeric[2], Z = factor(letters), stringsAsFactors = FALSE) x.matrix.num <- as.matrix(x.numeric) x.matrix.letter <- as.matrix(letters) x.dataframe.round <- x.dataframe x.dataframe.round$Y <- signif(x.dataframe.round$Y, 14) x.factor <- factor(letters) x.array.num <- as.array(x.numeric) x.formula <- a~b+c|d x.paren_formula <- a~(b+c) x.no_paren_formula <- a~b+c test.element <- list( # NULL NULL, # empty classes logical(0), integer(0), numeric(0), character(0), list(), data.frame(), # scalar TRUE, FALSE, 1L, 1, "a", # date. Make sure to add the time zone. Otherwise the test might fail as.POSIXct("2015-01-02 03:04:06.07", tz = "UTC"), # vector c(TRUE, FALSE), 1:3, seq(0, 10, length = 4), letters[1:3], factor(letters[4:6]), as.POSIXct(c("2015-01-02 03:04:06.07", "1960-12-31 23:59:59"), tz = "UTC") ) expect_true( identical( sha1(x.matrix.num), { z <- matrix( apply(x.matrix.num, 2, digest:::num2hex), ncol = ncol(x.matrix.num) ) z <- digest:::add_attributes(x.matrix.num, z) attr(z, "digest::sha1") <- list( class = "matrix", digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) expect_true( identical( sha1(x.matrix.letter), { z <- x.matrix.letter attr(z, "digest::sha1") <- list( class = "matrix", digits = 14L, zapsmall = 7L ) digest(z, algo = "sha1") } ) ) correct <- c( "8d9c05ec7ae28b219c4c56edbce6a721bd68af82", "d61eeea290dd09c5a3eba41c2b3174b6e4e2366d", "af23305d27f0409c91bdb86ba7c0cdb2e09a5dc6", "0c9ca70ce773deb0d9c0b0579c3b94856edf15cc", "095886422ad26e315c0960ef6b09842a1f9cc0ce", "6cc04c6c432bb91e210efe0b25c6ca809e6df2e3", "c1113ba008a349de64da2a7a724e501c1eb3929b", "6e12370bdc6fc457cc154f0525e22c6aa6439f4d", "1c1b5393c68a643bc79c277c6d7374d0b30cd985", "b48c17a2ac82601ff38df374f87d76005fb61cbd", "35280c99aa6a48bfc2810b72b763ccac0f632207", "f757cc017308d217f35ed8f0c001a57b97308fb7", "cfcf101b8449af67d34cdc1bcb0432fe9e4de08e", "a14384d1997440bad13b97b3ccfb3b8c0392e79a", "555f6bea49e58a2c2541060a21c2d4f9078c3086", "631d18dec342e2cb87614864ba525ebb9ad6a124", "b6c04f16b6fdacc794ea75c8c8dd210f99fafa65", "25485ba7e315956267b3fdc521b421bbb046325d", "6def3ca353dfc1a904bddd00e6a410d41ac7ab01", "cf220bcf84c3d0ab1b01f8f764396941d15ff20f", "2af8021b838f613aee7670bed19d0ddf1d6bc0c1", "270ed85d46524a59e3274d89a1bbf693521cb6af", "60e09482f12fda20f7d4a70e379c969c5a73f512", "10380001af2a541b5feefc7aab9f719b67330a42", "4580ff07f27eb8321421efac1676a80d9239572a", "d3022c5a223caaf77e9c564e024199e5d6f51bd5", "f54742ac61edd8c3980354620816c762b524dfc7" ) for (i in seq_along(test.element)) { if (i == 7 || i == 13 || i == 19) next expect_true( identical( sha1(test.element[[i]]), correct[i] ) ) } expect_true( sha1(matrix(integer(0))) == "e13485e1b995f3e36d43674dcbfedea08ce237bc" ) digest/inst/tinytest/test_aes.R0000644000176200001440000001470414446063606016342 0ustar liggesusers ## tests for AES, taken from the examples in the manual page suppressMessages(library(digest)) # FIPS-197 examples hextextToRaw <- function(text) { vals <- matrix(as.integer(as.hexmode(strsplit(text, "")[[1]])), ncol=2, byrow=TRUE) vals <- vals %*% c(16, 1) as.raw(vals) } plaintext <- hextextToRaw("00112233445566778899aabbccddeeff") aes128key <- hextextToRaw("000102030405060708090a0b0c0d0e0f") aes128output <- hextextToRaw("69c4e0d86a7b0430d8cdb78070b4c55a") aes <- AES(aes128key) #aes aes128 <- aes$encrypt(plaintext) #aes128 expect_true(identical(aes128, aes128output)) expect_true(identical(plaintext, aes$decrypt(aes128, raw=TRUE))) aes192key <- hextextToRaw("000102030405060708090a0b0c0d0e0f1011121314151617") aes192output <- hextextToRaw("dda97ca4864cdfe06eaf70a0ec0d7191") aes <- AES(aes192key) #aes aes192 <- aes$encrypt(plaintext) #aes192 expect_true(identical(aes192, aes192output)) expect_true(identical(plaintext, aes$decrypt(aes192, raw=TRUE))) aes256key <- hextextToRaw("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") aes256output <- hextextToRaw("8ea2b7ca516745bfeafc49904b496089") aes <- AES(aes256key) aes256 <- aes$encrypt(plaintext) #aes256 expect_true(identical(aes256, aes256output)) expect_true(identical(plaintext, aes$decrypt(aes256, raw=TRUE))) # SP800-38a examples plaintext <- hextextToRaw(paste("6bc1bee22e409f96e93d7e117393172a", "ae2d8a571e03ac9c9eb76fac45af8e51", "30c81c46a35ce411e5fbc1191a0a52ef", "f69f2445df4f9b17ad2b417be66c3710",sep="")) key <- hextextToRaw("2b7e151628aed2a6abf7158809cf4f3c") ecb128output <- hextextToRaw(paste("3ad77bb40d7a3660a89ecaf32466ef97", "f5d3d58503b9699de785895a96fdbaaf", "43b1cd7f598ece23881b00e3ed030688", "7b0c785e27e8ad3f8223207104725dd4",sep="")) aes <- AES(key) ecb128 <- aes$encrypt(plaintext) #ecb128 expect_true(identical(ecb128, ecb128output)) expect_true(identical(plaintext, aes$decrypt(ecb128, raw=TRUE))) cbc128output <- hextextToRaw(paste("7649abac8119b246cee98e9b12e9197d", "5086cb9b507219ee95db113a917678b2", "73bed6b8e3c1743b7116e69e22229516", "3ff1caa1681fac09120eca307586e1a7",sep="")) iv <- hextextToRaw("000102030405060708090a0b0c0d0e0f") aes <- AES(key, mode="CBC", IV=iv) cbc128 <- aes$encrypt(plaintext) #cbc128 expect_true(identical(cbc128, cbc128output)) aes <- AES(key, mode="CBC", IV=iv) expect_true(identical(plaintext, aes$decrypt(cbc128, raw=TRUE))) ctr128output <- hextextToRaw(paste("874d6191b620e3261bef6864990db6ce", "9806f66b7970fdff8617187bb9fffdff", "5ae4df3edbd5d35e5b4f09020db03eab", "1e031dda2fbe03d1792170a0f3009cee",sep="")) iv <- hextextToRaw("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff") aes <- AES(key, mode="CTR", IV=iv) ctr128 <- aes$encrypt(plaintext) #ctr128 expect_true(identical(ctr128, ctr128output)) aes <- AES(key, mode="CTR", IV=iv) expect_true(identical(plaintext, aes$decrypt(ctr128, raw=TRUE))) #cfb key <- hextextToRaw("2b7e151628aed2a6abf7158809cf4f3c") iv <- hextextToRaw("000102030405060708090a0b0c0d0e0f") #cfb not a multiplier of 16 text <- "This is very secret string" key <- hextextToRaw("2b7e151628aed2a6abf7158809cf4f3c") iv <- hextextToRaw("000102030405060708090a0b0c0d0e0f") cfb128output <- hextextToRaw(paste("04960ebfb9044196ac6c4590bbdc8903", "e1259a479e199af94518",sep="")) aes <- AES(key, mode="CFB", IV=iv) cfb128 <- aes$encrypt(text) expect_true(identical(cfb128, cfb128output)) aes <- AES(key, mode="CFB", IV=iv) expect_true(identical(text, aes$decrypt(cfb128, raw=FALSE))) #cfb128 cfb128output <- hextextToRaw(paste("3b3fd92eb72dad20333449f8e83cfb4a", "c8a64537a0b3a93fcde3cdad9f1ce58b", "26751f67a3cbb140b1808cf187a4f4df", "c04b05357c5d1c0eeac4c66f9ff7f2e6",sep="")) aes <- AES(key, mode="CFB", IV=iv) cfb128 <- aes$encrypt(plaintext) expect_true(identical(cfb128, cfb128output)) aes <- AES(key, mode="CFB", IV=iv) expect_true(identical(plaintext, aes$decrypt(cfb128, raw=TRUE))) # test throws exception on IV null or not a multiplier of 16 bytes aes <- AES(key, mode="CFB", IV=NULL) expect_error(aes$encrypt(plaintext)) expect_error(aes$decrypt(plaintext)) aes <- AES(key, mode="CFB", IV=raw(15)) expect_error(aes$encrypt(plaintext)) expect_error(aes$decrypt(plaintext)) # test that providing raw vs character inputs results in the same output text <- "0123456789ABCDEF" raw_text <- charToRaw(text) expect_identical( AES(key, mode="ECB")$encrypt(text), AES(key, mode="ECB")$encrypt(raw_text) ) # test padding for CBC expect_length(AES(raw_text, "CBC", raw_text)$encrypt(text), 16) expect_length(AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt(text), 32) expect_identical( AES(raw_text, "CBC", raw_text)$encrypt(text), head(AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt(text), 16) ) expect_identical( paste(AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt(text), collapse = ""), "9d2cda901b682d3359709a5ab24196242125333bdf540132179ac0de79e1837a" ) cipher <- AES(raw_text, "CBC", raw_text)$encrypt(text) expect_identical(AES(raw_text, "CBC", raw_text)$decrypt(cipher), text) cipher_padded <- AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt(text) expect_identical(AES(raw_text, "CBC", raw_text, padding = TRUE)$decrypt(cipher_padded), text) expect_error(AES(raw_text, "CBC", raw_text)$encrypt("testing")) expect_identical( paste(AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt("testing"), collapse = ""), "4017d340b6eafbb66c3dfceb10808cff" ) expect_identical( AES(raw_text, "CBC", raw_text, padding = TRUE)$decrypt( AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt("testing") ), "testing" ) expect_error(AES(raw_text, "CBC", raw_text)$encrypt("testingalongerstring")) expect_identical( paste(AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt("testingalongerstring"), collapse = ""), "2d870ada654f4bdff182497a88e08013a232bfb5701c12a4d3b29f4f66e74ecd" ) expect_identical( AES(raw_text, "CBC", raw_text, padding = TRUE)$decrypt( AES(raw_text, "CBC", raw_text, padding = TRUE)$encrypt("testingalongerstring") ), "testingalongerstring" ) digest/inst/tinytest/test_raw.R0000644000176200001440000000126413503674673016365 0ustar liggesusers ## tests for raw output suppressMessages(library(digest)) current <- hmac("Jefe", 'what do ya want for nothing?', "md5", raw=TRUE) expected <- as.raw(c(0x75, 0x0c, 0x78, 0x3e, 0x6a, 0xb0, 0xb5, 0x03, 0xea, 0xa8, 0x6e, 0x31, 0x0a, 0x5d, 0xb7, 0x38)) expect_equal(current, expected) current <- digest("The quick brown fox", algo="sha1", raw=TRUE) expected <- as.raw(c(0x5f, 0x79, 0x8c, 0xb4, 0xd8, 0x14, 0x4e, 0xec, 0x35, 0xf4, 0xd0, 0x79, 0x3e, 0xf2, 0x1e, 0x55, 0xce, 0xb6, 0xa7, 0x88)) expect_equal(current, expected) ## feed raw to sha1() to test sha1.raw() as well expect_true(sha1(expected) == "75a2995eeec0fcb5d7fa97c676a37f4e224981a1") digest/inst/tinytest/test_hmac.R0000644000176200001440000000710513503674673016504 0ustar liggesusers ## tests for hmac, taken from the examples in the manual page suppressMessages(library(digest)) ## Standard RFC 2104 test vectors current <- hmac('Jefe', 'what do ya want for nothing?', "md5") target <- '750c783e6ab0b503eaa86e310a5db738' expect_true(identical(target, as.character(current))) #cat(current, "\n") current <- hmac('Jefe', 'what do ya want for nothing?', "md5") target <- '750c783e6ab0b503eaa86e310a5db738' expect_true(identical(target, as.character(current))) #cat(current, "\n") current <- hmac(rep(0x0b, 16), 'Hi There', "md5") target <- '9294727a3638bb1c13f48ef8158bfc9d' expect_true(identical(target, as.character(current))) #cat(current, "\n") current <- hmac(rep(0xaa, 16), rep(0xdd, 50), "md5") target <- '56be34521d144c88dbb8c733f0e8b3f6' expect_true(identical(target, as.character(current))) #cat(current, "\n") ## test for key longer than block size. key should be hashed itself. current <- hmac('ThisKeyIsLongerThan64BytesAndThereforeLongerThanTheHashDigestByFour', 'what do ya want for nothing?', "md5") target <- '109af8ddd9cccbf8927d354da8be892b' expect_true(identical(target, as.character(current))) #cat(current, "\n") ## SHA1 tests inspired to the RFC 2104 and checked against the python ## hmac implementation. current <- hmac('Jefe', 'what do ya want for nothing?', "sha1") target <- 'effcdf6ae5eb2fa2d27416d5f184df9c259a7c79' expect_true(identical(target, as.character(current))) #cat(current, "\n") current <- hmac(rep(0x0b, 16), 'Hi There', "sha1") target <- '675b0b3a1b4ddf4e124872da6c2f632bfed957e9' expect_true(identical(target, as.character(current))) #cat(current, "\n") current <- hmac(rep(0xaa, 16), rep(0xdd, 50), "sha1") target <- 'd730594d167e35d5956fd8003d0db3d3f46dc7bb' expect_true(identical(target, as.character(current))) #cat(current, "\n") ## SHA256 tests from wikipedia current <- hmac('key', 'The quick brown fox jumps over the lazy dog', 'sha256') target <- 'f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8' expect_true(identical(target, as.character(current))) #cat(current, "\n") ## SHA512 tests inspired, one from RFC 2104 and one from user, checked against python hmac # python code: # from hashlib import sha512 # import hmac # print(hmac.new("love","message",sha512).hexdigest()) current <- hmac('Jefe', 'what do ya want for nothing?', "sha512") target <- '164b7a7bfcf819e2e395fbe73b56e0a387bd64222e831fd610270cd7ea2505549758bf75c05a994a6d034f65f8f0e6fdcaeab1a34d4a6b4b636e070a38bce737' expect_true(identical(target, as.character(current))) #cat(current, "\n") current <- hmac('love', 'message', 'sha512') target <- 'f955821b3f6161673eb20985c677e3dc101860cafef3321ee31641acad9fcd85d9c7d3481ed3e4e1f4fa7af41d7e6ea9606d51e9bc7205d0091a4ee87d90fb9c' expect_true(identical(target, as.character(current))) #cat(current, "\n") ## test for key longer than block size. key should be hashed itself. current <- hmac('verylongkeyinfactlongerthanthedigestandthereforehashedbyitselfonlyifwegettothislengthsoonpleasepleasebutiddoesnotlooksoogoodahthereweare', 'message', 'sha512') target <- '54b274484a8b8a371c8ae898f453d37c90d488b0c88c89dd705de06b18263d50c28069f1f778c8a997a61a4d53a06bc8e8719ad11a553c49140bb93a4636882e' expect_true(identical(target, as.character(current))) #cat(current, "\n") ## motivated by coverage report rw <- charToRaw("123456789ABCDEF") expect_true(all.equal(makeRaw(rw), rw)) current <- hmac('Jefe', 'what do ya want for nothing?', "md5", raw=TRUE) target <- as.raw(c(0x75, 0x0c, 0x78, 0x3e, 0x6a, 0xb0, 0xb5, 0x03, 0xea, 0xa8, 0x6e, 0x31, 0x0a, 0x5d, 0xb7, 0x38)) expect_true(identical(target, current)) #digest:::padWithZeros(rw, "crc32") digest/inst/tinytest/test_crc32.R0000644000176200001440000000104513503674673016505 0ustar liggesusers ## tests for old versus new (ie guaranteed eight chars, added in 0.6.16) format for crc32 suppressMessages(library(digest)) args <- c(0L, 51L, 126L, 8480L, 60929L, 180832L) resOld <- c("b7fa0888", "82a699e", "4754b3", "b3da3", "e67c", "872") resNew <- c("b7fa0888", "082a699e", "004754b3", "000b3da3", "0000e67c", "00000872") options("digestOldCRC32Format" = TRUE) expect_identical(sapply(args, digest, algo="crc32"), resOld) options("digestOldCRC32Format" = FALSE) expect_identical(sapply(args, digest, algo="crc32"), resNew) digest/inst/tinytest/test_num2hex.R0000644000176200001440000000320013537453061017143 0ustar liggesusers## tests for digest, taken from the examples in the manual page suppressMessages(library(digest)) # zap small numbers to zero zapsmall <- 1:10 border <- 2 ^ floor(log2(10 ^ -zapsmall)) expect_equal( sapply( seq_along(zapsmall), function(i) { digest:::num2hex(border[i] * -1:1, digits = 6, zapsmall = zapsmall[i]) } ), matrix( "0", ncol = length(zapsmall), nrow = 3 ) ) # handle 0 correct expect_equal(digest:::num2hex(0), "0") # digits are consistent x <- pi x.hex <- sapply(1:16, digest:::num2hex, x = x) x.hex <- x.hex[c(TRUE, diff(nchar(x.hex)) > 0)] exponent <- unique(gsub("^[0-9a-f]* ", "", x.hex)) expect_equal(length(exponent), 1L) mantissa <- gsub(" [0-9]*$", "", x.hex) ignore(expect_true)(all( sapply( head(seq_along(mantissa), -1), function(i){ all( grepl( paste0("^", mantissa[i], ".*"), tail(mantissa, -i) ) ) } ) )) #it keeps NA values x <- c(pi, NA, 0) expect_equal(is.na(digest:::num2hex(x)), is.na(x)) x <- c(pi, NA, pi) expect_equal(is.na(digest:::num2hex(x)), is.na(x)) x <- as.numeric(c(NA, NA, NA)) expect_equal(is.na(digest:::num2hex(x)), is.na(x)) # handles empty vectors expect_equal(digest:::num2hex(numeric(0)), character(0)) # example from FAQ 7.31 # tests if all trailing zero's are removed expect_true(identical(digest:::num2hex(2, digits = 14), digest:::num2hex(sqrt(2) ^ 2, digits = 14))) expect_true(!identical(digest:::num2hex(2, digits = 15), digest:::num2hex(sqrt(2) ^ 2, digits = 15))) digest/inst/tinytest/test_digest2int.R0000644000176200001440000000102713503674673017645 0ustar liggesusers suppressMessages(library(digest)) input <- letters[1:5] hashed_seed_0 <- c(-902917054L, 14385563L, -289776295L, 592496261L, 286663184L) expect_equal(digest2int(input), hashed_seed_0) hashed_seed_1 <- c(14385563L, -289776295L, 592496261L, 286663184L, 1208324078L) expect_equal(digest2int(input, 1L), hashed_seed_1) # should fail if uint32_t on the system is not a 32-bit unsigned integer expect_equal(digest2int("cat sat on the mat"), 562079877L) expect_equal(digest2int("The quick brown fox jumps over the lazy dog"), 1369346549L) digest/inst/tinytest/test_misc.R0000644000176200001440000000421113553177457016525 0ustar liggesusers## a few tests split off the test_digest.R file suppressMessages(library(digest)) ## test 'length' parameter and file input ##fname <- file.path(R.home(),"COPYING") ## not invariant across OSs fname <- system.file("GPL-2", package="digest") x <- readChar(fname, file.info(fname)$size) # read file xskip <- substring(x, first=20+1) for (alg in c("sha1", "md5", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32")) { # partial file h1 <- digest(x , length=18000, algo=alg, serialize=FALSE) h2 <- digest(fname, length=18000, algo=alg, serialize=FALSE, file=TRUE) expect_true(identical(h1,h2)) h3 <- digest(substr(x,1,18000) , algo=alg, serialize=FALSE) expect_true(identical(h1,h3)) #cat(h1, "\n", h2, "\n", h3, "\n") expect_identical( getVDigest(alg)(x, length = 18e3, serialize = FALSE), getVDigest(alg)(fname, length = 18e3, serialize = FALSE, file = TRUE) ) # whole file h4 <- digest(x , algo=alg, serialize=FALSE) h5 <- digest(fname, algo=alg, serialize=FALSE, file=TRUE) expect_true( identical(h4,h5) ) expect_identical( getVDigest(alg)(x, serialize = FALSE), getVDigest(alg)(fname, serialize = FALSE, file = TRUE) ) ## Assert that 'skip' works h6 <- digest(xskip, algo=alg, serialize=FALSE) h7 <- digest(fname, algo=alg, serialize=FALSE, skip=20, file=TRUE) expect_true( identical(h6, h7) ) expect_identical( getVDigest(alg)(xskip, serialize = FALSE), getVDigest(alg)(fname, serialize = FALSE, skip = 20, file = TRUE) ) } ## compare md5 algorithm to other tools library(tools) ##fname <- file.path(R.home(),"COPYING") ## not invariant across OSs fname <- system.file("GPL-2", package="digest") h1 <- as.character(md5sum(fname)) h2 <- digest(fname, algo="md5", file=TRUE) expect_true( identical(h1,h2) ) ## Make sure we don't core dump with unreadable files. fname <- tempfile() #cat("Hello World, you won't have access to read me", file=fname) on.exit(unlink(fname)) Sys.chmod(fname, mode="0000") try(res <- digest(fname, file=TRUE), silent=TRUE) digest/inst/tinytest/test_encoding.R0000644000176200001440000000135114122616624017345 0ustar liggesusers isWindows <- Sys.info()[["sysname"]] == "Windows" if (!isWindows) exit_file("Skipping test on non-Windows platform") d <- tempfile() dir.create(d) # A file path not representable in latin-1 f <- file.path(d, "tricky-ő") # We need to use a binary mode connection so the newlines # are consistent across platforms con <- file(f, open = "wb") writeLines("foobar", con = con) close(con) expect_identical(digest::digest(file = f), "14758f1afd44c09b7992073ccf00b43d") expect_identical(digest::digest(f, file = TRUE), "14758f1afd44c09b7992073ccf00b43d") vd <- getVDigest() expect_identical(vd(file = f), "14758f1afd44c09b7992073ccf00b43d") expect_identical(vd(f, file = TRUE), "14758f1afd44c09b7992073ccf00b43d") unlink(d, recursive = TRUE) digest/inst/tinytest/test_digest.R0000644000176200001440000003032114636020207017031 0ustar liggesusers ## tests for digest, taken from the examples in the manual page suppressMessages(library(digest)) ## Standard RFC 1321 test vectors md5Input <- c("", "a", "abc", "message digest", "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", paste("12345678901234567890123456789012345678901234567890123456789012", "345678901234567890", sep="")) md5Output <- c("d41d8cd98f00b204e9800998ecf8427e", "0cc175b9c0f1b6a831c399e269772661", "900150983cd24fb0d6963f7d28e17f72", "f96b697d7cb7938d525a2f31aaf161d0", "c3fcd3d76192e4007dfb496cca67e13b", "d174ab98d277d9f5a5611c2c9f419d9f", "57edf4a22be3c955ac49da2e2107b67a") for (i in seq(along.with=md5Input)) { md5 <- digest(md5Input[i], serialize=FALSE) expect_true(identical(md5, md5Output[i])) #cat(md5, "\n") } md5 <- getVDigest() expect_identical(md5(md5Input, serialize = FALSE), md5Output) expect_identical(digest(NULL), md5(NULL)) expect_identical(digest(character(0)), md5(character(0))) expect_identical(digest(list("abc")), md5(list(list("abc")))) expect_identical(digest(list(NULL)), md5(list(list(NULL)))) expect_identical(digest(character(0), serialize = FALSE), md5(character(0), serialize = FALSE)) ## md5 raw output test for (i in seq(along.with=md5Input)) { md5 <- digest(md5Input[i], serialize=FALSE, raw=TRUE) md5 <- gsub(" ","",capture.output(cat(md5))) expect_true(identical(md5, md5Output[i])) #cat(md5, "\n") } sha1Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", NULL) sha1Output <- c("a9993e364706816aba3e25717850c26c9cd0d89d", "84983e441c3bd26ebaae4aa1f95129e5e54670f1", "34aa973cd4c4daa4f61eeb2bdbad27316534016f") for (i in seq(along.with=sha1Input)) { sha1 <- digest(sha1Input[i], algo="sha1", serialize=FALSE) expect_true(identical(sha1, sha1Output[i])) #cat(sha1, "\n") } sha1 <- getVDigest(algo = 'sha1') expect_identical(sha1(sha1Input, serialize = FALSE), sha1Output[1:2]) ## sha1 raw output test for (i in seq(along.with=sha1Input)) { sha1 <- digest(sha1Input[i], algo="sha1", serialize=FALSE, raw=TRUE) #print(sha1) sha1 <- gsub(" ","",capture.output(cat(sha1))) #print(sha1) #print(sha1Output[i]) expect_true(identical(sha1, sha1Output[i])) #cat(sha1, "\n") } ## sha512 test sha512Input <-c( "", "The quick brown fox jumps over the lazy dog." ) sha512Output <- c( "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", "91ea1245f20d46ae9a037a989f54f1f790f0a47607eeb8a14d12890cea77a1bbc6c7ed9cf205e67b7f2b8fd4c7dfd3a7a8617e45f3c463d481c7e586c39ac1ed") for (i in seq(along.with=sha512Input)) { sha512 <- digest(sha512Input[i], algo="sha512", serialize=FALSE) expect_true(identical(sha512, sha512Output[i])) #cat(sha512, "\n") } sha512 <- getVDigest(algo = 'sha512') expect_identical(sha512(sha512Input, serialize = FALSE), sha512Output[1:2]) ## sha512 raw output test for (i in seq(along.with=sha512Input)) { sha512 <- digest(sha512Input[i], algo="sha512", serialize=FALSE, raw=TRUE) #print(sha512) sha512 <- gsub(" ","",capture.output(cat(sha512))) #print(sha512) #print(sha512Output[i]) expect_true(identical(sha512, sha512Output[i])) #cat(sha512, "\n") } crc32Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", NULL) crc32Output <- c("352441c2", "171a3f5f", "2ef80172") for (i in seq(along.with=crc32Input)) { crc32 <- digest(crc32Input[i], algo="crc32", serialize=FALSE) expect_true(identical(crc32, crc32Output[i])) #cat(crc32, "\n") } crc32 <- getVDigest(algo = 'crc32') expect_identical(crc32(crc32Input, serialize = FALSE), crc32Output[1:2]) ## one of the FIPS- sha1 <- digest("abc", algo="sha1", serialize=FALSE) expect_true(identical(sha1, "a9993e364706816aba3e25717850c26c9cd0d89d")) ## This one seems to give slightly different output depending on the R version used ## ## # example of a digest of a standard R list structure ## cat(digest(list(LETTERS, data.frame(a=letters[1:5], ## b=matrix(1:10, ## ncol=2)))), "\n") ## these outputs were calculated using xxh32sum ## [ Correction: These reproduce via the Python xxhash package and its hexdigest() output ## but not the xxh64sum command-line tool as the original comment here implies. ] xxhash32Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") xxhash32Output <- c("32d153ff", "89ea60c3", "02cc5d05") for (i in seq(along.with=xxhash32Input)) { xxhash32 <- digest(xxhash32Input[i], algo="xxhash32", serialize=FALSE) #cat(xxhash32, "\n") expect_true(identical(xxhash32, xxhash32Output[i])) } xxhash32 <- getVDigest(algo = 'xxhash32') expect_identical(xxhash32(xxhash32Input, serialize = FALSE), xxhash32Output) ## these outputs were calculated using xxh64sum ## [ Correction: These reproduce via the Python xxhash package and its hexdigest() output ## but not the xxh64sum command-line tool as the original comment here implies. ] xxhash64Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") xxhash64Output <- c("44bc2cf5ad770999", "f06103773e8585df", "ef46db3751d8e999") for (i in seq(along.with=xxhash64Input)) { xxhash64 <- digest(xxhash64Input[i], algo="xxhash64", serialize=FALSE) #cat(xxhash64, "\n") expect_true(identical(xxhash64, xxhash64Output[i])) } xxhash64 <- getVDigest(algo = 'xxhash64') expect_identical(xxhash64(xxhash64Input, serialize = FALSE), xxhash64Output) ## these outputs were calculated using mmh3 python package ## the first two are also shown at this StackOverflow question on test vectors ## https://stackoverflow.com/questions/14747343/murmurhash3-test-vectors murmur32Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") murmur32Output <- c("b3dd93fa", "ee925b90", "00000000") for (i in seq(along.with=murmur32Input)) { murmur32 <- digest(murmur32Input[i], algo="murmur32", serialize=FALSE) #cat(murmur32, "\n") expect_true(identical(murmur32, murmur32Output[i])) } murmur32 <- getVDigest(algo = 'murmur32') expect_identical(murmur32(murmur32Input, serialize = FALSE), murmur32Output) ## tests for digest spooky ## Per PR 205, see comment in https://github.com/facebook/folly/blob/4c603f8c2add8d0228de0e073c5ae3ce9b02b6f3/folly/hash/SpookyHashV2.h#L35-L36 ## Values ought to be sensible on big endian too but different from little endian reference ## so we do not test on big endian if (isTRUE(.Call(digest:::is_little_endian))) { ## test vectors (originally for md5) spookyInput <- c("", "a", "abc", "message digest", "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", paste("12345678901234567890123456789012345678901234567890123456789012", "345678901234567890", sep="")) # from spooky import hash128 # from binascii import hexlify # # spookyInput = [ # "", # "a", # "abc", # "message digest", # "abcdefghijklmnopqrstuvwxyz", # "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", # "12345678901234567890123456789012345678901234567890123456789012345678901234567890" # ] # # for s in spookyInput: # hexlify(hash128(s).to_bytes(16, 'little')).decode() # # '1909f56bfc062723c751e8b465ee728b' # 'bdc9bba09181101a922a4161f0584275' # '67c93775f715ab8ab01178caf86713c6' # '9630c2a55c0987a0db44434f9d67a192' # '5172de938ce149a98f4d06d3c3168ffe' # 'b5b3b2d0f08b58aa07f551895f929f81' # '3621ec01112dafa1610a4bd23041966b' spookyOutputPython <- c('1909f56bfc062723c751e8b465ee728b', 'bdc9bba09181101a922a4161f0584275', '67c93775f715ab8ab01178caf86713c6', '9630c2a55c0987a0db44434f9d67a192', '5172de938ce149a98f4d06d3c3168ffe', 'b5b3b2d0f08b58aa07f551895f929f81', '3621ec01112dafa1610a4bd23041966b') ## spooky raw output test for (i in seq(along.with=spookyInput)) { # skip = 30 skips the entire serialization header for a length 1 character vector # this is equivalent to raw = TRUE and matches the python spooky implementation for those vectors spooky <- digest(spookyInput[i], algo = "spookyhash", skip = 30) expect_true(identical(spooky, spookyOutputPython[i])) #cat(spooky, "\n") } expect_identical( getVDigest(algo = 'spookyhash')(spookyInput, skip = 30), spookyOutputPython ) ## some extras to get coverage up - these aren't tested against reference output, ## just output from R 3.6.0 spookyInput <- c("a", "aaaaaaaaa", "aaaaaaaaaaaaa") spookyOutput <- c("b7a3573ba6139dfdc52db30acba87f46", "fd876ecaa5d1e442600333118f223e02", "91848873bf91d06ad321bbd47400a556") for (i in seq(along.with=spookyInput)) { spooky <- digest(spookyInput[i], algo = "spookyhash") expect_true(identical(spooky, spookyOutput[i])) ##cat(spooky, "\n") } expect_identical( getVDigest(algo = 'spookyhash')(spookyInput), spookyOutput ) ## test a bigger object spooky <- digest(iris, algo = "spookyhash") expect_true(identical(spooky, "af58add8b4f7044582b331083bc239ff")) expect_identical(getVDigest('spookyhash')(list(iris)), "af58add8b4f7044582b331083bc239ff") ##cat(spooky, "\n") # test error message #error.message <- try(digest(spookyInput[i], algo = "spookyhash", serialize = FALSE)) #expect_true( # grepl("spookyhash algorithm is not available without serialization.", error.message) #) } ## Ensure that all values of algo are actually allowed (in case a new one is ## added in the future). The call to match.arg() passes choices explicitly ## because it is significantly faster to do it than to have it automatically ## infer the possible choices from the function's formals. # Grab the possible values of algo, then call digest() for each one. algos <- eval(formals(digest)$algo) for (algo in algos) { digest(123, algo = algo) } # Same for getVDigest algos <- eval(formals(getVDigest)$algo) for (algo in algos) { getVDigest(algo = algo) } ## xxhash h3_64 variant ## reference values computed via xxhash and its xxh3_64 object and hexdiges printer: ## ie print(xxhash.xxh3_64("abc").hexdigest()) xxh3_64Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") xxh3_64Output <- c("78af5f94892f3950", "5bbcbbabcdcc3d3f", "2d06800538d394c2") for (i in seq(along.with=xxh3_64Input)) { xxh3_64 <- digest(xxh3_64Input[i], algo="xxh3_64", serialize=FALSE) #cat(xxh3_64, "\n") expect_true(identical(xxh3_64, xxh3_64Output[i])) } xxh3_64 <- getVDigest(algo = 'xxh3_64') expect_identical(xxh3_64(xxh3_64Input, serialize = FALSE), xxh3_64Output) ## xxhash h3_128 variant ## reference values computed via xxhash and its xxh3_128 object and hexdiges printer: ## ie print(xxhash.xxh3_128("abc").hexdigest()) xxh3_128Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") xxh3_128Output <- c("06b05ab6733a618578af5f94892f3950", "3d62d22a5169b016c0d894fd4828a1a7", "99aa06d3014798d86001c324468d497f") for (i in seq(along.with=xxh3_128Input)) { xxh3_128 <- digest(xxh3_128Input[i], algo="xxh3_128", serialize=FALSE) #cat(xxh3_128, "\n") expect_true(identical(xxh3_128, xxh3_128Output[i])) } xxh3_128 <- getVDigest(algo = 'xxh3_128') expect_identical(xxh3_128(xxh3_128Input, serialize = FALSE), xxh3_128Output) digest/inst/GPL-20000644000176200001440000004325412247504516013230 0ustar liggesusers GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. digest/README.md0000644000176200001440000001011314645352543013017 0ustar liggesusers## digest: Compact hash representations of arbitrary R objects [![ci](https://github.com/eddelbuettel/digest/workflows/ci/badge.svg)](https://github.com/eddelbuettel/digest/actions?query=workflow%3Aci) [![License](http://img.shields.io/badge/license-GPL%20%28%3E=%202%29-brightgreen.svg?style=flat)](http://www.gnu.org/licenses/gpl-2.0.html) [![CRAN](https://www.r-pkg.org/badges/version/digest)](https://cran.r-project.org/package=digest) [![Dependencies](https://tinyverse.netlify.app/badge/digest)](https://cran.r-project.org/package=digest) [![Debian package](https://img.shields.io/debian/v/r-cran-digest/sid?color=green)](https://packages.debian.org/sid/r-cran-digest) [![r-universe](https://eddelbuettel.r-universe.dev/badges/digest)](https://eddelbuettel.r-universe.dev/digest) [![Downloads (monthly)](https://cranlogs.r-pkg.org/badges/digest?color=brightgreen)](https://www.r-pkg.org:443/pkg/digest) [![Downloads (total)](https://cranlogs.r-pkg.org/badges/grand-total/digest?color=brightgreen)](https://www.r-pkg.org:443/pkg/digest) [![CRAN use](https://jangorecki.gitlab.io/rdeps/digest/CRAN_usage.svg?sanitize=true)](https://cran.r-project.org/package=digest) [![CRAN indirect](https://jangorecki.gitlab.io/rdeps/digest/indirect_usage.svg?sanitize=true)](https://cran.r-project.org/package=digest) [![BioConductor use](https://jangorecki.gitlab.io/rdeps/digest/BioC_usage.svg?sanitize=true)](https://cran.r-project.org/package=digest) [![Code Coverage](https://img.shields.io/codecov/c/github/eddelbuettel/digest/master.svg)](https://app.codecov.io/gh/eddelbuettel/digest) [![Last Commit](https://img.shields.io/github/last-commit/eddelbuettel/digest)](https://github.com/eddelbuettel/digest) [![Documentation](https://img.shields.io/badge/documentation-is_here-blue)](https://eddelbuettel.github.io/digest/) Compact hash representations of arbitrary R objects ### Overview The digest package provides a principal function `digest()` for the creation of hash digests of arbitrary R objects (using the md5, sha-1, sha-256, crc32, xxhash, murmurhash, spookyhash, blake3, crc32c, xxh3\_64, and xxh3\_128 algorithms) permitting easy comparison of R language objects. Extensive documentation is available at the [package documentation site](https://eddelbuettel.github.io/digest/). #### Examples As R can serialize any object, we can run `digest()` on any object: ```r R> library(digest) R> digest(trees) [1] "12412cbfa6629c5c80029209b2717f08" R> digest(lm(log(Height) ~ log(Girth), data=trees)) [1] "e25b62de327d079b3ccb98f3e96987b1" R> digest(summary(lm(log(Height) ~ log(Girth), data=trees))) [1] "86c8c979ee41a09006949e2ad95feb41" R> ``` By using the hash sum, which is very likely to be unique, to identify an underlying object or calculation, one can easily implement caching strategies. This is a common use of the digest package. #### Other Functions A small number of additional functions is available: - `sha1()` for numerally stable hashsums, - `hmac()` for hashed message authentication codes based on a key, - `AES()` for Advanced Encryption Standard block ciphers, - `getVDigest()` as a function generator for vectorised versions. ### Note Please note that this package is not meant to be deployed for cryptographic purposes. More comprehensive and widely tested libraries such as OpenSSL should be used instead. ### Installation The package is on [CRAN](https://cran.r-project.org) and can be installed via a standard ```r install.packages("digest") ``` ### Continued Testing As we rely on the [tinytest](https://cran.r-project.org/package=tinytest) package, the already-installed package can also be verified via ```r tinytest::test_package("digest") ``` at any later point. ### Author Dirk Eddelbuettel, with contributions by Antoine Lucas, Jarek Tuszynski, Henrik Bengtsson, Simon Urbanek, Mario Frasca, Bryan Lewis, Murray Stokely, Hannes Muehleisen, Duncan Murdoch, Jim Hester, Wush Wu, Qiang Kou, Thierry Onkelinx, Michel Lang, Viliam Simko, Kurt Hornik, Radford Neal, Kendon Bell, Matthew de Queljoe, Ion Suruceanu, Bill Denney, Dirk Schumacher, Winston Chang, Dean Attali, and Michael Chirico. ### License GPL (>= 2) digest/build/0000755000176200001440000000000014660633605012641 5ustar liggesusersdigest/build/vignette.rds0000644000176200001440000000032214660633605015175 0ustar liggesusersb```b`aad`b2 1# 'f/H4MA kh*+dhhk(A`KBd Ik^bnj1\R RR@g;<E T [fN*ސ89 d Bw(,/׃ @?{49'ݣ\)%ziE@ w [Qܮdigest/man/0000755000176200001440000000000014636020207012304 5ustar liggesusersdigest/man/vdigest.Rd0000644000176200001440000000673714573367376014302 0ustar liggesusers\name{getVDigest} \alias{getVDigest} \title{Set a vectorised function for creating hash function digests} \description{ The \code{getVDigest} function extends \code{digest} by allowing one to set a function that returns hash summaries as a character vector of the same length as the input. It also provides a performance advantage when repeated calls are necessary (e.g. applying a hash function repeatedly to an object). The returned function contains the same arguments as \code{digest} with the exception of the \code{raw} argument (see \code{\link{digest}} for more details). } \usage{ getVDigest(algo=c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "spookyhash", "blake3", "crc32c", "xxh3_64", "xxh3_128"), errormode=c("stop","warn","silent")) } \arguments{ \item{algo}{The algorithms to be used; currently available choices are \code{md5}, which is also the default, \code{sha1}, \code{crc32}, \code{sha256}, \code{sha512}, \code{xxhash32}, \code{xxhash64}, \code{murmur32}, \code{spookyhash}, \code{blake3}, \code{crc32c}, \code{xxh3_64}, and \code{xxh3_128}.} \item{errormode}{A character value denoting a choice for the behaviour in the case of error: \sQuote{stop} aborts (and is the default value), \sQuote{warn} emits a warning and returns \code{NULL} and \sQuote{silent} suppresses the error and returns an empty string.} } \value{ The \code{getVDigest} function returns a function for a given algorithm and error-mode. } \details{ Note that since one hash summary will be returned for each element passed as input, care must be taken when determining whether or not to include the data structure as part of the object. For instance, to return the equivalent output of \code{digest(list("a"))} it would be necessary to wrap the list object itself \code{getVDigest()(list(list("a")))} } \seealso{\code{\link{digest}}, \code{\link{serialize}}, \code{\link{md5sum}}} \examples{ stretch_key <- function(d, n) { md5 <- getVDigest() for (i in seq_len(n)) d <- md5(d, serialize = FALSE) d } stretch_key('abc123', 65e3) sha1 <- getVDigest(algo = 'sha1') sha1(letters) md5Input <- c("", "a", "abc", "message digest", "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", paste("12345678901234567890123456789012345678901234567890123456789012", "345678901234567890", sep="")) md5Output <- c("d41d8cd98f00b204e9800998ecf8427e", "0cc175b9c0f1b6a831c399e269772661", "900150983cd24fb0d6963f7d28e17f72", "f96b697d7cb7938d525a2f31aaf161d0", "c3fcd3d76192e4007dfb496cca67e13b", "d174ab98d277d9f5a5611c2c9f419d9f", "57edf4a22be3c955ac49da2e2107b67a") md5 <- getVDigest() stopifnot(identical(md5(md5Input, serialize = FALSE), md5Output)) stopifnot(identical(digest(list("abc")), md5(list(list("abc"))))) sha512Input <-c( "", "The quick brown fox jumps over the lazy dog." ) sha512Output <- c( paste0("cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce", "47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"), paste0("91ea1245f20d46ae9a037a989f54f1f790f0a47607eeb8a14d12890cea77a1bb", "c6c7ed9cf205e67b7f2b8fd4c7dfd3a7a8617e45f3c463d481c7e586c39ac1ed") ) sha512 <- getVDigest(algo = 'sha512') stopifnot(identical(sha512(sha512Input, serialize = FALSE), sha512Output)) } \keyword{misc} digest/man/digest.Rd0000644000176200001440000003446714636020207014070 0ustar liggesusers\name{digest} \alias{digest} \title{Create hash function digests for arbitrary R objects or files} \description{ The \code{digest} function applies one of several cryptographic or non-cryptographics hash function to arbitrary R objects or files. By default, the objects are internally serialized, and the selected hash functions algorithms can be used to compute a compact digest of the serialized object. In order to compare this implementation with others, serialization of the input argument can also be turned off in which the input argument must be a character string for which its digest is returned. } \usage{ digest(object, algo=c("md5", "sha1", "crc32", "sha256", "sha512", "xxhash32", "xxhash64", "murmur32", "spookyhash", "blake3", "crc32c", "xxh3_64", "xxh3_128"), serialize=TRUE, file=FALSE, length=Inf, skip="auto", ascii=FALSE, raw=FALSE, seed=0, errormode=c("stop","warn","silent"), serializeVersion=.getSerializeVersion()) } \arguments{ \item{object}{An arbitrary R object which will then be passed to the \code{\link{serialize}} function, unless the \code{serialize} argument is set to \code{FALSE}.} \item{algo}{The algorithms to be used; currently available choices are \code{md5}, which is also the default, \code{sha1}, \code{crc32}, \code{sha256}, \code{sha512}, \code{xxhash32}, \code{xxhash64}, \code{murmur32}, \code{spookyhash}, \code{blake3}, \code{crc32c}, \code{xxh3_64}, and \code{xxh3_128}.} \item{serialize}{A logical variable indicating whether the object should be serialized using \code{serialize} (in ASCII form). Setting this to \code{FALSE} allows to compare the digest output of given character strings to known control output. It also allows the use of raw vectors such as the output of non-ASCII serialization. } \item{file}{A logical variable indicating whether the object is a file name or a file name if \code{object} is not specified.} \item{length}{Number of characters to process. By default, when \code{length} is set to \code{Inf}, the whole string or file is processed.} \item{skip}{Number of input bytes to skip before calculating the digest. Negative values are invalid and currently treated as zero. Special value \code{"auto"} will cause serialization header to be skipped if \code{serialize} is set to \code{TRUE} (the serialization header contains the R version number thus skipping it allows the comparison of hashes across platforms and some R versions). } \item{ascii}{This flag is passed to the \code{serialize} function if \code{serialize} is set to \code{TRUE}, determining whether the hash is computed on the ASCII or binary representation.} \item{raw}{A logical variable with a default value of FALSE, implying \code{digest} returns digest output as ASCII hex values. Set to TRUE to return \code{digest} output in raw (binary) form. Note that this option is supported by most but not all of the implemented hashing algorithms} \item{seed}{an integer to seed the random number generator. This is only used in the \code{xxhash32}, \code{xxhash64} and \code{murmur32} functions and can be used to generate additional hashes for the same input if desired.} \item{errormode}{A character value denoting a choice for the behaviour in the case of error: \sQuote{stop} aborts (and is the default value), \sQuote{warn} emits a warning and returns \code{NULL} and \sQuote{silent} suppresses the error and returns an empty string.} \item{serializeVersion}{An integer value specifying the internal version of the serialization format, with 2 being the default; see \code{\link{serialize}} for details. The \code{serializeVersion} field of \code{\link{option}} can also be used to set a different value.} } \value{ The \code{digest} function returns a character string of a fixed length containing the requested digest of the supplied R object. This string is of length 32 for MD5; of length 40 for SHA-1; of length 8 for CRC32 a string; of length 8 for for xxhash32; of length 16 for xxhash64; and of length 8 for murmur32. } \details{ Cryptographic hash functions are well researched and documented. The MD5 algorithm by Ron Rivest is specified in RFC 1321. The SHA-1 algorithm is specified in FIPS-180-1, SHA-2 is described in FIPS-180-2. For md5, sha-1 and sha-256, this R implementation relies on standalone implementations in C by Christophe Devine. For crc32, code from the zlib library by Jean-loup Gailly and Mark Adler is used. For sha-512, a standalone implementation from Aaron Gifford is used. For xxhash32, xxhash64, xxh3_64 and xxh3_128 the reference implementation by Yann Collet is used. For murmur32, the progressive implementation by Shane Day is used. For spookyhash, the original source code by Bob Jenkins is used. The R implementation that integrates R's serialization directly with the algorithm allowing for memory-efficient incremental calculation of the hash is by Gabe Becker. For blake3, the C implementation by Samuel Neves and Jack O'Connor is used. For crc32c, the portable (i.e. non-hardware accelerated) version from Google is used. Please note that this package is not meant to be used for cryptographic purposes for which more comprehensive (and widely tested) libraries such as OpenSSL should be used. Also, it is known that crc32 is not collision-proof. For sha-1, recent results indicate certain cryptographic weaknesses as well. For more details, see for example \url{https://www.schneier.com/blog/archives/2005/02/cryptanalysis_o.html}. } \section{Change Management}{ Version 0.6.16 of digest corrects an error in which \code{crc32} was not guaranteeing an eight-character return. We now pad with zero to always return eight characters. Should the previous behaviour be required, set \code{option("digestOldCRC32Format"=TRUE)} and the output will be consistent with prior version (but not be consistentnly eight characters). } \references{ MD5: \url{https://www.ietf.org/rfc/rfc1321.txt}. SHA-1: \url{https://en.wikipedia.org/wiki/SHA-1}. SHA-256: \url{https://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf}. CRC32: The original reference webpage at \code{rocksoft.com} has vanished from the web; see \url{https://en.wikipedia.org/wiki/Cyclic_redundancy_check} for general information on CRC algorithms. \url{https://aarongifford.com/computers/sha.html} for the integrated C implementation of sha-512. The page for the code underlying the C functions used here for sha-1 and md5, and further references, is no longer accessible. Please see \url{https://en.wikipedia.org/wiki/SHA-1} and \url{https://en.wikipedia.org/wiki/MD5}. \url{https://zlib.net} for documentation on the zlib library which supplied the code for crc32. \url{https://en.wikipedia.org/wiki/SHA_hash_functions} for documentation on the sha functions. \url{https://github.com/Cyan4973/xxHash} for documentation on the xxHash functions. \url{https://github.com/aappleby/smhasher} for documentation on MurmurHash. \url{https://burtleburtle.net/bob/hash/spooky.html} for the original source code of SpookyHash. \url{https://github.com/BLAKE3-team/BLAKE3/} for the original source code of blake3. \url{https://github.com/google/crc32c} for the (non-hardware-accelerated) crc32c code. } \author{Dirk Eddelbuettel \email{edd@debian.org} for the \R interface; Antoine Lucas for the integration of crc32; Jarek Tuszynski for the file-based operations; Henrik Bengtsson and Simon Urbanek for improved serialization patches; Christophe Devine for the hash function implementations for sha-1, sha-256 and md5; Jean-Loup Gailly and Mark Adler for crc32; Hannes Muehleisen for the integration of sha-512; Jim Hester for the integration of xxhash32, xxhash64 and murmur32; Kendon Bell for the integration of spookyhash using Gabe Becker's R package fastdigest.} \seealso{\code{\link{serialize}}, \code{\link{md5sum}}} \examples{ ## Standard RFC 1321 test vectors md5Input <- c("", "a", "abc", "message digest", "abcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", paste("12345678901234567890123456789012345678901234567890123456789012", "345678901234567890", sep="")) md5Output <- c("d41d8cd98f00b204e9800998ecf8427e", "0cc175b9c0f1b6a831c399e269772661", "900150983cd24fb0d6963f7d28e17f72", "f96b697d7cb7938d525a2f31aaf161d0", "c3fcd3d76192e4007dfb496cca67e13b", "d174ab98d277d9f5a5611c2c9f419d9f", "57edf4a22be3c955ac49da2e2107b67a") for (i in seq(along=md5Input)) { md5 <- digest(md5Input[i], serialize=FALSE) stopifnot(identical(md5, md5Output[i])) } sha1Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") sha1Output <- c("a9993e364706816aba3e25717850c26c9cd0d89d", "84983e441c3bd26ebaae4aa1f95129e5e54670f1") for (i in seq(along=sha1Input)) { sha1 <- digest(sha1Input[i], algo="sha1", serialize=FALSE) stopifnot(identical(sha1, sha1Output[i])) } crc32Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") crc32Output <- c("352441c2", "171a3f5f") for (i in seq(along=crc32Input)) { crc32 <- digest(crc32Input[i], algo="crc32", serialize=FALSE) stopifnot(identical(crc32, crc32Output[i])) } sha256Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") sha256Output <- c("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1") for (i in seq(along=sha256Input)) { sha256 <- digest(sha256Input[i], algo="sha256", serialize=FALSE) stopifnot(identical(sha256, sha256Output[i])) } # SHA 512 example sha512Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") sha512Output <- c(paste("ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a", "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", sep=""), paste("204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c335", "96fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445", sep="")) for (i in seq(along=sha512Input)) { sha512 <- digest(sha512Input[i], algo="sha512", serialize=FALSE) stopifnot(identical(sha512, sha512Output[i])) } ## xxhash32 example xxhash32Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") xxhash32Output <- c("32d153ff", "89ea60c3", "02cc5d05") for (i in seq(along=xxhash32Input)) { xxhash32 <- digest(xxhash32Input[i], algo="xxhash32", serialize=FALSE) cat(xxhash32, "\n") stopifnot(identical(xxhash32, xxhash32Output[i])) } ## xxhash64 example xxhash64Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") xxhash64Output <- c("44bc2cf5ad770999", "f06103773e8585df", "ef46db3751d8e999") for (i in seq(along=xxhash64Input)) { xxhash64 <- digest(xxhash64Input[i], algo="xxhash64", serialize=FALSE) cat(xxhash64, "\n") stopifnot(identical(xxhash64, xxhash64Output[i])) } ## these outputs were calculated using mmh3 python package murmur32Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") murmur32Output <- c("b3dd93fa", "ee925b90", "00000000") for (i in seq(along=murmur32Input)) { murmur32 <- digest(murmur32Input[i], algo="murmur32", serialize=FALSE) cat(murmur32, "\n") stopifnot(identical(murmur32, murmur32Output[i])) } ## these outputs were calculated using spooky python package spookyInput <- c("a", "abc", "message digest") spookyOutput <- c("bdc9bba09181101a922a4161f0584275", "67c93775f715ab8ab01178caf86713c6", "9630c2a55c0987a0db44434f9d67a192") for (i in seq(along=spookyInput)) { # skip = 30 skips the serialization header and just hashes the strings spooky <- digest(spookyInput[i], algo="spookyhash", skip = 30) cat(spooky, "\n") ## we can only compare to reference output on little-endian systems if (isTRUE(.Call(digest:::is_little_endian))) stopifnot(identical(spooky, spookyOutput[i])) } ## blake3 example blake3Input <- c("abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "") blake3Output <- c("6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85", "c19012cc2aaf0dc3d8e5c45a1b79114d2df42abb2a410bf54be09e891af06ff8", "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262") for (i in seq(along=blake3Input)) { blake3 <- digest(blake3Input[i], algo="blake3", serialize=FALSE) cat(blake3, "\n") stopifnot(identical(blake3, blake3Output[i])) } ## crc32c crc32cInput <- c("123456789", "The quick brown fox jumps over the lazy dog") crc32cOutput <- c("e3069283", "22620404") for (i in seq_along(crc32cInput)) { crc32c <- digest(crc32cInput[i], algo="crc32c", serialize=FALSE) cat(crc32c, "\n") stopifnot(identical(crc32c, crc32cOutput[i])) } # example of a digest of a standard R list structure digest(list(LETTERS, data.frame(a=letters[1:5], b=matrix(1:10,ncol=2)))) # test 'length' parameter and file input fname <- file.path(R.home(),"COPYING") x <- readChar(fname, file.info(fname)$size) # read file for (alg in c("sha1", "md5", "crc32")) { # partial file h1 <- digest(x , length=18000, algo=alg, serialize=FALSE) h2 <- digest(fname, length=18000, algo=alg, serialize=FALSE, file=TRUE) h3 <- digest( substr(x,1,18000) , algo=alg, serialize=FALSE) stopifnot( identical(h1,h2), identical(h1,h3) ) # whole file h1 <- digest(x , algo=alg, serialize=FALSE) h2 <- digest(fname, algo=alg, serialize=FALSE, file=TRUE) stopifnot( identical(h1,h2) ) } # compare md5 algorithm to other tools library(tools) fname <- file.path(R.home(),"COPYING") h1 <- as.character(md5sum(fname)) h2 <- digest(fname, algo="md5", file=TRUE) stopifnot( identical(h1,h2) ) ## digest is _designed_ to return one has summary per object to for a desired ## For vectorised output see digest::getVDigest() which provides ## better performance than base::Vectorize() md5 <- getVDigest() v <- md5(1:5) # digest integers 1 to 5 stopifnot(identical(v[1], digest(1L)), # check first and third result identical(v[3], digest(3L))) } \keyword{misc} digest/man/digest2int.Rd0000644000176200001440000000211213577222753014660 0ustar liggesusers\name{digest2int} \alias{digest2int} \title{hash arbitrary string to integer} \description{ The \code{digest2int} function calculates integer hash of an arbitrary string. This is useful for randomized experiments, feature hashing, etc. } \usage{ digest2int(x, seed = 0L) } \arguments{ \item{x}{An arbitrary character vector.} \item{seed}{an integer for algorithm initial state. Function will produce different hashes for same input and different seed values.} } \value{ The \code{digest2int} function returns integer vector of the same length as input vector \code{x}. } \references{ Jenkins's \code{one_at_a_time} hash: \url{https://en.wikipedia.org/wiki/Jenkins_hash_function#one_at_a_time}. } \author{Dmitriy Selivanov \email{selivanov.dmitriy@gmail.com} for the \R interface; Bob Jenkins for original implementation \url{http://www.burtleburtle.net/bob/hash/doobs.html} } \seealso{\code{\link{digest}}} \examples{ current <- digest2int("The quick brown fox jumps over the lazy dog", 0L) target <- 1369346549L stopifnot(identical(target, current)) } \keyword{misc} digest/man/AES.Rd0000644000176200001440000001606114446063606013220 0ustar liggesusers\name{AES} \alias{AES} \title{ Create AES block cipher object } \description{ This creates an object that can perform the Advanced Encryption Standard (AES) block cipher. } \usage{ AES(key, mode=c("ECB", "CBC", "CFB", "CTR"), IV=NULL, padding=FALSE) } \arguments{ \item{key}{ The key as a 16, 24 or 32 byte raw vector for AES-128, AES-192 or AES-256 respectively. } \item{mode}{ The encryption mode to use. Currently only \dQuote{electronic codebook} (ECB), \dQuote{cipher-block chaining} (CBC), \dQuote{cipher feedback} (CFB) and \dQuote{counter} (CTR) modes are supported. } \item{IV}{ The initial vector for CBC and CFB mode or initial counter for CTR mode. } \item{padding}{ Whether or not PKCS#7 padding is used during encryption and decryption in CBC mode. } } \value{ An object of class \code{"AES"}. This is a list containing the following component functions: \item{encrypt(text)}{A function to encrypt a text vector. The text may be a single element character vector or a raw vector. It returns the ciphertext as a raw vector.} \item{decrypt(ciphertext, raw = FALSE)}{A function to decrypt the ciphertext. In ECB mode, the same AES object can be used for both encryption and decryption, but in CBC, CFB and CTR modes a new object needs to be created, using the same initial \code{key} and \code{IV} values. Returns a single element character vector, or a raw vector if \code{raw = TRUE}.} \item{IV()}{Report on the current state of the initialization vector. As blocks are encrypted or decrypted in CBC, CFB or CTR mode, the initialization vector is updated, so both operations can be performed sequentially on subsets of the text or ciphertext.} \item{block_size(), key_size(), mode()}{Report on these aspects of the AES object.} } \details{ The standard NIST definition of CTR mode doesn't define how the counter is updated, it just requires that it be updated with each block and not repeat itself for a long time. This implementation treats it as a 128 bit integer and adds 1 with each successive block. } \author{ The R interface was written by Duncan Murdoch. The design is loosely based on the Python Crypto implementation. The underlying AES implementation is by Christophe Devine. } \references{ United States National Institute of Standards and Technology (2001). "Announcing the ADVANCED ENCRYPTION STANDARD (AES)". Federal Information Processing Standards Publication 197. \url{https://csrc.nist.gov/publications/fips/fips197/fips-197.pdf}. Morris Dworkin (2001). "Recommendation for Block Cipher Modes of Operation". NIST Special Publication 800-38A 2001 Edition. } \examples{ # First in ECB mode: the repeated block is coded the same way each time msg <- as.raw(c(1:16, 1:16)) key <- as.raw(1:16) aes <- AES(key, mode="ECB") aes$encrypt(msg) aes$decrypt(aes$encrypt(msg), raw=TRUE) # Now in CBC mode: each encoding is different iv <- sample(0:255, 16, replace=TRUE) aes <- AES(key, mode="CBC", iv) code <- aes$encrypt(msg) code # Need a new object for decryption in CBC mode aes <- AES(key, mode="CBC", iv) aes$decrypt(code, raw=TRUE) # In CBC mode, the input length must be a multiple of 16 bytes. # You can use `padding = TRUE` to ensure the input length is always valid. AES(key, mode="CBC", iv, padding = TRUE)$encrypt(as.raw(1:15)) # CFB mode: IV must be the same length as the Block's block size # Two different instances of AES are required for encryption and decryption iv <- sample(0:255, 16, replace=TRUE) aes <- AES(key, mode="CFB", iv) code <- aes$encrypt(msg) code #decrypt aes <- AES(key, mode="CFB", iv) aes$decrypt(code) # FIPS-197 examples hextextToRaw <- function(text) { vals <- matrix(as.integer(as.hexmode(strsplit(text, "")[[1]])), ncol=2, byrow=TRUE) vals <- vals \%*\% c(16, 1) as.raw(vals) } plaintext <- hextextToRaw("00112233445566778899aabbccddeeff") aes128key <- hextextToRaw("000102030405060708090a0b0c0d0e0f") aes128output <- hextextToRaw("69c4e0d86a7b0430d8cdb78070b4c55a") aes <- AES(aes128key) aes128 <- aes$encrypt(plaintext) stopifnot(identical(aes128, aes128output)) stopifnot(identical(plaintext, aes$decrypt(aes128, raw=TRUE))) aes192key <- hextextToRaw("000102030405060708090a0b0c0d0e0f1011121314151617") aes192output <- hextextToRaw("dda97ca4864cdfe06eaf70a0ec0d7191") aes <- AES(aes192key) aes192 <- aes$encrypt(plaintext) stopifnot(identical(aes192, aes192output)) stopifnot(identical(plaintext, aes$decrypt(aes192, raw=TRUE))) aes256key <- hextextToRaw("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") aes256output <- hextextToRaw("8ea2b7ca516745bfeafc49904b496089") aes <- AES(aes256key) aes256 <- aes$encrypt(plaintext) stopifnot(identical(aes256, aes256output)) stopifnot(identical(plaintext, aes$decrypt(aes256, raw=TRUE))) # SP800-38a examples plaintext <- hextextToRaw(paste("6bc1bee22e409f96e93d7e117393172a", "ae2d8a571e03ac9c9eb76fac45af8e51", "30c81c46a35ce411e5fbc1191a0a52ef", "f69f2445df4f9b17ad2b417be66c3710",sep="")) key <- hextextToRaw("2b7e151628aed2a6abf7158809cf4f3c") ecb128output <- hextextToRaw(paste("3ad77bb40d7a3660a89ecaf32466ef97", "f5d3d58503b9699de785895a96fdbaaf", "43b1cd7f598ece23881b00e3ed030688", "7b0c785e27e8ad3f8223207104725dd4",sep="")) aes <- AES(key) ecb128 <- aes$encrypt(plaintext) stopifnot(identical(ecb128, ecb128output)) stopifnot(identical(plaintext, aes$decrypt(ecb128, raw=TRUE))) cbc128output <- hextextToRaw(paste("7649abac8119b246cee98e9b12e9197d", "5086cb9b507219ee95db113a917678b2", "73bed6b8e3c1743b7116e69e22229516", "3ff1caa1681fac09120eca307586e1a7",sep="")) iv <- hextextToRaw("000102030405060708090a0b0c0d0e0f") aes <- AES(key, mode="CBC", IV=iv) cbc128 <- aes$encrypt(plaintext) stopifnot(identical(cbc128, cbc128output)) aes <- AES(key, mode="CBC", IV=iv) stopifnot(identical(plaintext, aes$decrypt(cbc128, raw=TRUE))) cfb128output <- hextextToRaw(paste("3b3fd92eb72dad20333449f8e83cfb4a", "c8a64537a0b3a93fcde3cdad9f1ce58b", "26751f67a3cbb140b1808cf187a4f4df", "c04b05357c5d1c0eeac4c66f9ff7f2e6",sep="")) aes <- AES(key, mode="CFB", IV=iv) cfb128 <- aes$encrypt(plaintext) stopifnot(identical(cfb128, cfb128output)) aes <- AES(key, mode="CFB", IV=iv) stopifnot(identical(plaintext, aes$decrypt(cfb128, raw=TRUE))) ctr128output <- hextextToRaw(paste("874d6191b620e3261bef6864990db6ce", "9806f66b7970fdff8617187bb9fffdff", "5ae4df3edbd5d35e5b4f09020db03eab", "1e031dda2fbe03d1792170a0f3009cee",sep="")) iv <- hextextToRaw("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff") aes <- AES(key, mode="CTR", IV=iv) ctr128 <- aes$encrypt(plaintext) stopifnot(identical(ctr128, ctr128output)) aes <- AES(key, mode="CTR", IV=iv) stopifnot(identical(plaintext, aes$decrypt(ctr128, raw=TRUE))) } digest/man/makeRaw.Rd0000644000176200001440000000107313022663127014165 0ustar liggesusers\name{makeRaw} \alias{makeRaw} \alias{makeRaw.default} \alias{makeRaw.raw} \alias{makeRaw.character} \alias{makeRaw.digest} \title{Create a raw object} \usage{ makeRaw(object) \method{makeRaw}{raw}(object) \method{makeRaw}{character}(object) \method{makeRaw}{digest}(object) \method{makeRaw}{raw}(object) } \arguments{ \item{object}{The object to convert into a \code{raw} vector} } \description{ A helper function used to create \code{raw} methods. } \value{ A \code{raw} vector is returned. } \author{Dirk Eddelbuettel} \examples{ makeRaw("1234567890ABCDE") }digest/man/hmac.Rd0000644000176200001440000000662513742111031013505 0ustar liggesusers\name{hmac} \alias{hmac} \title{compute a hash-based message authentication code} \description{ The \code{hmac} function calculates a message authentication code (MAC) involving the specified cryptographic hash function in combination with a given secret key. } \usage{ hmac(key, object, algo = c("md5", "sha1", "crc32", "sha256", "sha512"), serialize = FALSE, raw = FALSE, ...) } \arguments{ \item{key}{An arbitrary character or numeric vector, to use as pre-shared secret key.} \item{object}{An arbitrary R object which will then be passed to the \code{\link{serialize}} function, unless the \code{serialize} argument is set to \code{FALSE}.} \item{algo}{The algorithms to be used; currently available choices are \code{md5}, which is also the default, \code{sha1}, \code{crc32} and \code{sha256}.} \item{serialize}{default value of \code{serialize} is here FALSE, not TRUE as it is in \code{digest}.} \item{raw}{This flag alters the type of the output. Setting this to \code{TRUE} causes the function to return an object of type \code{"raw"} instead of \code{"character"}.} \item{...}{All remaining arguments are passed to \code{digest}. } } \value{ The \code{hmac} function uses the \code{digest} to return a hash digest as specified in the RFC 2104. } \references{ MD5: \url{https://www.ietf.org/rfc/rfc1321.txt}. SHA-1: \url{https://en.wikipedia.org/wiki/SHA-1}. SHA-256: \url{https://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf}. CRC32: The original reference webpage at \code{rocksoft.com} has vanished from the web; see \url{https://en.wikipedia.org/wiki/Cyclic_redundancy_check} for general information on CRC algorithms. \url{https://aarongifford.com/computers/sha.html} for the integrated C implementation of sha-512. The page for the code underlying the C functions used here for sha-1 and md5, and further references, is no longer accessible. Please see \url{https://en.wikipedia.org/wiki/SHA-1} and \url{https://en.wikipedia.org/wiki/MD5}. \url{https://zlib.net} for documentation on the zlib library which supplied the code for crc32. \url{https://en.wikipedia.org/wiki/SHA_hash_functions} for documentation on the sha functions. } \author{Mario Frasca \email{mfrasca@zonnet.nl}.} \seealso{\code{\link{digest}}} \examples{ ## Standard RFC 2104 test vectors current <- hmac('Jefe', 'what do ya want for nothing?', "md5") target <- '750c783e6ab0b503eaa86e310a5db738' stopifnot(identical(target, as.character(current))) current <- hmac(rep(0x0b, 16), 'Hi There', "md5") target <- '9294727a3638bb1c13f48ef8158bfc9d' stopifnot(identical(target, as.character(current))) current <- hmac(rep(0xaa, 16), rep(0xdd, 50), "md5") target <- '56be34521d144c88dbb8c733f0e8b3f6' stopifnot(identical(target, as.character(current))) ## SHA1 tests inspired to the RFC 2104 and checked against the python ## hmac implementation. current <- hmac('Jefe', 'what do ya want for nothing?', "sha1") target <- 'effcdf6ae5eb2fa2d27416d5f184df9c259a7c79' stopifnot(identical(target, as.character(current))) current <- hmac(rep(0x0b, 16), 'Hi There', "sha1") target <- '675b0b3a1b4ddf4e124872da6c2f632bfed957e9' stopifnot(identical(target, as.character(current))) current <- hmac(rep(0xaa, 16), rep(0xdd, 50), "sha1") target <- 'd730594d167e35d5956fd8003d0db3d3f46dc7bb' stopifnot(identical(target, as.character(current))) } \keyword{misc} digest/man/sha1.Rd0000644000176200001440000001055213630267061013436 0ustar liggesusers\docType{methods} \name{sha1} \alias{sha1} \alias{sha1_attr_digest} \alias{sha1_digest} \alias{sha1.array} \alias{sha1.integer} \alias{sha1.numeric} \alias{sha1.character} \alias{sha1.complex} \alias{sha1.Date} \alias{sha1.factor} \alias{sha1.NULL} \alias{sha1.logical} \alias{sha1.matrix} \alias{sha1.data.frame} \alias{sha1.list} \alias{sha1.pairlist} \alias{sha1.name} \alias{sha1.POSIXlt} \alias{sha1.POSIXct} \alias{sha1.anova} \alias{sha1.function} \alias{sha1.call} \alias{sha1.raw} \alias{sha1.formula} \alias{sha1.(} \title{Calculate a SHA1 hash of an object} \author{Thierry Onkelinx} \usage{ sha1(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{numeric}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{complex}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{Date}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{matrix}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{data.frame}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{array}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{list}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{pairlist}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{POSIXlt}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{POSIXct}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{anova}(x, digits = 4, zapsmall = 7, ..., algo = "sha1") \method{sha1}{function}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{formula}(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{`(`}(...) sha1_digest(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{NULL}(...) \method{sha1}{name}(...) sha1_attr_digest(x, digits = 14, zapsmall = 7, ..., algo = "sha1") \method{sha1}{call}(...) \method{sha1}{character}(...) \method{sha1}{factor}(...) \method{sha1}{integer}(...) \method{sha1}{logical}(...) \method{sha1}{raw}(...) } \arguments{ \item{x}{the object to calculate the SHA1} \item{digits}{the approximate number of significant digits in base 10. Will be converted to a base 16 equivalent. Defaults to \code{digits = 14}, except for sha1.anova where \code{digits = 4}} \item{zapsmall}{the approximate negative magnitude of the smallest relevant digit. Will be converted to a base 2 equivalent. Values smaller than this number are equivalent to 0. Defaults to \code{zapsmall = 7}} \item{...}{If it is the only defined argument, passed to another \code{sha1} method. If other arguments exist, see Details for usage.} \item{algo}{The hashing algoritm to be used by \code{\link{digest}}. Defaults to "sha1"} } \description{ Calculate a SHA1 hash of an object. The main difference with \code{digest(x, algo = "sha1")} is that \code{sha1()} will give the same hash on 32-bit and 64-bit systems. Note that the results depends on the setting of \code{digits} and \code{zapsmall} when handling floating point numbers. The current defaults keep \code{digits} and \code{zapsmall} as large as possible while maintaining the same hash on 32 bit and 64 bit systems. } \details{ \code{sha1_digest()} is a convenience function for objects where attributes cannot be added to apply the \code{digest()} function to its arguments. \code{sha1_attr_digest()} is a convenience function for objects where objects can be added to generate the hash. If generating hashes for objects in other packages, one of these two functions is recommended for use (typically, \code{sha1_attr_digest()}). Extra arguments: environment: An optional extra argument for \code{sha1.function} and \code{sha1.formula} should be TRUE, FALSE or missing. \code{sha1.function} and \code{sha1.formula} will ignore the enviroment of the function only when \code{environment = FALSE}. } \note{ \code{sha1} gained an \code{algo} argument since version 0.6.15. This allows \code{sha1()} to use all hashing algoritms available in \code{digest()}. The hashes created with \code{sha1(x)} from digest >= 0.6.15 are identical to \code{sha1(x)} from digest <= 0.6.14. The only exceptions are hashes created with \code{sha1(x, algo = "sha1")}, they will be different starting from digest 0.6.15 Until version 0.6.22, \code{sha1} ignored the attributes of the object for some classes. This was fixed in version 0.6.23. Use \code{options(sha1PackageVersion = "0.6.22")} to get the old behaviour. Version 0.6.24 and later ignore attributes named \code{srcref}. } digest/DESCRIPTION0000644000176200001440000001340514660651077013256 0ustar liggesusersPackage: digest Authors@R: c(person("Dirk", "Eddelbuettel", role = c("aut", "cre"), email = "edd@debian.org", comment = c(ORCID = "0000-0001-6419-907X")), person("Antoine", "Lucas", role="ctb"), person("Jarek", "Tuszynski", role="ctb"), person("Henrik", "Bengtsson", role="ctb", comment = c(ORCID = "0000-0002-7579-5165")), person("Simon", "Urbanek", role="ctb", comment = c(ORCID = "0000-0003-2297-1732")), person("Mario", "Frasca", role="ctb"), person("Bryan", "Lewis", role="ctb"), person("Murray", "Stokely", role="ctb"), person("Hannes", "Muehleisen", role="ctb"), person("Duncan", "Murdoch", role="ctb"), person("Jim", "Hester", role="ctb"), person("Wush", "Wu", role="ctb", comment = c(ORCID = "0000-0001-5180-0567")), person("Qiang", "Kou", role="ctb", comment = c(ORCID = "0000-0001-6786-5453")), person("Thierry", "Onkelinx", role="ctb", comment = c(ORCID = "0000-0001-8804-4216")), person("Michel", "Lang", role="ctb", comment = c(ORCID = "0000-0001-9754-0393")), person("Viliam", "Simko", role="ctb"), person("Kurt", "Hornik", role="ctb", comment = c(ORCID = "0000-0003-4198-9911")), person("Radford", "Neal", role="ctb", comment = c(ORCID = "0000-0002-2473-3407")), person("Kendon", "Bell", role="ctb", comment = c(ORCID = "0000-0002-9093-8312")), person("Matthew", "de Queljoe", role="ctb"), person("Dmitry", "Selivanov", role="ctb"), person("Ion", "Suruceanu", role="ctb"), person("Bill", "Denney", role="ctb"), person("Dirk", "Schumacher", role="ctb"), person("András", "Svraka", role="ctb"), person("Sergey", "Fedorov", role="ctb"), person("Will", "Landau", role="ctb", comment = c(ORCID = "0000-0003-1878-3253")), person("Floris", "Vanderhaeghe", role="ctb", comment = c(ORCID = "0000-0002-6378-6229")), person("Kevin", "Tappe", role="ctb"), person("Harris", "McGehee", role="ctb"), person("Tim", "Mastny", role="ctb"), person("Aaron", "Peikert", role="ctb", comment = c(ORCID = "0000-0001-7813-818X")), person("Mark", "van der Loo", role="ctb", comment = c(ORCID = "0000-0002-9807-4686")), person("Chris", "Muir", role="ctb", comment = c(ORCID = "0000-0003-2555-3878")), person("Moritz", "Beller", role="ctb", comment = c(ORCID = "0000-0003-4852-0526")), person("Sebastian", "Campbell", role="ctb"), person("Winston", "Chang", role="ctb", comment = c(ORCID = "0000-0002-1576-2126")), person("Dean", "Attali", role="ctb", comment = c(ORCID = "0000-0002-5645-3493")), person("Michael", "Chirico", role="ctb", comment = c(ORCID = "0000-0003-0787-087X")), person("Kevin", "Ushey", role="ctb")) Version: 0.6.37 Date: 2024-08-19 Title: Create Compact Hash Digests of R Objects Description: Implementation of a function 'digest()' for the creation of hash digests of arbitrary R objects (using the 'md5', 'sha-1', 'sha-256', 'crc32', 'xxhash', 'murmurhash', 'spookyhash', 'blake3', 'crc32c', 'xxh3_64', and 'xxh3_128' algorithms) permitting easy comparison of R language objects, as well as functions such as'hmac()' to create hash-based message authentication code. Please note that this package is not meant to be deployed for cryptographic purposes for which more comprehensive (and widely tested) libraries such as 'OpenSSL' should be used. URL: https://github.com/eddelbuettel/digest, https://dirk.eddelbuettel.com/code/digest.html BugReports: https://github.com/eddelbuettel/digest/issues Depends: R (>= 3.3.0) Imports: utils License: GPL (>= 2) Suggests: tinytest, simplermarkdown VignetteBuilder: simplermarkdown Encoding: UTF-8 NeedsCompilation: yes Packaged: 2024-08-19 12:16:05 UTC; edd Author: Dirk Eddelbuettel [aut, cre] (), Antoine Lucas [ctb], Jarek Tuszynski [ctb], Henrik Bengtsson [ctb] (), Simon Urbanek [ctb] (), Mario Frasca [ctb], Bryan Lewis [ctb], Murray Stokely [ctb], Hannes Muehleisen [ctb], Duncan Murdoch [ctb], Jim Hester [ctb], Wush Wu [ctb] (), Qiang Kou [ctb] (), Thierry Onkelinx [ctb] (), Michel Lang [ctb] (), Viliam Simko [ctb], Kurt Hornik [ctb] (), Radford Neal [ctb] (), Kendon Bell [ctb] (), Matthew de Queljoe [ctb], Dmitry Selivanov [ctb], Ion Suruceanu [ctb], Bill Denney [ctb], Dirk Schumacher [ctb], András Svraka [ctb], Sergey Fedorov [ctb], Will Landau [ctb] (), Floris Vanderhaeghe [ctb] (), Kevin Tappe [ctb], Harris McGehee [ctb], Tim Mastny [ctb], Aaron Peikert [ctb] (), Mark van der Loo [ctb] (), Chris Muir [ctb] (), Moritz Beller [ctb] (), Sebastian Campbell [ctb], Winston Chang [ctb] (), Dean Attali [ctb] (), Michael Chirico [ctb] (), Kevin Ushey [ctb] Maintainer: Dirk Eddelbuettel Repository: CRAN Date/Publication: 2024-08-19 14:10:07 UTC