"
message <- sprintf("UNRELIABLE VALUE: One of the %s iterations (%s) unexpectedly generated random numbers without declaring so. There is a risk that those random numbers are not statistically sound and the overall results might be invalid. To fix this, specify 'future.seed=TRUE'. This ensures that proper, parallel-safe random numbers are produced via the L'Ecuyer-CMRG method. To disable this check, use 'future.seed = NULL', or set option 'future.rng.onMisuse' to \"ignore\".", sQuote(.packageName), sQuote(label))
cond$message <- message
if (inherits(cond, "warning")) {
warning(cond)
invokeRestart("muffleWarning")
} else if (inherits(cond, "error")) {
stop(cond)
}
}) ## withCallingHandlers()
} else {
values <- value(fs)
}
## Not needed anymore
rm(list = "fs")
if (debug) {
mdebugf(" - Number of value chunks collected: %d", length(values))
mdebugf("Resolving %d futures (chunks) ... DONE", nchunks)
}
## Sanity check
stop_if_not(length(values) == nchunks)
if (debug) mdebugf("Reducing values from %d chunks ...", nchunks)
values2 <- do.call(c, args = values)
if (debug) {
mdebugf(" - Number of values collected after concatenation: %d",
length(values2))
mdebugf(" - Number of values expected: %d", nX)
}
assert_values2(nX, values, values2, fcn_name = fcn_name, debug = debug)
values <- values2
rm(list = "values2")
## Sanity check (this may happen if the future backend is broken)
stop_if_not(length(values) == nX)
## Were elements processed in a custom order?
if (length(values) > 1L && !is.null(ordering)) {
invOrdering <- vector(mode(ordering), length = nX)
idx <- 1:nX
invOrdering[.subset(ordering, idx)] <- idx
rm(list = c("ordering", "idx"))
if (debug) mdebugf("Reverse index remapping (attribute 'ordering'): [n = %d] %s", length(invOrdering), hpaste(invOrdering))
values <- .subset(values, invOrdering)
rm(list = c("invOrdering"))
}
if (debug) mdebugf("Reducing values from %d chunks ... DONE", nchunks)
values
} ## future_xapply()
})
future.apply/R/future_replicate.R 0000644 0001762 0000144 00000002140 14104262231 016566 0 ustar ligges users #' @inheritParams future_lapply
#'
#' @param n The number of replicates.
#'
#' @param expr An \R expression to evaluate repeatedly.
#'
#' @return
#' `future_replicate()` is a wrapper around `future_sapply()` and return
#' simplified object according to the `simplify` argument.
#' See [base::replicate()] for details.
#' Since `future_replicate()` usually involves random number generation (RNG),
#' it uses `future.seed = TRUE` by default in order produce sound random
#' numbers regardless of future backend and number of background workers used.
#'
#' @export
#'
#' @rdname future_lapply
future_replicate <- function(n, expr, simplify = "array",
future.seed = TRUE, ...,
future.envir = parent.frame(),
future.label = "future_replicate-%d")
future_sapply(X = integer(n),
FUN = eval.parent(substitute(function(...)expr)),
simplify = simplify,
future.seed = future.seed,
...,
future.envir = future.envir,
future.label = future.label)
future.apply/R/future_sapply.R 0000644 0001762 0000144 00000002036 14104262246 016140 0 ustar ligges users #' @inheritParams future_lapply
#'
#' @param simplify See [base::sapply()] and [base::tapply()], respectively.
#'
#' @param USE.NAMES See [base::sapply()].
#'
#' @return
#' For `future_sapply()`, a vector with same length and names as \code{X}.
#' See [base::sapply()] for details.
#'
#' @export
#'
#' @author
#' The implementations of `future_replicate()`, `future_sapply()`, and
#' `future_tapply()` are adopted from the source code of the corresponding
#' base \R functions, which are licensed under GPL (>= 2) with
#' 'The R Core Team' as the copyright holder.
#'
#' @rdname future_lapply
future_sapply <- function(X, FUN, ..., simplify = TRUE, USE.NAMES = TRUE, future.envir = parent.frame(), future.label = "future_sapply-%d") {
answer <- future_lapply(X = X, FUN = FUN, ..., future.envir = future.envir, future.label = future.label)
if (USE.NAMES && is.character(X) && is.null(names(answer)))
names(answer) <- X
if (!isFALSE(simplify) && length(answer))
simplify2array(answer, higher = (simplify == "array"))
else
answer
}
future.apply/R/future_apply.R 0000644 0001762 0000144 00000015550 14104262510 015754 0 ustar ligges users #' Apply Functions Over Array Margins via Futures
#'
#' `future_apply()` implements [base::apply()] using future with perfect
#' replication of results, regardless of future backend used.
#' It returns a vector or array or list of values obtained by applying a
#' function to margins of an array or matrix.
#'
#' @inheritParams future_lapply
#'
#' @param X an array, including a matrix.
#'
#' @param MARGIN A vector giving the subscripts which the function will be
#' applied over. For example, for a matrix `1` indicates rows, `2` indicates
#' columns, `c(1, 2)` indicates rows and columns.
#' Where `X` has named dimnames, it can be a character vector selecting
#' dimension names.
#'
#' @param \ldots (optional) Additional arguments passed to `FUN()`, except
#' `future.*` arguments, which are passed on to [future_lapply()] used
#' internally.
#'
#' @param simplify a logical indicating whether results should be simplified
#' if possible.
#'
#' @return
#' Returns a vector or array or list of values obtained by applying a
#' function to margins of an array or matrix.
#' See [base::apply()] for details.
#'
#' @author
#' The implementations of `future_apply()` is adopted from the source code
#' of the corresponding base \R function, which is licensed under GPL (>= 2)
#' with 'The R Core Team' as the copyright holder.
#'
#' @example incl/future_apply.R
#'
#' @importFrom future nbrOfWorkers
#' @export
future_apply <- function(X, MARGIN, FUN, ..., simplify = TRUE, future.envir = parent.frame(), future.stdout = TRUE, future.conditions = "condition", future.globals = TRUE, future.packages = NULL, future.lazy = FALSE, future.seed = FALSE, future.scheduling = 1.0, future.chunk.size = NULL, future.label = "future_apply-%d") {
debug <- getOption("future.apply.debug", getOption("future.debug", FALSE))
FUN <- match.fun(FUN)
simplify <- isTRUE(simplify)
## Ensure that X is an array object
dl <- length(dim(X))
if(!dl) stop("dim(X) must have a positive length")
if(is.object(X))
X <- if(dl == 2L) as.matrix(X) else as.array(X)
## now record dim as coercion can change it
## (e.g. when a data frame contains a matrix).
d <- dim(X)
dn <- dimnames(X)
ds <- seq_len(dl)
## Extract the margins and associated dimnames
if (is.character(MARGIN)) {
if(is.null(dnn <- names(dn))) # names(NULL) is NULL
stop("'X' must have named dimnames")
MARGIN <- match(MARGIN, dnn)
if (anyNA(MARGIN))
stop("not all elements of 'MARGIN' are names of dimensions")
}
s.call <- ds[-MARGIN]
s.ans <- ds[MARGIN]
d.call <- d[-MARGIN]
d.ans <- d[MARGIN]
dn.call <- dn[-MARGIN]
dn.ans <- dn[MARGIN]
## dimnames(X) <- NULL
## do the calls
d2 <- prod(d.ans)
if(d2 == 0L) {
## arrays with some 0 extents: return ``empty result'' trying
## to use proper mode and dimension:
## The following is still a bit `hackish': use non-empty X
newX <- array(vector(typeof(X), 1L), dim = c(prod(d.call), 1L))
ans <- forceAndCall(1, FUN, if(length(d.call) < 2L) newX[,1] else
array(newX[, 1L], d.call, dn.call), ...)
return(if(is.null(ans)) ans else if(length(d.ans) < 2L) ans[1L][-1L]
else array(ans, d.ans, dn.ans))
}
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## Globals and Packages
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
gp <- getGlobalsAndPackagesXApply(
FUN,
args = list(X = X, ...),
envir = environment(),
future.globals = future.globals,
future.packages = future.packages,
debug = debug
)
globals <- gp$globals
packages <- gp$packages
gp <- NULL
## Check size of global variables?
## Doing it here, on the matrix object, is much faster than doing it for
## the list elements passed to future_lapply()
oldMaxSize <- maxSize <- getOption("future.globals.maxSize")
if (is.null(maxSize) || is.finite(maxSize)) {
if (is.null(maxSize)) maxSize <- 500 * 1024^2
objectSize <- import_future("objectSize")
size <- objectSize(X)
nWorkers <- nbrOfWorkers()
chunk_size <- size / nWorkers
other_size <- attr(globals, "total_size")
if (is.numeric(other_size)) chunk_size <- chunk_size + other_size
if (chunk_size > maxSize) {
asIEC <- import_future("asIEC")
msg <- sprintf("The total size of %s (of class %s and type %s) is %s and the total size of the other argument is %s. With %d workers, this translates to %s per worker needed for future_apply(), which exceeds the maximum allowed size of %s (option 'future.globals.maxSize').", sQuote("X"), sQuote(class(X)[1]), sQuote(typeof(X)), asIEC(size), asIEC(other_size), nWorkers, asIEC(chunk_size), asIEC(maxSize))
if (debug) mdebug(msg)
stop(msg)
}
on.exit(options(future.globals.maxSize = oldMaxSize), add = TRUE)
options(future.globals.maxSize = +Inf)
}
newX <- aperm(X, c(s.call, s.ans))
dim(newX) <- c(prod(d.call), d2)
if(length(d.call) < 2L) {# vector
if (length(dn.call)) dimnames(newX) <- c(dn.call, list(NULL))
newX <- lapply(1L:d2, FUN = function(i) newX[,i])
} else
newX <- lapply(1L:d2, FUN = function(i)
array(newX[,i], dim = d.call, dimnames = dn.call))
globals$...future.FUN <- NULL
ans <- future_lapply(
X = newX,
FUN = FUN,
...,
future.envir = future.envir,
future.stdout = future.stdout,
future.conditions = future.conditions,
future.lazy = future.lazy,
future.seed = future.seed,
future.scheduling = future.scheduling,
future.chunk.size = future.chunk.size,
future.globals = globals,
future.packages = packages,
future.label = future.label
)
## answer dims and dimnames
ans.list <- !simplify || is.recursive(ans[[1L]])
l.ans <- length(ans[[1L]])
ans.names <- names(ans[[1L]])
if(!ans.list)
ans.list <- any(lengths(ans) != l.ans)
if(!ans.list && length(ans.names)) {
all.same <- vapply(ans, function(x) identical(names(x), ans.names), NA)
if (!all(all.same)) ans.names <- NULL
}
len.a <- if(ans.list) d2 else length(ans <- unlist(ans, recursive = FALSE))
if(length(MARGIN) == 1L && len.a == d2) {
names(ans) <- if(length(dn.ans[[1L]])) dn.ans[[1L]] # else NULL
ans
}
else if(len.a == d2)
array(ans, d.ans, dn.ans)
else if(len.a && len.a %% d2 == 0L) {
if(is.null(dn.ans)) dn.ans <- vector(mode="list", length(d.ans))
dn1 <- list(ans.names)
if(length(dn.call) && !is.null(n1 <- names(dn <- dn.call[1])) &&
nzchar(n1) && length(ans.names) == length(dn[[1]]))
names(dn1) <- n1
dn.ans <- c(dn1, dn.ans)
array(ans, c(len.a %/% d2, d.ans),
if(!is.null(names(dn.ans)) || !all(vapply(dn.ans, is.null, NA)))
dn.ans)
} else
ans
}
future.apply/R/future_Map.R 0000644 0001762 0000144 00000001420 14104262162 015336 0 ustar ligges users #' @inheritParams future_mapply
#'
#' @param f A function of the arity \eqn{k} if `future_Map()` is called with
#' \eqn{k} arguments.
#'
#' @return
#' `future_Map()` is a simple wrapper to `future_mapply()` which does not
#' attempt to simplify the result.
#' See [base::Map()] for details.
#'
#' @export
#'
#' @author
#' The implementations of `future_Map()` is adopted from the source code
#' of the corresponding base \R function `Map()`, which is licensed under
#' GPL (>= 2) with 'The R Core Team' as the copyright holder.
#'
#' @rdname future_mapply
future_Map <- function(f, ..., future.envir = parent.frame(), future.label = "future_Map-%d") {
f <- match.fun(f)
future_mapply(FUN = f, ..., SIMPLIFY = FALSE, future.envir = future.envir, future.label = future.label)
}
future.apply/R/rng.R 0000644 0001762 0000144 00000015051 14104217314 014021 0 ustar ligges users get_random_seed <- function() {
env <- globalenv()
env$.Random.seed
}
set_random_seed <- function(seed) {
env <- globalenv()
if (is.null(seed)) {
rm(list = ".Random.seed", envir = env, inherits = FALSE)
} else {
env$.Random.seed <- seed
}
}
next_random_seed <- function(seed = get_random_seed()) {
sample.int(n = 1L, size = 1L, replace = FALSE)
seed_next <- get_random_seed()
stop_if_not(!any(seed_next != seed))
invisible(seed_next)
}
is_valid_random_seed <- function(seed) {
oseed <- get_random_seed()
on.exit(set_random_seed(oseed))
env <- globalenv()
env$.Random.seed <- seed
res <- tryCatch({
sample.int(n = 1L, size = 1L, replace = FALSE)
}, simpleWarning = function(w) w)
!inherits(res, "simpleWarning")
}
## For RNGkind("L'Ecuyer-CMRG") we should have (see help('RNGkind')):
## .Random.seed <- c(rng.kind, n) where length(n) == 6L.
## From R source code: check for rng.kind %% 10000L == 407L
is_lecyer_cmrg_seed <- function(seed) {
is.numeric(seed) &&
length(seed) == 7L &&
all(is.finite(seed)) &&
(seed[1] %% 10000L == 407L)
}
# @importFrom utils capture.output
as_lecyer_cmrg_seed <- function(seed) {
## Generate a L'Ecuyer-CMRG seed (existing or random)?
if (is.logical(seed)) {
stop_if_not(length(seed) == 1L)
if (!is.na(seed) && !seed) {
stop("Argument 'seed' must be TRUE if logical: ", seed)
}
oseed <- get_random_seed()
## Already a L'Ecuyer-CMRG seed? Then use that as is.
if (!is.na(seed) && seed) {
if (is_lecyer_cmrg_seed(oseed)) return(oseed)
}
## Otherwise, generate a random one.
on.exit(set_random_seed(oseed), add = TRUE)
RNGkind("L'Ecuyer-CMRG")
return(get_random_seed())
}
stop_if_not(is.numeric(seed), all(is.finite(seed)))
seed <- as.integer(seed)
## Already a L'Ecuyer-CMRG seed?
if (is_lecyer_cmrg_seed(seed)) {
return(seed)
}
## Generate a new L'Ecuyer-CMRG seed?
if (length(seed) == 1L) {
oseed <- get_random_seed()
on.exit(set_random_seed(oseed), add = TRUE)
RNGkind("L'Ecuyer-CMRG")
set.seed(seed)
return(get_random_seed())
}
stop("Argument 'seed' must be L'Ecuyer-CMRG RNG seed as returned by parallel::nextRNGStream() or an single integer: ", capture.output(str(seed)))
}
#' Produce Reproducible Seeds for Parallel Random Number Generation
#'
#' @param count The number of RNG seeds to produce.
#'
#' @param seed A logical specifying whether RNG seeds should be generated
#' or not. (`seed = NULL` corresponds to `seed = FALSE`).
#' If a list, then it should be of length `count` and each element should
#' consist of a valid RNG seed.
#'
#' @param debug If `TRUE`, debug output is produced, otherwise not.
#'
#' @return Returns a non-named list of length `count`, or `NULL`.
#' Any seed returned is a valid RNG seed.
#'
#' @importFrom parallel nextRNGStream nextRNGSubStream splitIndices
#' @importFrom utils capture.output str
#'
#' @keywords internal
make_rng_seeds <- function(count, seed = FALSE, debug = NA) {
if (is.na(debug)) debug <- getOption("future.apply.debug", getOption("future.debug", FALSE))
## Don't use RNGs? (seed = {FALSE, NULL})
if (is.null(seed)) return(NULL)
if (is.logical(seed) && !is.na(seed) && !seed) return(NULL)
stop_if_not(is.numeric(count), length(count) == 1L, !is.na(count),
count >= 0L)
## Placeholder for all RNG stream seeds.
seeds <- NULL
# Use RNGs?
if (debug) mdebug("Generating random seeds ...")
## A pregenerated sequence of random seeds?
if (is.list(seed)) {
if (debug) mdebugf("Using a pre-define stream of %d random seeds ...", count)
seeds <- seed
nseeds <- length(seeds)
if (nseeds != count) {
stop(sprintf("Argument 'seed' is a list, which specifies the sequence of seeds to be used for each element iterated over, but length(seed) != number of elements: %g != %g", nseeds, count))
}
## Assert same type of RNG seeds?
ns <- unique(unlist(lapply(seeds, FUN = length), use.names = FALSE))
if (length(ns) != 1L) {
stop("The elements of the list specified in argument 'seed' are not all of the same lengths (did you really pass RNG seeds?): ", hpaste(ns))
}
## Did use specify scalar integers as meant for set.seed()?
if (ns == 1L) {
stop("Argument 'seed' is invalid. Pre-generated random seeds must be valid .Random.seed seeds, which means they should be all integers and consists of two or more elements, not just one.")
}
types <- unlist(lapply(seeds, FUN = typeof), use.names = FALSE)
if (!all(types == "integer")) {
stop("The elements of the list specified in argument 'seed' are not all integers (did you really pass RNG seeds?): ", hpaste(unique(types)))
}
## Check if valid random seeds are specified.
## For efficiency, only look at the first one.
if (!is_valid_random_seed(seeds[[1]])) {
stop("The list in argument 'seed' does not seem to hold elements that are valid .Random.seed values: ", capture.output(str(seeds[[1]])))
}
if (debug) {
mdebugf("Using a pre-define stream of %d random seeds ... DONE", count)
mdebug("Generating random seeds ... DONE")
}
return(seeds)
}
if (debug) mdebugf("Generating random seed streams for %d elements ...", count)
## Generate sequence of _all_ RNG seeds starting with an initial seed
## '.seed' that is based on argument 'seed'.
.seed <- as_lecyer_cmrg_seed(seed)
## future_*apply() should return with the same RNG state regardless of
## future strategy used. This is be done such that RNG kind is preserved
## and the seed is "forwarded" one step from what it was when this
## function was called. The forwarding is done by generating one random
## number. Note that this approach is also independent on the number of
## elements iterated over and the different FUN() calls.
oseed <- next_random_seed()
on.exit(set_random_seed(oseed))
seeds <- vector("list", length = count)
for (ii in seq_len(count)) {
## RNG substream seed used when calling FUN() for element(s) 'ii':
## This way each future can in turn generate further seeds, also
## recursively, with minimal risk of generating the same seeds as
## another future. This should make it safe to recursively call
## future_*apply(). /HB 2017-01-11
seeds[[ii]] <- nextRNGSubStream(.seed)
## Main random seed for next iteration (= ii + 1)
.seed <- nextRNGStream(.seed)
}
if (debug) {
mdebugf("Generating random seed streams for %d elements ... DONE", count)
mdebug("Generating random seeds ... DONE")
}
seeds
} # make_rng_seeds()
future.apply/R/future_lapply.R 0000644 0001762 0000144 00000024540 14104451747 016143 0 ustar ligges users #' Apply a Function over a List or Vector via Futures
#'
#' `future_lapply()` implements [base::lapply()] using futures with perfect
#' replication of results, regardless of future backend used.
#' Analogously, this is true for all the other `future_nnn()` functions.
#'
#' @param X A vector-like object to iterate over.
#'
#' @param FUN A function taking at least one argument.
#'
#' @param \ldots (optional) Additional arguments passed to `FUN()`.
#' For `future_*apply()` functions and `replicate()`, any `future.*` arguments
#' part of `\ldots` are passed on to `future_lapply()` used internally.
#'
#' @param future.envir An [environment] passed as argument `envir` to
#' [future::future()] as-is.
#'
#' @param future.stdout If `TRUE` (default), then the standard output of the
#' underlying futures is captured, and re-outputted as soon as possible.
#' If `FALSE`, any output is silenced (by sinking it to the null device
#' as it is outputted).
#' If `NA` (not recommended), output is _not_ intercepted.
#'
#' @param future.conditions A character string of conditions classes to be
#' captured and relayed. The default is the same as the `condition`
#' argument of [future::Future()].
#' To not intercept conditions, use `conditions = character(0L)`.
#' Errors are always relayed.
#'
#' @param future.globals A logical, a character vector, or a named list for
#' controlling how globals are handled. For details, see below section.
#'
#' @param future.packages (optional) a character vector specifying packages
#' to be attached in the R environment evaluating the future.
#'
#' @param future.lazy Specifies whether the futures should be resolved
#' lazily or eagerly (default).
#'
#' @param future.seed A logical or an integer (of length one or seven),
#' or a list of `length(X)` with pre-generated random seeds.
#' For details, see below section.
#'
#' @param future.scheduling Average number of futures ("chunks") per worker.
#' If `0.0`, then a single future is used to process all elements
#' of `X`.
#' If `1.0` or `TRUE`, then one future per worker is used.
#' If `2.0`, then each worker will process two futures
#' (if there are enough elements in `X`).
#' If `Inf` or `FALSE`, then one future per element of
#' `X` is used.
#' Only used if `future.chunk.size` is `NULL`.
#'
#' @param future.chunk.size The average number of elements per future ("chunk").
#' If `Inf`, then all elements are processed in a single future.
#' If `NULL`, then argument `future.scheduling` is used.
#'
#' @param future.label If a character string, then each future is assigned
#' a label `sprintf(future.label, chunk_idx)`. If TRUE, then the
#' same as `future.label = "future_lapply-%d"`. If FALSE, no labels
#' are assigned.
#'
#' @return
#' For `future_lapply()`, a list with same length and names as `X`.
#' See [base::lapply()] for details.
#'
#' @section Global variables:
#' Argument `future.globals` may be used to control how globals
#' should be handled similarly how the `globals` argument is used with
#' `future()`.
#' Since all function calls use the same set of globals, this function can do
#' any gathering of globals upfront (once), which is more efficient than if
#' it would be done for each future independently.
#' If `TRUE`, `NULL` or not is specified (default), then globals
#' are automatically identified and gathered.
#' If a character vector of names is specified, then those globals are gathered.
#' If a named list, then those globals are used as is.
#' In all cases, `FUN` and any `\ldots` arguments are automatically
#' passed as globals to each future created as they are always needed.
#'
#' @section Reproducible random number generation (RNG):
#' Unless `future.seed = FALSE`, this function guarantees to generate
#' the exact same sequence of random numbers _given the same initial
#' seed / RNG state_ - this regardless of type of futures, scheduling
#' ("chunking") strategy, and number of workers.
#'
#' RNG reproducibility is achieved by pregenerating the random seeds for all
#' iterations (over `X`) by using L'Ecuyer-CMRG RNG streams. In each
#' iteration, these seeds are set before calling `FUN(X[[ii]], ...)`.
#' _Note, for large `length(X)` this may introduce a large overhead._
#' As input (`future.seed`), a fixed seed (integer) may be given, either
#' as a full L'Ecuyer-CMRG RNG seed (vector of 1+6 integers) or as a seed
#' generating such a full L'Ecuyer-CMRG seed.
#' If `future.seed = TRUE`, then \code{\link[base:Random]{.Random.seed}}
#' is returned if it holds a L'Ecuyer-CMRG RNG seed, otherwise one is created
#' randomly.
#' If `future.seed = NA`, a L'Ecuyer-CMRG RNG seed is randomly created.
#' If none of the function calls `FUN(X[[ii]], ...)` uses random number
#' generation, then `future.seed = FALSE` may be used.
#'
#' In addition to the above, it is possible to specify a pre-generated
#' sequence of RNG seeds as a list such that
#' `length(future.seed) == length(X)` and where each element is an
#' integer seed vector that can be assigned to
#' \code{\link[base:Random]{.Random.seed}}. One approach to generate a
#' set of valid RNG seeds based on fixed initial seed (here `42L`) is:
#' ```r
#' seeds <- future_lapply(seq_along(X), FUN = function(x) .Random.seed,
#' future.chunk.size = Inf, future.seed = 42L)
#' ```
#' **Note that `as.list(seq_along(X))` is _not_ a valid set of such
#' `.Random.seed` values.**
#'
#' In all cases but `future.seed = FALSE`, the RNG state of the calling
#' R processes after this function returns is guaranteed to be
#' "forwarded one step" from the RNG state that was before the call and
#' in the same way regardless of `future.seed`, `future.scheduling`
#' and future strategy used. This is done in order to guarantee that an \R
#' script calling `future_lapply()` multiple times should be numerically
#' reproducible given the same initial seed.
#'
#' @section Control processing order of elements:
#' Attribute `ordering` of `future.chunk.size` or `future.scheduling` can
#' be used to control the ordering the elements are iterated over, which
#' only affects the processing order and _not_ the order values are returned.
#' This attribute can take the following values:
#' * index vector - an numeric vector of length `length(X)`
#' * function - an function taking one argument which is called as
#' `ordering(length(X))` and which must return an
#' index vector of length `length(X)`, e.g.
#' `function(n) rev(seq_len(n))` for reverse ordering.
#' * `"random"` - this will randomize the ordering via random index
#' vector `sample.int(length(X))`.
#' For example, `future.scheduling = structure(TRUE, ordering = "random")`.
#' _Note_, when elements are processed out of order, then captured standard
#' output and conditions are also relayed in that order, that is out of order.
#'
#' @example incl/future_lapply.R
#'
#' @keywords manip programming iteration
#'
#' @importFrom globals findGlobals
#' @export
future_lapply <- local({
tmpl_expr <- bquote_compile({
lapply(seq_along(...future.elements_ii), FUN = function(jj) {
...future.X_jj <- ...future.elements_ii[[jj]]
.(expr_FUN)
})
})
tmpl_expr_with_rng <- bquote_compile({
lapply(seq_along(...future.elements_ii), FUN = function(jj) {
...future.X_jj <- ...future.elements_ii[[jj]]
assign(".Random.seed", ...future.seeds_ii[[jj]], envir = globalenv(), inherits = FALSE)
.(expr_FUN)
})
})
function(X, FUN, ..., future.envir = parent.frame(), future.stdout = TRUE, future.conditions = "condition", future.globals = TRUE, future.packages = NULL, future.lazy = FALSE, future.seed = FALSE, future.scheduling = 1.0, future.chunk.size = NULL, future.label = "future_lapply-%d") {
fcn_name <- "future_lapply"
args_name <- "X"
## Coerce to as.list()?
if (!is.vector(X) || is.object(X)) X <- as.list(X)
## Nothing to do?
nX <- length(X)
if (nX == 0L) return(as.list(X))
debug <- getOption("future.apply.debug", getOption("future.debug", FALSE))
if (debug) mdebugf("%s() ...", fcn_name)
## NOTE TO SELF: We'd ideally have a 'future.envir' argument also for
## this function, cf. future(). However, it's not yet clear to me how
## to do this, because we need to have globalsOf() to search for globals
## from the current environment in order to identify the globals of
## arguments 'FUN' and '...'. /HB 2017-03-10
envir <- environment()
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## Future expression
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
...future.FUN <- NULL ## To please R CMD check
## Does FUN() rely on '...' being a global?
global_dotdotdot <- ("..." %in% findGlobals(FUN, dotdotdot = "return"))
if (global_dotdotdot) {
expr_FUN <- quote(...future.FUN(...future.X_jj))
} else {
expr_FUN <- quote(...future.FUN(...future.X_jj, ...))
}
## With or without RNG?
expr <- bquote_apply(
if (is.null(future.seed) || isFALSE(future.seed) || isNA(future.seed)) {
tmpl_expr
} else {
tmpl_expr_with_rng
}
)
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## Process
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
values <- future_xapply(
FUN = FUN,
nX = nX,
chunk_args = X,
args = list(...),
get_chunk = `[`,
expr = expr,
envir = envir,
future.envir = future.envir,
future.globals = future.globals,
future.packages = future.packages,
future.scheduling = future.scheduling,
future.chunk.size = future.chunk.size,
future.stdout = future.stdout,
future.conditions = future.conditions,
future.seed = future.seed,
future.lazy = future.lazy,
future.label = future.label,
fcn_name = fcn_name,
args_name = args_name,
debug = debug
)
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## Reduce
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
names(values) <- names(X)
if (debug) mdebugf("%s() ... DONE", fcn_name)
values
}
}) future.apply/R/makeChunks.R 0000644 0001762 0000144 00000010112 14104216357 015323 0 ustar ligges users #' Create Chunks of Index Vectors
#'
#' _This is an internal function._
#'
#' @param nbrOfElements (integer) Total number of elements to iterate over.
#'
#' @param nbrOfWorkers (integer) Number of workers available.
#'
#' @param future.scheduling (numeric) A strictly positive scalar.
#' Only used if argument `future.chunk.size` is `NULL`.
#'
#' @param future.chunk.size (numeric) The maximum number of elements per
#' chunk, or `NULL`. If `NULL`, then the chunk sizes are given by the
#' `future.scheduling` argument.
#'
#' @return A list of chunks, where each chunk is an integer vector of
#' unique indices \code{[1, nbrOfElements]}. The union of all chunks
#' holds `nbrOfElements` elements and equals `1:nbrOfElements`.
#' If `nbrOfElements == 0`, then an empty list is returned.
#'
#' @section Control processing order of elements:
#' Attribute `ordering` of `future.chunk.size` or `future.scheduling` can
#' be used to control the ordering the elements are iterated over, which
#' only affects the processing order _not_ the order values are returned.
#' This attribute can take the following values:
#' * index vector - an numeric vector of length `nbrOfElements` specifying
#' how elements are remapped
#' * function - an function taking one argument which is called as
#' `ordering(nbrOfElements)` and which must return an
#' index vector of length `nbrOfElements`, e.g.
#' `function(n) rev(seq_len(n))` for reverse ordering.
#' * `"random"` - this will randomize the ordering via random index
#' vector `sample.int(nbrOfElements)`.
#'
#' @importFrom parallel splitIndices
#' @keywords internal
makeChunks <- function(nbrOfElements, nbrOfWorkers,
future.scheduling = 1.0, future.chunk.size = NULL) {
stop_if_not(nbrOfElements >= 0L, nbrOfWorkers >= 1L)
## 'future.chunk.size != NULL' takes precedence over 'future.scheduling'
if (!is.null(future.chunk.size)) {
stop_if_not(length(future.chunk.size) == 1L, !is.na(future.chunk.size),
future.chunk.size > 0)
## Same definition as parallel:::staticNChunks() in R (>= 3.5.0)
nbrOfChunks <- max(1, ceiling(nbrOfElements / future.chunk.size))
## Customized ordering?
ordering <- attr(future.chunk.size, "ordering", exact = TRUE)
} else {
if (is.logical(future.scheduling)) {
stop_if_not(length(future.scheduling) == 1L, !is.na(future.scheduling))
if (future.scheduling) {
nbrOfChunks <- nbrOfWorkers
if (nbrOfChunks > nbrOfElements) nbrOfChunks <- nbrOfElements
} else {
nbrOfChunks <- nbrOfElements
}
} else {
## Treat 'future.scheduling' as the number of chunks per worker, i.e.
## the number of chunks each worker should process on average.
stop_if_not(length(future.scheduling) == 1L, !is.na(future.scheduling),
future.scheduling >= 0)
if (nbrOfWorkers > nbrOfElements) nbrOfWorkers <- nbrOfElements
nbrOfChunks <- future.scheduling * nbrOfWorkers
if (nbrOfChunks < 1L) {
nbrOfChunks <- 1L
} else if (nbrOfChunks > nbrOfElements) {
nbrOfChunks <- nbrOfElements
}
}
## Customized ordering?
ordering <- attr(future.scheduling, "ordering", exact = TRUE)
}
chunks <- splitIndices(nbrOfElements, ncl = nbrOfChunks)
## Customized ordering?
if (nbrOfElements > 1L && !is.null(ordering)) {
if (is.character(ordering) && ordering == "random") {
map <- stealth_sample.int(nbrOfElements)
} else if (is.numeric(ordering)) {
map <- ordering
} else if (is.function(ordering)) {
map <- ordering(nbrOfElements)
} else {
stop(sprintf("Unknown value of attribute %s for argument %s: ", "ordering", if (!is.null(future.chunk.size)) "future.chunk.size" else "future.scheduling"), mode(ordering))
}
if (!is.null(map)) {
## Simple validity check of "ordering". Looking for NAs, range,
## uniqueness is too expensive so skipped.
stop_if_not(length(map) == nbrOfElements)
attr(chunks, "ordering") <- map
}
}
chunks
}
future.apply/R/future_vapply.R 0000644 0001762 0000144 00000005476 14104262270 016153 0 ustar ligges users #' @inheritParams future_lapply
#'
#' @param FUN.VALUE A template for the required return value from
#' each `FUN(X[ii], ...)`.
#' Types may be promoted to a higher type within the ordering
#' logical < integer < double < complex, but not demoted.
#' See [base::vapply()] for details.
#'
#' @return
#' For `future_vapply()`, a vector with same length and names as \code{X}.
#' See [base::vapply()] for details.
#'
#' @export
#'
#' @rdname future_lapply
future_vapply <- function(X, FUN, FUN.VALUE, ..., USE.NAMES = TRUE, future.envir = parent.frame(), future.label = "future_vapply-%d") {
## Coerce to as.list()?
if (!is.vector(X) || is.object(X)) X <- as.list(X)
n <- length(X)
stop_if_not(is.function(FUN))
stop_if_not(is.vector(FUN.VALUE) || is.array(FUN.VALUE))
type <- typeof(FUN.VALUE)
times <- length(FUN.VALUE)
dim <- dim(FUN.VALUE)
stop_if_not(is.logical(USE.NAMES), length(USE.NAMES) == 1L, !is.na(USE.NAMES))
valid_types <- switch(
type,
logical = "logical",
integer = c("logical", "integer"),
double = c("logical", "integer", "double"),
complex = c("logical", "integer", "double", "complex"),
type
)
x_FUN <- FUN
res <- future_lapply(X, FUN = function(x, ...) {
value <- x_FUN(x, ...)
if (length(value) != times) {
stop(sprintf(
"values must be length %d, but FUN(X[[ii]]) result is length %d",
times, length(value)))
}
stop_if_not(all(dim(value) == dim), typeof(value) %in% valid_types)
value
}, ..., future.envir = future.envir, future.label = future.label)
if (!is.null(dim)) {
dim_res <- c(dim, n)
} else if (times != 1L) {
dim_res <- c(times, n)
} else {
dim_res <- NULL
}
if (USE.NAMES && length(res) > 0L) {
if (is.null(dim)) {
names_FUN.VALUE <- names(FUN.VALUE)
if (is.null(names_FUN.VALUE)) names_FUN.VALUE <- names(res[[1]])
} else {
names_FUN.VALUE <- dimnames(FUN.VALUE)
if (is.null(names_FUN.VALUE)) names_FUN.VALUE <- dimnames(res[[1]])
}
}
res <- unlist(res, use.names = FALSE)
if (is.null(res)) res <- vector(mode = type, length = 0L)
if (!is.null(dim_res)) dim(res) <- dim_res
if (USE.NAMES) {
if (is.array(res)) {
n_dim <- length(dim(res))
dimnames <- vector("list", length = n_dim)
if (is.null(dim)) {
names <- names(X)
if (!is.null(names)) dimnames[[2]] <- names
names <- names_FUN.VALUE
if (!is.null(names)) dimnames[[1]] <- names
} else {
names <- names(X)
if (!is.null(names)) dimnames[[n_dim]] <- names
names <- names_FUN.VALUE
if (!is.null(names)) dimnames[-n_dim] <- names
}
if (!all(unlist(lapply(dimnames, FUN = is.null), use.names = FALSE))) {
dimnames(res) <- dimnames
}
} else {
names(res) <- names(X)
}
}
res
}
future.apply/R/future.apply-package.R 0000644 0001762 0000144 00000007703 14024036060 017265 0 ustar ligges users #' future.apply: Apply Function to Elements in Parallel using Futures
#'
#' The \pkg{future.apply} packages provides parallel implementations of
#' common "apply" functions provided by base \R. The parallel processing
#' is performed via the \pkg{future} ecosystem, which provides a large
#' number of parallel backends, e.g. on the local machine, a remote cluster,
#' and a high-performance compute cluster.
#'
#' Currently implemented functions are:
#'
#' * [future_apply()]: a parallel version of [apply()][base::apply]
#' * [future_by()]: a parallel version of [by()][base::by]
#' * [future_eapply()]: a parallel version of [eapply()][base::lapply]
#' * [future_lapply()]: a parallel version of [lapply()][base::lapply]
#' * [future_mapply()]: a parallel version of [mapply()][base::mapply]
#' * [future_sapply()]: a parallel version of [sapply()][base::sapply]
#' * [future_tapply()]: a parallel version of [tapply()][base::tapply]
#' * [future_vapply()]: a parallel version of [vapply()][base::vapply]
#' * [future_Map()]: a parallel version of [Map()][base::Map]
#' * [future_replicate()]: a parallel version of [replicate()][base::replicate]
#' * [future_.mapply()]: a parallel version of [.mapply()][base::.mapply]
#'
#' Reproducibility is part of the core design, which means that perfect,
#' parallel random number generation (RNG) is supported regardless of the
#' amount of chunking, type of load balancing, and future backend being used.
#'
#' Since these `future_*()` functions have the same arguments as the
#' corresponding base \R function, start using them is often as simple as
#' renaming the function in the code. For example, after attaching the package:
#' ```r
#' library(future.apply)
#' ```
#' code such as:
#' ```r
#' x <- list(a = 1:10, beta = exp(-3:3), logic = c(TRUE,FALSE,FALSE,TRUE))
#' y <- lapply(x, quantile, probs = 1:3/4)
#' ```
#' can be updated to:
#' ```r
#' y <- future_lapply(x, quantile, probs = 1:3/4)
#' ```
#'
#' The default settings in the \pkg{future} framework is to process code
#' _sequentially_. To run the above in parallel on the local machine
#' (on any operating system), use:
#' ```r
#' plan(multisession)
#' ```
#' first. That's it!
#'
#' To go back to sequential processing, use `plan(sequential)`.
#' If you have access to multiple machines on your local network, use:
#' ```r
#' plan(cluster, workers = c("n1", "n2", "n2", "n3"))
#' ```
#' This will set up four workers, one on `n1` and `n3`, and two on `n2`.
#' If you have SSH access to some remote machines, use:
#' ```r
#' plan(cluster, workers = c("m1.myserver.org", "m2.myserver.org))
#' ```
#' See the \pkg{future} package and [future::plan()] for more examples.
#'
#' The \pkg{future.batchtools} package provides support for high-performance
#' compute (HPC) cluster schedulers such as SGE, Slurm, and TORQUE / PBS.
#' For example,
#'
#' * `plan(batchtools_slurm)`:
#' Process via a Slurm scheduler job queue.
#' * `plan(batchtools_torque)`:
#' Process via a TORQUE / PBS scheduler job queue.
#'
#' This builds on top of the queuing framework that the \pkg{batchtools}
#' package provides. For more details on backend configuration, please see
#' the \pkg{future.batchtools} and \pkg{batchtools} packages.
#'
#' These are just a few examples of parallel/distributed backend for the
#' future ecosystem. For more alternatives, see the 'Reverse dependencies'
#' section on the
#' [future CRAN package page](https://cran.r-project.org/package=future).
#'
#' @author
#' Henrik Bengtsson, except for the implementations of `future_apply()`,
#' `future_Map()`, `future_replicate()`, `future_sapply()`, and
#' `future_tapply()`, which are adopted from the source code of the
#' corresponding base \R functions, which are licensed under GPL (>= 2)
#' with 'The R Core Team' as the copyright holder.
#' Because of these dependencies, the license of this package is GPL (>= 2).
#'
#' @keywords manip programming iteration
#'
#' @docType package
#' @aliases future.apply-package
#' @name future.apply
NULL
future.apply/R/future_by.R 0000644 0001762 0000144 00000011666 14104262756 015261 0 ustar ligges users #' Apply a Function to a Data Frame Split by Factors via Futures
#'
#' @inheritParams future_lapply
#'
#' @param data An \R object, normally a data frame, possibly a matrix.
#'
#' @param INDICES A factor or a list of factors, each of length `nrow(data)`.
#'
#' @param FUN a function to be applied to (usually data-frame) subsets of `data`.
#'
#' @param \ldots Additional arguments pass to [future_lapply()] and
#' then to `FUN()`.
#'
#' @param simplify logical: see [base::tapply].
#'
#' @return
#' An object of class "by", giving the results for each subset.
#' This is always a list if simplify is false, otherwise a list
#' or array (see [base::tapply]).
#' See also [base::by()] for details.
#'
#' @example incl/future_by.R
#'
#' @details
#' Internally, `data` is grouped by `INDICES` into a list of `data`
#' subset elements which is then processed by [future_lapply()].
#' When the groups differ significantly in size, the processing time
#' may differ significantly between the groups.
#' To correct for processing-time imbalances, adjust the amount of chunking
#' via arguments `future.scheduling` and `future.chunk.size`.
#'
#' @section Note on 'stringsAsFactors':
#' The `future_by()` is modeled as closely as possible to the
#' behavior of `base::by()`. Both functions have "default" S3 methods that
#' calls `data <- as.data.frame(data)` internally. This call may in turn call
#' an S3 method for `as.data.frame()` that coerces strings to factors or not
#' depending on whether it has a `stringsAsFactors` argument and what its
#' default is.
#' For example, the S3 method of `as.data.frame()` for lists changed its
#' (effective) default from `stringsAsFactors = TRUE` to
#' `stringsAsFactors = TRUE` in R 4.0.0.
#'
#'
#' @rdname future_by
#' @export
future_by <- function(data, INDICES, FUN, ..., simplify = TRUE, future.envir = parent.frame()) {
future.envir <- force(future.envir)
UseMethod("future_by")
}
#' @export
future_by.default <- function(data, INDICES, FUN, ..., simplify = TRUE, future.envir = parent.frame()) {
ndim <- length(dim(data))
.SUBSETTER <- if (ndim == 0L) {
function(row) data[row, , drop = TRUE]
} else {
function(row) data[row, , drop = FALSE]
}
data <- as.data.frame(data)
future_by_internal(data = data, INDICES = INDICES, FUN = FUN, ...,
simplify = simplify,
.INDICES.NAME = deparse(substitute(INDICES))[1L],
.CALL = match.call(),
.SUBSETTER = .SUBSETTER,
future.envir = future.envir)
}
#' @export
future_by.data.frame <- function(data, INDICES, FUN, ..., simplify = TRUE, future.envir = parent.frame()) {
future_by_internal(data = data, INDICES = INDICES, FUN = FUN, ...,
simplify = simplify,
.INDICES.NAME = deparse(substitute(INDICES))[1L],
.CALL = match.call(),
.SUBSETTER = function(row) data[row, , drop = FALSE],
future.envir = future.envir)
}
future_by_internal <- function(data, INDICES, FUN, ..., simplify = TRUE, .SUBSETTER, .CALL, .INDICES.NAME, future.envir = parent.frame(), future.label = "future_by-%d") {
FUN <- if (!is.null(FUN)) match.fun(FUN)
stop_if_not(is.function(.SUBSETTER))
if (!is.list(INDICES)) {
INDEX <- vector("list", length = 1L)
INDEX[[1L]] <- INDICES
names(INDEX) <- .INDICES.NAME
INDICES <- INDEX
INDEX <- NULL ## Not needed anymore
}
INDICES <- lapply(INDICES, FUN = as.factor)
nI <- length(INDICES)
if (!nI) stop("'INDICES' is of length zero")
nd <- nrow(data)
if (!all(lengths(INDICES) == nd)) {
stop("All elements of argument 'INDICES' must have same length as 'data'")
}
namelist <- lapply(INDICES, FUN = levels)
extent <- lengths(namelist, use.names = FALSE)
cumextent <- cumprod(extent)
if (cumextent[nI] > .Machine$integer.max)
stop("total number of levels >= 2^31")
storage.mode(cumextent) <- "integer"
ngroup <- cumextent[nI]
group <- as.integer(INDICES[[1L]])
if (nI > 1L) {
for (i in 2L:nI) {
group <- group + cumextent[i - 1L] * (as.integer(INDICES[[i]]) - 1L)
}
}
cumextent <- NULL ## Not needed anymore
levels(group) <- as.character(seq_len(ngroup))
class(group) <- "factor"
ans <- split(seq_len(nd), f = group)
names(ans) <- NULL
index <- as.logical(lengths(ans) > 0L)
group <- NULL ## Not needed anymore
grouped_data <- lapply(X = ans[index], FUN = .SUBSETTER)
ans <- future_lapply(X = grouped_data, FUN = FUN, ..., future.envir = future.envir, future.label = future.label)
grouped_data <- NULL ## Not needed anymore
ansmat <- array({
if (simplify && all(lengths(ans) == 1L)) {
ans <- unlist(ans, recursive = FALSE, use.names = FALSE)
if (!is.null(ans) && is.atomic(ans)) vector(typeof(ans)) else NA
} else {
vector("list", length = prod(extent))
}
}, dim = extent, dimnames = namelist)
if (length(ans) > 0L) ansmat[index] <- ans
ans <- NULL ## Not needed anymore
structure(ansmat,
call = .CALL,
class = "by"
)
}
future.apply/MD5 0000644 0001762 0000144 00000006076 14104474122 013230 0 ustar ligges users 6efdd5ade379c3bb9eb3881171b8dacd *DESCRIPTION
3204e62059c5100b47722db1bf65ae7b *NAMESPACE
66fcb537c2bc77eecf1a002288324b7f *NEWS
c856395e17c07373fa14c21e5c8dc5f1 *R/000.import.R
59030753c107d6d43d70edb62249890f *R/001.bquote.R
140d32b29a7ab921bd7cee341830bab5 *R/fold.R
7747a7df12d3edf74da86fe498131e9d *R/future.apply-package.R
df54f772bddd565a18cd86469e37bd07 *R/future_Map.R
4abdbb471ee68e27744330b0a74345bf *R/future_apply.R
e63429bfed0625d3ce10ba2f28fcd177 *R/future_by.R
c4408ff4824298a480a5057f98dab66e *R/future_eapply.R
9c9e08ac351bf3a6a4383e24ac70ca82 *R/future_lapply.R
f11fb674a6a323277e893305831c2327 *R/future_mapply.R
381866b20dd5e68d76ae2c0bdc9eef79 *R/future_replicate.R
5ef8b5ab6e8e3c2c2122fe8bcfcf450a *R/future_sapply.R
27d3f28dae275d093fbe17e742c95f33 *R/future_tapply.R
f27027d45335a1cff2d0e39fbbdc91f9 *R/future_vapply.R
8224feae7bb003c984d034919bc51b8a *R/future_xapply.R
49c203c0e67c9bdac344900138526a90 *R/globals.R
85dedfbb022fc7e0f5dc3a79293dd239 *R/makeChunks.R
c932ad14e2982a4927dd56071a06113a *R/options.R
82a022733b127c50f15238f845364f0f *R/rng.R
3a3deebafe38f5282209ad2940a63111 *R/utils.R
b0d20cba1cf067a274d2d768befc2440 *R/zzz.R
3d294d7702823554135941a9a6d9092e *build/vignette.rds
2eba7d5b97ff92e245df449f9e4de52a *inst/CITATION
f7a83e97061ddc1cd975cf4367c8a934 *inst/WORDLIST
40842e92d752732f7ee36d5c6d35faf6 *inst/doc/future.apply-1-overview.html
8d75f91d2a7b541daae17ddcbbf22646 *inst/doc/future.apply-1-overview.md.rsp
823b12da71976538c0523b3cfa2b10aa *man/fold.Rd
ee4ce14fd8e8102c73edc35838bb2dee *man/future.apply.Rd
7a89b92bb2ab6205e7460be5d87accc9 *man/future.apply.options.Rd
15ab0a106d5aefb0a6a0c964b629e0eb *man/future_apply.Rd
84eed4efe8ceef7052103039d54fc82b *man/future_by.Rd
85924fe27f0808068b7d838241521806 *man/future_lapply.Rd
c5b10b36ce799c6d540c124d93d20416 *man/future_mapply.Rd
d64916ea7b40e821495f14c05522ed09 *man/makeChunks.Rd
96b1e638537cf8a012e4c59589113471 *man/make_rng_seeds.Rd
7205b468aaede70bf75ff613fb62acc1 *tests/fold.R
e5682aa8ba2e36695171ddf5c8cae0f6 *tests/future_apply.R
475ac0461e474ebf2d264c4e41e9062e *tests/future_by.R
16ce9cc54c711594ba68858573da0ebb *tests/future_eapply.R
6368c0b7cf1c330162596a0962c13f33 *tests/future_lapply,RNG.R
6fbf69a774e2169e81766e9b48424b36 *tests/future_lapply,globals.R
5545c54b64fa338f3246a6a15a4e6e38 *tests/future_lapply.R
334c7b0b165e54e0506fd6fcf49427fe *tests/future_mapply,globals.R
f2f3498a424ae7fb836903549876eb8a *tests/future_mapply.R
8a0e88bd5caff070ee6023064ec3a85b *tests/future_replicate.R
9ee25286d8bbeee78e57b9867895a82e *tests/future_sapply.R
d640c99128ff228abe80c17ed5f5c789 *tests/future_tapply.R
c40caf1803dab94cfd5cfd68281729ba *tests/future_vapply.R
befa096c99ef36f2194b7bd036565a46 *tests/globals,tricky_recursive.R
d1b606b24136d2d7ad8cda37792cbe62 *tests/incl/end.R
a22ffb53f44b646dc66473fec5ebb5f6 *tests/incl/start,load-only.R
0d757150beb30ecef6a18a539d432264 *tests/incl/start.R
c18b68f21db39e30a734f3080d68c21d *tests/rng.R
d23d1bde69f1d66c3922111d6f1fc875 *tests/stdout.R
12395d516dfae147fb8957b3d0b4c15f *tests/utils.R
8d75f91d2a7b541daae17ddcbbf22646 *vignettes/future.apply-1-overview.md.rsp
future.apply/inst/ 0000755 0001762 0000144 00000000000 14104466760 013675 5 ustar ligges users future.apply/inst/doc/ 0000755 0001762 0000144 00000000000 14104466760 014442 5 ustar ligges users future.apply/inst/doc/future.apply-1-overview.html 0000644 0001762 0000144 00000047466 14104466760 022011 0 ustar ligges users
A Future for R: Apply Function to Elements in Parallel
A Future for R: Apply Function to Elements in Parallel
Introduction
The purpose of this package is to provide worry-free parallel alternatives to base-R “apply” functions, e.g. apply(), lapply(), and vapply(). The goal is that one should be able to replace any of these in the core with its futurized equivalent and things will just work. For example, instead of doing:
library("datasets")
library("stats")
y <- lapply(mtcars, FUN = mean, trim = 0.10)
one can do:
library("future.apply")
plan(multisession) ## Run in parallel on local computer
library("datasets")
library("stats")
y <- future_lapply(mtcars, FUN = mean, trim = 0.10)
Reproducibility is part of the core design, which means that perfect, parallel random number generation (RNG) is supported regardless of the amount of chunking, type of load balancing, and future backend being used. To enable parallel RNG, use argument future.seed = TRUE.
Role
Where does the future.apply package fit in the software stack? You can think of it as a sibling to foreach, furrr, BiocParallel, plyr, etc. Just as parallel provides parLapply(), foreach provides foreach(), BiocParallel provides bplapply(), and plyr provides llply(), future.apply provides future_lapply(). Below is a table summarizing this idea:
| Package |
Functions |
Backends |
future.apply
|
Future-versions of common goto *apply() functions available in base R (of the ‘base’ package):
future_apply(),
future_by(),
future_eapply(),
future_lapply(),
future_Map(),
future_mapply(),
future_.mapply(),
future_replicate(),
future_sapply(),
future_tapply(), and
future_vapply().
The following function is yet not implemented:
future_rapply()
|
All future backends
|
|
parallel
|
mclapply(), mcmapply(),
clusterMap(), parApply(), parLapply(), parSapply(), …
|
Built-in and conditional on operating system
|
|
foreach
|
foreach(),
times()
|
All future backends via doFuture
|
|
furrr
|
future_imap(),
future_map(),
future_pmap(),
future_map2(),
…
|
All future backends
|
|
BiocParallel
|
Bioconductor’s parallel mappers:
bpaggregate(),
bpiterate(),
bplapply(), and
bpvec()
|
All future backends via doFuture (because it supports foreach) or via BiocParallel.FutureParam (direct BiocParallelParam support; prototype)
|
|
plyr
|
**ply(..., .parallel = TRUE) functions:
aaply(),
ddply(),
dlply(),
llply(), …
|
All future backends via doFuture (because it uses foreach internally)
|
Note that, except for the built-in parallel package, none of these higher-level APIs implement their own parallel backends, but they rather enhance existing ones. The foreach framework leverages backends such as doParallel, doMC and doFuture, and the future.apply framework leverages the future ecosystem and therefore backends such as built-in parallel, future.callr, and future.batchtools.
By separating future_lapply() and friends from the future package, it helps clarifying the purpose of the future package, which is to define and provide the core Future API, which higher-level parallel APIs can build on and for which any futurized parallel backends can be plugged into.
Roadmap
Implement future_*apply() versions for all common *apply() functions that exist in base R. This also involves writing a large set of package tests asserting the correctness and the same behavior as the corresponding *apply() functions.
Harmonize all future_*apply() functions with each other, e.g. the future-specific arguments.
Consider additional future_*apply() functions and features that fit in this package but don't necessarily have a corresponding function in base R. Examples of this may be “apply” functions that return futures rather than values, mechanisms for benchmarking, and richer control over load balancing.
The API and identity of the future.apply package will be kept close to the *apply() functions in base R. In other words, it will neither keep growing nor be expanded with new, more powerful apply-like functions beyond those core ones in base R. Such extended functionality should be part of a separate package.
future.apply/inst/doc/future.apply-1-overview.md.rsp 0000644 0001762 0000144 00000015367 14104216357 022236 0 ustar ligges users <%@meta language="R-vignette" content="--------------------------------
%\VignetteIndexEntry{A Future for R: Apply Function to Elements in Parallel}
%\VignetteAuthor{Henrik Bengtsson}
%\VignetteKeyword{R}
%\VignetteKeyword{package}
%\VignetteKeyword{vignette}
%\VignetteKeyword{future}
%\VignetteKeyword{lazy evaluation}
%\VignetteKeyword{synchronous}
%\VignetteKeyword{asynchronous}
%\VignetteKeyword{parallel}
%\VignetteKeyword{cluster}
%\VignetteEngine{R.rsp::rsp}
%\VignetteTangle{FALSE}
--------------------------------------------------------------------"%>
# A Future for R: Apply Function to Elements in Parallel
## Introduction
The purpose of this package is to provide worry-free parallel alternatives to base-R "apply" functions, e.g. `apply()`, `lapply()`, and `vapply()`. The goal is that one should be able to replace any of these in the core with its futurized equivalent and things will just work. For example, instead of doing:
```r
library("datasets")
library("stats")
y <- lapply(mtcars, FUN = mean, trim = 0.10)
```
one can do:
```r
library("future.apply")
plan(multisession) ## Run in parallel on local computer
library("datasets")
library("stats")
y <- future_lapply(mtcars, FUN = mean, trim = 0.10)
```
Reproducibility is part of the core design, which means that perfect, parallel random number generation (RNG) is supported regardless of the amount of chunking, type of load balancing, and future backend being used. _To enable parallel RNG, use argument `future.seed = TRUE`._
## Role
Where does the [future.apply] package fit in the software stack? You can think of it as a sibling to [foreach], [furrr], [BiocParallel], [plyr], etc. Just as parallel provides `parLapply()`, foreach provides `foreach()`, BiocParallel provides `bplapply()`, and plyr provides `llply()`, future.apply provides `future_lapply()`. Below is a table summarizing this idea:
| Package |
Functions |
Backends |
future.apply
|
Future-versions of common goto *apply() functions available in base R (of the 'base' package):
future_apply(),
future_by(),
future_eapply(),
future_lapply(),
future_Map(),
future_mapply(),
future_.mapply(),
future_replicate(),
future_sapply(),
future_tapply(), and
future_vapply().
The following function is yet not implemented:
future_rapply()
|
All future backends
|
|
parallel
|
mclapply(), mcmapply(),
clusterMap(), parApply(), parLapply(), parSapply(), ...
|
Built-in and conditional on operating system
|
|
foreach
|
foreach(),
times()
|
All future backends via doFuture
|
|
furrr
|
future_imap(),
future_map(),
future_pmap(),
future_map2(),
...
|
All future backends
|
|
BiocParallel
|
Bioconductor's parallel mappers:
bpaggregate(),
bpiterate(),
bplapply(), and
bpvec()
|
All future backends via doFuture (because it supports foreach) or via BiocParallel.FutureParam (direct BiocParallelParam support; prototype)
|
|
plyr
|
**ply(..., .parallel = TRUE) functions:
aaply(),
ddply(),
dlply(),
llply(), ...
|
All future backends via doFuture (because it uses foreach internally)
|
Note that, except for the built-in parallel package, none of these higher-level APIs implement their own parallel backends, but they rather enhance existing ones. The foreach framework leverages backends such as [doParallel], [doMC] and [doFuture], and the future.apply framework leverages the [future] ecosystem and therefore backends such as built-in parallel, [future.callr], and [future.batchtools].
By separating `future_lapply()` and friends from the [future] package, it helps clarifying the purpose of the future package, which is to define and provide the core Future API, which higher-level parallel APIs can build on and for which any futurized parallel backends can be plugged into.
## Roadmap
1. Implement `future_*apply()` versions for all common `*apply()` functions that exist in base R. This also involves writing a large set of package tests asserting the correctness and the same behavior as the corresponding `*apply()` functions.
2. Harmonize all `future_*apply()` functions with each other, e.g. the future-specific arguments.
3. Consider additional `future_*apply()` functions and features that fit in this package but don't necessarily have a corresponding function in base R. Examples of this may be "apply" functions that return futures rather than values, mechanisms for benchmarking, and richer control over load balancing.
The API and identity of the future.apply package will be kept close to the `*apply()` functions in base R. In other words, it will _neither_ keep growing nor be expanded with new, more powerful apply-like functions beyond those core ones in base R. Such extended functionality should be part of a separate package.
[batchtools]: https://cran.r-project.org/package=batchtools
[BiocParallel]: https://bioconductor.org/packages/BiocParallel/
[doFuture]: https://cran.r-project.org/package=doFuture
[doMC]: https://cran.r-project.org/package=doMC
[doParallel]: https://cran.r-project.org/package=doParallel
[foreach]: https://cran.r-project.org/package=foreach
[future]: https://cran.r-project.org/package=future
[future.apply]: https://cran.r-project.org/package=future.apply
[future.batchtools]: https://cran.r-project.org/package=future.batchtools
[future.callr]: https://cran.r-project.org/package=future.callr
[furrr]: https://cran.r-project.org/package=furrr
[plyr]: https://cran.r-project.org/package=plyr
future.apply/inst/CITATION 0000644 0001762 0000144 00000001676 14104465322 015035 0 ustar ligges users citHeader("Please cite 'future' and the future framework using the following references:")
citEntry(
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# BibTeX entry:
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
entry = "Misc",
author = "Henrik Bengtsson",
title = "A Unifying Framework for Parallel and Distributed Processing in R using Futures",
year = "2021",
# doi = "10.32614/RJ-2021-048", ## until DOI URL exist
note = "10.32614/RJ-2021-048",
url = "https://journal.r-project.org/archive/2021/RJ-2021-048/index.html",
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Plain-text citation:
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
textVersion = paste(sep="",
"H. Bengtsson, ",
"A Unifying Framework for Parallel and Distributed Processing in R using Futures, ",
"The R Journal, ",
"2021, ",
"doi:10.32614/RJ-2021-048"
)
)
future.apply/inst/WORDLIST 0000644 0001762 0000144 00000000441 14024036060 015051 0 ustar ligges users AppVeyor
arity
batchtools
benchmarking
BiocParallel
callr
CMD
CMRG
doFuture
doMC
doParallel
eapply
foreach
furrr
futurized
globals
HPC
L'Ecuyer
lapply
macOS
mapply
plyr
pre
Pre
pregenerating
reproducibility
Reproducibility
Roadmap
sapply
SGE
Slurm
stringsAsFactors
tapply
vapply
vectorize