BatchJobs/0000755000176200001440000000000013207317714012116 5ustar liggesusersBatchJobs/inst/0000755000176200001440000000000012502333614013065 5ustar liggesusersBatchJobs/inst/CITATION0000644000176200001440000000177312502333614014232 0ustar liggesusers## -*- mode: r -*- citHeader("To cite BatchJobs or BatchExperiments in publications use:") citEntry(entry = "Article", title = "{BatchJobs} and {BatchExperiments}: Abstraction Mechanisms for Using {R} in Batch Environments", author = personList(as.person("Bernd Bischl"), as.person("Michel Lang"), as.person("Olaf Mersmann"), as.person("J{\\\"o}rg Rahnenf{\\\"u}hrer"), as.person("Claus Weihs")), journal = "Journal of Statistical Software", year = "2015", volume = "64", number = "11", pages = "1--25", url = "http://www.jstatsoft.org/v64/i11/", textVersion = paste("Bernd Bischl, Michel Lang, Olaf Mersmann, Joerg Rahnenfuehrer, Claus Weihs (2015).", "BatchJobs and BatchExperiments: Abstraction Mechanisms for Using R in Batch Environments.", "Journal of Statistical Software, 64(11), 1-25.", "URL http://www.jstatsoft.org/v64/i11/.") ) BatchJobs/inst/bin/0000755000176200001440000000000012357424573013652 5ustar liggesusersBatchJobs/inst/bin/linux-helper0000755000176200001440000000746712357424573016232 0ustar liggesusers#!/bin/bash ## linux-helper: Helper for the multicore and SSH cluster functions of the BatchJobs R ## package. ## ## Requires the following Unix command line utilities: ## ## * grep, wc, ps, kill, uptime, echo, cat, possibly setsid ## ## The following commands are implemented. First argument is always the command name. ## For other arguments see below. Each command returns a character vector. ## ## number-of-cpus ## Return the number of PEs on worker. ## ## start-job RHOME NICE ROPTIONS RFILE OUTFILE ## Start an R CMD BATCH process running $RFILE and log ## the output in $OUTFILE. ## RHOME is the path to the R installation. ## Returns: PID of sh process which spawned R. We use that as batch.job.id. ## ## kill-job PID ## Kill the R job with PPID $PID. The PPID is the PID of ## the sh process returned by start-job. ## First a TERM is sent, then 1 sec delay, then KILL. ## ## status FILEDIR ## Return 4 numbers: ## - load average of last 1 min, as given by e.g. uptime ## - number of R processes by _all_ users ## - number of R processes by _all_ users which have a load of >= 50% ## - number of R processes by current user ## which match $FILEDIR/jobs in the cmd call of R ## ## list-jobs FILEDIR ## Return the PPIDs of running R jobs operating on $FILEDIR/jobs. ### efficient [ -n `which [prog]` ] command_exists () { hash "$1" 2> /dev/null; } CMD="$1"; shift export LC_ALL=C ### Avoid any localization issues. shopt -s nocasematch ### Case insensitive regular expressions case $CMD in number-of-cpus) if [[ `uname` =~ "Linux" ]]; then cat /proc/cpuinfo | grep '^processor' | wc -l else sysctl -n hw.ncpu ## darwin fi ;; start-job) RCMD="R" if [ -n "$1" ]; then RCMD="${1}/bin/R" fi # only use setsid on linux, not on darwin (not available there) if [[ `uname` =~ "Linux" ]] && command_exists setsid; then RCMD="setsid $RCMD" fi if [ -n "$2" ] && command_exists nice; then RCMD="nice -n ${2} $RCMD" fi shift 2 # nothing should be on all 3 streams except maybe a segfault. throw away. # see comment also in cfLocal # we use setsid to start the process in a different session. other ctrl+C in the master # session leads to killing of child processes in multicore mode as the SIGINT is passed down # $@ ${RCMD} CMD BATCH $@ \ > /dev/null 2> /dev/null < /dev/null & echo $! ;; kill-job) RPPID=$1 # find R process whose ppid is the batch.job.id, i.e. the one of the sh process # we must list all processes (without terminal) for SSH RPID=$(ps -e -o pid= -o ppid= | awk -v j=$RPPID '$2 == j {print $1}') kill -TERM $RPID > /dev/null 2> /dev/null sleep 1 kill -KILL $RPID > /dev/null 2> /dev/null exit 0 ;; status) # remove everyting till load average(s): # then delete commas LOAD=$(uptime | awk '{gsub(/.*:/,""); {gsub(/,/,"")}; print $1}') JOBDIR="$1/jobs" # print 3 columns for all processes # use ww for unlimited width in ps for command output # we count all R procs, all R50, and all where JOBDIR was in the call args ps -e -ww -o pcpu= -o ucomm= -o command= | \ awk -v j=$JOBDIR -v sysload=$LOAD ' BEGIN {rprocs=0;rprocs_50=0;njobs=0} $2 != "R" {next} {rprocs++} $1 > 50.0 {rprocs_50++} $0 ~ j {njobs++} END {print sysload " " rprocs " " rprocs_50 " " njobs}' ;; list-jobs) JOBDIR="$1/jobs" ps -e -ww -o ppid= -o ucomm= -o command= | \ awk -v j=$JOBDIR '$2 == "R" && $0 ~ j { print $1 }' ;; *) esac BatchJobs/inst/etc/0000755000176200001440000000000012357424573013655 5ustar liggesusersBatchJobs/inst/etc/BatchJobs_global_config.R0000644000176200001440000000025012357424573020501 0ustar liggesuserscluster.functions = makeClusterFunctionsInteractive() mail.start = "none" mail.done = "none" mail.error = "none" db.driver = "SQLite" db.options = list() debug = FALSE BatchJobs/tests/0000755000176200001440000000000012571265761013267 5ustar liggesusersBatchJobs/tests/run-all.R0000644000176200001440000000007512571265761014766 0ustar liggesuserslibrary(testthat) library(checkmate) test_check("BatchJobs") BatchJobs/tests/testthat/0000755000176200001440000000000013207301764015116 5ustar liggesusersBatchJobs/tests/testthat/test_sweepRegistry.R0000644000176200001440000000134112413244720021147 0ustar liggesuserscontext("sweepRegistry") test_that("sweepRegistry", { countFiles = function(reg, type) { fun = switch(type, scripts = BatchJobs:::getRScriptFilePath, logs = BatchJobs:::getLogFilePath ) fs = sapply(getJobIds(reg), fun, reg = reg) sum(sapply(fs, file.exists)) } reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) expect_equal(countFiles(reg, "scripts"), 3L) expect_equal(countFiles(reg, "logs"), 3L) sweepRegistry(reg, "scripts") expect_equal(countFiles(reg, "scripts"), 0L) expect_equal(countFiles(reg, "logs"), 3L) sweepRegistry(reg, "logs") expect_equal(countFiles(reg, "scripts"), 0L) expect_equal(countFiles(reg, "logs"), 0L) }) BatchJobs/tests/testthat/test_export.R0000644000176200001440000000416713207234653017633 0ustar liggesuserscontext("exports") test_that("exports", { reg = makeTestRegistry() p = file.path(reg$file.dir, "exports") save2(exported_x = 1:3, file = file.path(p, "x.RData")) loadExports(reg) expect_true(exists("exported_x", where=.GlobalEnv)) expect_equal(exported_x, 1:3) batchMap(reg, function(i) exported_x[i], i = 1:3) submitJobs(reg) waitForJobs(reg) res = loadResults(reg, simplify=TRUE, use.names="none") expect_equal(res, 1:3) }) test_that("exports with batchExport and batchUnexport", { reg = makeTestRegistry() expect_error(batchExport(reg, a = 1, li = list(a = 2)), "more than once") expect_equal(batchExport(reg, a = 1, b = 99), c("a", "b")) batchMap(reg, function(x) x + a + b, 1) submitJobs(reg) waitForJobs(reg) expect_equal(loadResult(reg, 1), 101) if (interactive()) { expect_equal(testJob(reg, 1), 101) expect_equal(testJob(reg, 1, external = FALSE), 101) expect_equal(testJob(reg, 1, external = TRUE), 101) } expect_equal(batchUnexport(reg, "a"), "a") suppressWarnings(rm(list = "a", envir = .GlobalEnv)) submitJobs(reg, 1) waitForJobs(reg) expect_equal(length(findErrors(reg)), 1) expect_true(grepl("(found|gefunden)", getErrorMessages(reg, 1), fixed = FALSE)) }) test_that("export: load defined files with loadExports", { reg = makeTestRegistry() batchExport(reg, x.1 = 4, x.2 = 3, x.3 = 2, x.4 = 1) suppressMessages(loadExports(reg)) expect_true(exists("x.1", where = .GlobalEnv)) expect_true(exists("x.2", where = .GlobalEnv)) expect_true(exists("x.3", where = .GlobalEnv)) expect_true(exists("x.4", where = .GlobalEnv)) }) #FIXME: # I currently do not know how to run this test so it does not break R CMD Check # I get # > cannot open file 'startup.Rs': No such file or directory # 'make test' would work # --> run it only in interactive tests for now if (interactive()) { test_that("exports work with external testJob", { reg = makeTestRegistry() p = file.path(reg$file.dir, "exports") save2(exported_x = 1:3, file = file.path(p, "x.RData")) batchMap(reg, function(i) exported_x[i], i = 1:2) res = testJob(reg, 2L, external=TRUE) expect_equal(res, 2L) }) } BatchJobs/tests/testthat/helpers.R0000644000176200001440000000234712705156101016704 0ustar liggesuserslibrary(BBmisc) library(checkmate) options(BBmisc.ProgressBar.style = "off") options(BatchJobs.verbose = FALSE) conf = getConfig() conf$default.resources$walltime = 120 conf$default.resources$memory = 256 conf$mail.start = conf$mail.done = conf$mail.error = "none" conf$raise.warnings = FALSE conf$max.concurrent.jobs = Inf setConfig(conf = conf) rm(conf) getWorkDir = function() { work.dir = "unittests-files" # if env var is set we do the tests in the current wd, # if not, we do everything in a subdir of tempdir() # required for debians autopkgtest if (isExpensiveExampleOk()) file.path(getwd(), work.dir) else file.path(tempdir(), work.dir) } getTempDir = function() { file.path(getWorkDir(), basename(tempfile("tmp"))) } makeTestRegistry = function(packages = character(0L), work.dir = getWorkDir(), file.dir = getTempDir(), ...) { dir.create(file.dir, recursive = TRUE, showWarnings = FALSE) makeRegistry( id = "unittests", seed = 1L, packages = packages, file.dir = file.dir, work.dir = work.dir, ... ) } # cleanup stray files cleanup = function() { unittests.dir = getWorkDir() if (file.exists(unittests.dir)) return(unlink(unittests.dir, recursive = TRUE) == 0L) return(TRUE) } BatchJobs/tests/testthat/test_reduceResults.R0000644000176200001440000001527512623105424021137 0ustar liggesuserscontext("reduceResults") test_that("reduceResults", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) y = reduceResults(reg, fun=function(aggr, job, res) aggr+res, init=0L) expect_equal(y, sum((1:3)^2)) }) test_that("reduceResults works with empty results", { reg = makeTestRegistry() batchMap(reg, function(x) x, 1) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1), NULL) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1, init=0L), 0L) submitJobs(reg) waitForJobs(reg) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1, ids=integer(0L)), NULL) expect_equal(reduceResults(reg, fun=function(aggr, job, res) 1, ids=integer(0L), init=0L), 0L) }) test_that("reduceResults works with imputations", { reg = makeTestRegistry() batchMap(reg, identity, 1:5) submitJobs(reg, c(1:2, 4:5)) waitForJobs(reg) expect_equal(reduceResults(reg, ids=1:5, fun=function(aggr, job, res) aggr+res, impute.val=3), 15) expect_equal(reduceResults(reg, ids=1:5, fun=function(aggr, job, res) sum(aggr, res, na.rm=TRUE), impute.val=NA), 12) }) test_that("reduceResults with multiple.result.files", { reg = makeTestRegistry(multiple.result.files=TRUE) batchMap(reg, function(x) list(foo = x, bar = 1), 1:10) submitJobs(reg) waitForJobs(reg) # quick check for named list as return value of loadResult expect_equal(loadResult(reg, 2)$foo, 2) expect_equal(loadResult(reg, 2, part="foo"), list(foo=2)) expect_equal(loadResult(reg, 2, part="bar"), list(bar=1)) expect_equal(loadResult(reg, 2, part=c("bar", "foo")), list(bar=1, foo=2)) # no part = all parts expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c()), 1:10) expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c(), part = c("foo", "bar")), 1:10) # wrong part expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c(), part="bar"), NULL) # right part expect_equal(reduceResults(reg, fun=function(aggr, job, res) c(aggr, res$foo), init = c(), part="foo"), 1:10) }) test_that("reduceResultsReturnValue", { xs = 1:3 reg = makeTestRegistry() batchMap(reg, function(x) x^2, xs) submitJobs(reg) waitForJobs(reg) z1 = (1:3)^2 expect_equal(reduceResultsVector(reg, use.names="none"), z1) expect_equal(reduceResultsList(reg, use.names="none"), as.list(z1)) expect_equal(reduceResultsMatrix(reg, use.names="none"), matrix(z1, ncol=1)) # data.frame w/o colnames is nothing we want to test ... # y = data.frame(z1); colnames(y) = NULL y = data.frame(z1); colnames(y) = "X"; rownames(y) = as.character(rownames(y)) expect_equal(reduceResultsDataFrame(reg), y) z2 = z1 names(z2) = xs names(z2) = xs expect_equal(reduceResultsVector(reg, use.names="ids"), z2) expect_equal(reduceResultsList(reg, use.names="ids"), as.list(z2)) y = matrix(z2, ncol=1); rownames(y)=xs; colnames(y) = NULL expect_equal(reduceResultsMatrix(reg, use.names="ids"), y) reg = makeTestRegistry() batchMap(reg, function(x) list(a=x, b=x^2), xs) submitJobs(reg) waitForJobs(reg) expect_equal(reduceResultsVector(reg, fun=function(job, res) res$a, use.names="none"), xs) expect_equal(reduceResultsVector(reg, fun=function(job, res) res$b, use.names="none"), xs^2) expect_equal(reduceResultsList(reg, fun=function(job, res) res$a, use.names="none"), as.list(xs)) expect_equal(reduceResultsList(reg, fun=function(job, res) res$b, use.names="none"), as.list((xs)^2)) y = cbind(xs, z1); dimnames(y) = NULL expect_equal(reduceResultsMatrix(reg, use.names="none"), y) rownames(y) = xs; colnames(y) = c("a", "b") expect_equal(reduceResultsMatrix(reg, use.names="ids"), y) dimnames(y) = NULL; y = t(y) expect_equal(reduceResultsMatrix(reg, use.names="none", rows=FALSE), y) colnames(y) = xs; rownames(y) = c("a", "b") expect_equal(reduceResultsMatrix(reg, use.names="ids", rows=FALSE), y) y = data.frame(xs, z1); rownames(y) = as.character(xs); colnames(y) = c("a", "b") expect_equal(reduceResultsDataFrame(reg), y) reg = makeTestRegistry() batchMap(reg, function(i) list(a="a", b="b"), 1:2) submitJobs(reg) waitForJobs(reg) y = data.frame(a=rep("a", 2), b=rep("b", 2), stringsAsFactors=FALSE) rownames(y) = as.character(rownames(y)) expect_equal(reduceResultsDataFrame(reg, strings.as.factors=FALSE), y) y = data.frame(a=rep("a", 2), b=rep("b", 2), stringsAsFactors=TRUE) rownames(y) = as.character(rownames(y)) expect_equal(reduceResultsDataFrame(reg, strings.as.factors=TRUE), y) library(data.table) reg = makeTestRegistry() batchMap(reg, function(i) list(a="a", b="b"), 1:2) submitJobs(reg) waitForJobs(reg) x = reduceResultsDataTable(reg) y = data.table(a=rep("a", 2), b=rep("b", 2)) expect_equal(x, y) }) test_that("reduceResultsReturnValue works with empty results", { xs = integer(0) reg = makeTestRegistry() batchMap(reg, function(x) identity, xs) submitJobs(reg) waitForJobs(reg) expect_equal(reduceResultsVector(reg, use.names="none"), c()) expect_equal(reduceResultsList(reg, use.names="none"), list()) expect_equal(reduceResultsMatrix(reg, use.names="none"), matrix(0, nrow=0,ncol=0)) expect_equal(reduceResultsDataFrame(reg, use.names="none"), data.frame()) }) test_that("reduceResultsReturnValue works with imputations", { reg = makeTestRegistry() ids = batchMap(reg, identity, 1:5) submitJobs(reg, 1:4) waitForJobs(reg) res = c(1:4, NA_real_) expect_equal(reduceResultsVector(reg, ids, impute.val=NA_real_), setNames(res, 1:5)) expect_equal(reduceResultsList(reg, ids, impute.val=NA_real_), setNames(as.list(res), 1:5)) expect_equal(reduceResultsMatrix(reg, ids, impute.val=NA_real_), setRowNames(matrix(res), 1:5)) expect_equal(reduceResultsDataFrame(reg, ids, impute.val=NA_real_, use.names="none"), data.frame(X = res)) }) test_that("reduceResultsReturnValue works with other args", { reg = makeTestRegistry() xs = 1:3 batchMap(reg, identity, xs) submitJobs(reg) waitForJobs(reg) z = reduceResultsMatrix(reg, fun = function(job, res, y) (res-y)^2, y = 1, use.names = "none") expect_equal(z[,1], (xs-1)^2) }) test_that("reduceResultsReturnValue works with job names", { reg = makeTestRegistry() ns = letters[1:3] xs = setNames(1:3, ns) batchMap(reg, identity, xs, use.names=TRUE) submitJobs(reg) waitForJobs(reg) expect_equal(names(reduceResultsList(reg, use.names="names")), ns) expect_equal(names(reduceResultsVector(reg, use.names="names")), ns) expect_equal(rownames(reduceResultsDataFrame(reg, use.names="names")), ns) expect_equal(rownames(reduceResultsMatrix(reg, use.names="names")), ns) expect_equal(colnames(reduceResultsMatrix(reg, use.names="names", rows=FALSE)), ns) }) BatchJobs/tests/testthat/test_getJobInfo.R0000644000176200001440000000437712623105424020335 0ustar liggesuserscontext("getJobInfo") test_that("getJobInfo", { reg = makeTestRegistry() batchMap(reg, function(x, i) x^i, 1:3, i = rep(2, 3)) mycheck = function(tab) { expect_true(is.data.frame(tab)) expect_equal(tab$id, 1:3) expect_true(nrow(tab) == 3) expect_true(is(tab$time.submitted, "POSIXt")) expect_true(is(tab$time.started, "POSIXt")) expect_true(is(tab$time.done, "POSIXt")) expect_true(is.numeric(tab$time.running)) expect_true(is.numeric(tab$memory)) expect_true(is.numeric(tab$time.queued)) expect_true(all(is.na(tab$error.msg))) expect_true(is.integer(tab$r.pid)) expect_true(is.integer(tab$seed)) } tab = getJobInfo(reg) mycheck(tab) submitJobs(reg) waitForJobs(reg) tab = getJobInfo(reg) mycheck(tab) tab = getJobInfo(reg, ids = integer(0)) expect_true(is.data.frame(tab)) expect_true(nrow(tab) == 0L) tab = getJobInfo(reg, pars=TRUE) expect_equal(tab$X, 1:3) expect_equal(tab$i, rep(2, 3)) expect_equal(tab$pars, NULL) tab = getJobInfo(reg, select = "time.running") expect_true(ncol(tab) == 2) # id always selected expect_true(names(tab)[1] == "id") tab = getJobInfo(reg, select = c("id", "time.running")) expect_true(ncol(tab) == 2) expect_error(getJobInfo(reg, select = "fooo"), "subset") }) test_that("getJobInfo works with error and time.running", { reg = makeTestRegistry() f = function(x) { if (x == 1) return(1) else stop("xxx") } batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) i = getJobInfo(reg) expect_equal(is.na(i$error), c(TRUE, FALSE)) expect_false(any(is.na(i$time.running))) }) test_that("turn off memory calculation", { reg = makeTestRegistry() ids = batchMap(reg, function(n) { x = 1:(10^n); x + rev(x) }, n = c(1, 5)) submitJobs(reg, ids) waitForJobs(reg, ids) mem = getJobInfo(reg, select = "memory")$memory expect_numeric(mem, any.missing = FALSE) expect_true(mem[2] >= mem[1]) conf = getConfig() on.exit(setConfig(conf = conf)) setConfig(measure.mem = FALSE) reg = makeTestRegistry() ids = batchMap(reg, function(n) { x = 1:(10^n); x + rev(x) }, n = c(1, 5)) submitJobs(reg, ids) waitForJobs(reg, ids) mem = getJobInfo(reg, select = "memory")$memory expect_equal(mem, c(-1, -1)) }) BatchJobs/tests/testthat/test_sanitizePath.R0000644000176200001440000000277412413244720020751 0ustar liggesuserscontext("sanitizePath") test_that("sanitizePath", { cS = function(x) gsub("\\", "/", x, fixed = TRUE) if (isLinux()) { setwd(tempdir()) paths = c(tempfile(), "..", "~") expect_identical(sanitizePath(paths, make.absolute = TRUE), c(paths[1], dirname(tempdir()), path.expand("~"))) expect_equal(sanitizePath(paths, make.absolute = TRUE) == paths, c(TRUE, FALSE, FALSE)) } else if (isWindows()) { setwd(tempdir()) paths = c(tempfile(), "..") expect_identical(sanitizePath(paths), cS(c(paths[1], dirname(tempdir())))) expect_equal(sanitizePath(paths, normalize = TRUE) == cS(paths), c(TRUE, FALSE)) } else if (isDarwin()) { # mac os often has a symlinked tempdir setwd(tempdir()) paths = c(tempfile(), "~") expect_identical(sanitizePath(paths, make.absolute = TRUE), cS(c(paths[1], path.expand("~")))) expect_equal(sanitizePath(paths, normalize = TRUE) == cS(paths), c(TRUE, FALSE)) } path = "c:\\temp" expect_false(sanitizePath(path, make.absolute = FALSE) == path) expect_identical(sanitizePath(path, make.absolute = FALSE), cS(path)) expect_identical(sanitizePath(path, make.absolute = TRUE), cS(path)) path = file.path(tempdir(), "..", fsep = "/") expect_false(sanitizePath(path, make.absolute = TRUE, normalize.absolute = TRUE) == cS(path)) expect_identical(sanitizePath(path, make.absolute = FALSE), cS(path)) if (!isDarwin()) expect_identical(sanitizePath(path, make.absolute = TRUE, normalize.absolute = TRUE), cS(dirname(dirname(path)))) }) BatchJobs/tests/testthat/test_testJob.R0000644000176200001440000000144412357424573017726 0ustar liggesuserscontext("testJob") test_that("testJob", { reg = makeTestRegistry() id = 1L batchMap(reg, identity, 1) res = suppressAll(testJob(reg, id, external=FALSE)) expect_equal(res, 1) ids = findNotDone(reg) expect_equal(ids, id) expect_output({ st = showStatus(reg) }, "Status for 1 jobs") expect_equal(st$submitted, 0) expect_equal(st$started, 0) expect_equal(st$done, 0) expect_equal(st$error, 0) reg = makeTestRegistry() f = function(i) {library(xxxxx);1} batchMap(reg, f, 1) expect_error(testJob(reg, id, external=FALSE), "xxxxx") ids = findNotDone(reg) expect_equal(ids, id) expect_output({ df = showStatus(reg) }, "Status for 1 jobs") expect_equal(st$submitted, 0) expect_equal(st$started, 0) expect_equal(st$done, 0) expect_equal(st$error, 0) }) BatchJobs/tests/testthat/test_filterResults.R0000644000176200001440000000106012357424573021155 0ustar liggesuserscontext("filterResults") test_that("filterResults", { reg = makeTestRegistry() batchMap(reg, function(x) x, 1:5) submitJobs(reg) waitForJobs(reg) ids = filterResults(reg, fun=function(job, res) res > 3) expect_equal(ids, c(4, 5)) ids = filterResults(reg, getJobIds(reg)[2:4], fun=function(job, res) res > 3) expect_equal(ids, c(4)) }) test_that("filterResults works with empty results", { reg = makeTestRegistry() batchMap(reg, function(x) x, 1) ids = filterResults(reg, fun=function(job, res) TRUE) expect_equal(ids, integer(0)) }) BatchJobs/tests/testthat/test_getJobParamDf.R0000644000176200001440000000037212413244720020743 0ustar liggesuserscontext("getJobParamDf") test_that("getJobParamDf", { reg = makeTestRegistry() grid = batchExpandGrid(reg, function(x,y) x*y, x = 1:3, y = 5) expect_equal(getJobParamDf(reg), grid) expect_equal(getJobParamDf(reg, ids = 2:3), grid[2:3,]) }) BatchJobs/tests/testthat/test_batchMapQuick.R0000644000176200001440000000261312416240471021014 0ustar liggesuserscontext("batchMapQuick") test_that("batchMapQuick", { reg = batchMapQuick(function(x) x^2, 1:3, file.dir = getTempDir()) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, (1:3)^2, check.attributes = FALSE) reg = batchMapQuick(function(x,y,z) (x+y)*z, 1:3, 4:6, more.args = list(z = 2), file.dir = getTempDir()) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, ((1:3) + (4:6))*2, check.attributes = FALSE) reg = batchMapQuick(identity, 1:3, packages = "MASS", chunk.size = 2, file.dir = getTempDir()) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, 1:3, check.attributes = FALSE) reg = batchMapQuick(identity, 1:3, inds = c(1,3), file.dir = getTempDir()) waitForJobs(reg) expect_equal(findDone(reg), c(1,3)) y = sapply(c(1,3), function(id) loadResult(reg, id)) expect_equal(y, c(1,3), check.attributes = FALSE) reg = batchMapQuick(identity, file.dir = getTempDir()) waitForJobs(reg) expect_equal(getJobNr(reg), 0L) }) test_that("batchMapQuick chunks properly", { prev = getOption("BatchJobs.verbose", TRUE) options(BatchJobs.verbose = TRUE) expect_message({ reg = batchMapQuick(identity, 1:4, chunk.size = 2, file.dir = getTempDir()) }, "Submitting 2 chunks / 4 jobs") options(BatchJobs.verbose = prev) waitForJobs(reg) }) BatchJobs/tests/testthat/test_loadResults.R0000644000176200001440000000346212571265761020617 0ustar liggesuserscontext("loadResults") test_that("loadResults", { reg = makeTestRegistry() ids = 1:2 batchMap(reg, identity, ids) submitJobs(reg) waitForJobs(reg) ys1 = 1:2 ys2 = loadResults(reg, simplify=TRUE, use.names="none") expect_equal(ys1, ys2) ys1 = as.list(ys1) ys2 = loadResults(reg, simplify=FALSE, use.names="none") expect_equal(ys1, ys2) names(ys1) = ids ys2 = loadResults(reg, simplify=FALSE, use.names="ids") expect_equal(ys1, ys2) ys1 = unlist(ys1) ys2 = loadResults(reg, simplify=TRUE, use.names="ids") expect_equal(ys1, ys2) ys2 = loadResults(reg, 2) expect_equal(list("2"=2), ys2) nl = list() names(nl) = character(0L) expect_equal(loadResults(reg, ids=integer(0), simplify=TRUE, use.names="ids"), nl) expect_equal(loadResults(reg, ids=integer(0), simplify=FALSE, use.names="ids"), nl) expect_equal(loadResults(reg, ids=integer(0), simplify=TRUE, use.names="none"), list()) expect_equal(loadResults(reg, ids=integer(0), simplify=FALSE, use.names="none"), list()) # test names of loadResults reg = makeTestRegistry() batchMap(reg, identity, letters, use.names=TRUE) submitJobs(reg) waitForJobs(reg) expect_equal(names(loadResults(reg, use.names = "ids")), as.character(1:26)) expect_equal(names(loadResults(reg, use.names = "names")), letters) }) test_that("impute.val works", { reg = makeTestRegistry() ids = 1:2 batchMap(reg, function(x) if (x > 1) stop("nope") else 0L, ids) submitJobs(reg) waitForJobs(reg) expect_equal(loadResults(reg, use.names= "none"), list(0)) expect_error(loadResults(reg, ids = 1:2, use.names= "none"), "do not exist") expect_equal(loadResults(reg, ids = 1:2, missing.ok = TRUE, use.names= "none"), list(0, NULL)) expect_equal(loadResults(reg, ids = 1:2, missing.ok = TRUE, impute.val = 1, use.names= "none"), list(0, 1)) }) BatchJobs/tests/testthat/test_submitJobs.R0000644000176200001440000000311512357424573020432 0ustar liggesuserscontext("submitJobs") test_that("submitJobs", { reg = makeTestRegistry() batchMap(reg, identity, 123) submitJobs(reg) waitForJobs(reg) y = loadResult(reg, 1L) expect_equal(y, 123) # check conversion reg = makeTestRegistry() batchMap(reg, identity, 1:2) submitJobs(reg) waitForJobs(reg) submitJobs(reg, 1:2) waitForJobs(reg) submitJobs(reg, c(1,2)) waitForJobs(reg) }) test_that("submitJobs works with empty id vector", { reg = makeTestRegistry() batchMap(reg, identity, 1:10) submitJobs(reg, ids = integer(0L)) waitForJobs(reg) expect_equal(findSubmitted(reg), integer(0L)) }) test_that("submitJobs works with multiple result files", { reg = makeTestRegistry(multiple.result.files=TRUE) # no list returned batchMap(reg, identity, 1) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1L) reg = makeTestRegistry(multiple.result.files=TRUE) f = function(x) list(a=x, b=2*x) ids = 1:2 batchMap(reg, f, ids) submitJobs(reg) waitForJobs(reg) expect_equal(findDone(reg), ids) ys = loadResults(reg, ids) expect_equal(ys, list("1"=list(a=1, b=2), "2"=list(a=2, b=4))) ys = loadResults(reg, 2, part="b") expect_equal(ys, list("2"=list(b=4))) }) test_that("submitJobs works with chunking", { reg = makeTestRegistry() batchMap(reg, identity, 1:5) ch = chunk(getJobIds(reg), chunk.size=2) submitJobs(reg, ids=ch[1:2]) waitForJobs(reg) expect_equal(findDone(reg), 1:4) submitJobs(reg, ids=ch) waitForJobs(reg) expect_equal(findDone(reg), 1:5) expect_equal(loadResults(reg, simplify=TRUE, use.names="none"), 1:5) }) BatchJobs/tests/testthat/test_resources.R0000644000176200001440000000243712430500247020312 0ustar liggesuserscontext("resources") test_that("resources", { # FIXME this is still broken for all major cluster system # because the resources are stupid! # use resources on slave reg = makeTestRegistry() batchMap(reg, function(i) getResources(), 1:2) res = list(walltime=5*60, memory=500) submitJobs(reg, resources=res) waitForJobs(reg) expect_equal(loadResult(reg, 1)[names(res)], res) # see FIXME in testJob.R # expect_equal(testJob(reg, 1, resources=res, external=FALSE)[names(res)], res) # query on master res1 = getJobResources(reg, 1)[[1]] res2 = getJobResources(reg, 2)[[1]] expect_equal(res1, res2) expect_equal(res1[names(res)], res) #submit only a few jobs resetJobs(reg, ids=1:2, force=TRUE) submitJobs(reg, ids=1, resources=res) waitForJobs(reg) expect_equal(loadResult(reg, 1)[names(res)], res) expect_error(getJobResources(reg, 1:2), "not been submitted") res1 = getJobResources(reg) res2 = getJobResources(reg, 1) expect_equal(res1, res2) # defaults in conf # conf = BatchJobs:::getBatchJobsConf() # conf$default.resources = list(walltime=1, memory=2, xxx=3) # reg = makeTestRegistry() # batchMap(reg, function(i) getResources(), 1) # expect_error(submitJobs(reg, resources=list(memory=200)), "Illegal resources used") # waitForJobs(reg) }) BatchJobs/tests/testthat/test_resetJobs.R0000644000176200001440000000071612357424573020255 0ustar liggesuserscontext("resetJobs") test_that("resetJobs", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) ids = getJobIds(reg) submitJobs(reg) waitForJobs(reg) done = findDone(reg) # test that nothing happens on empty id vector resetJobs(reg) resetJobs(reg, ids=integer(0L), force=TRUE) expect_equal(done, findDone(reg)) # now really reset the first job resetJobs(reg, done[1], force=TRUE) expect_equal(setdiff(done, findDone(reg)), 1) }) BatchJobs/tests/testthat/test_getLogFiles.R0000644000176200001440000000067112357424573020521 0ustar liggesuserscontext("getLogFiles") test_that("getLogFiles", { reg = makeTestRegistry() batchMap(reg, identity, 1:10) ids = getJobIds(reg) ch = chunk(ids[1:8], chunk.size=4) submitJobs(reg, ch) submitJobs(reg, ids[9:10]) waitForJobs(reg) fs = getLogFiles(reg, ids) expect_equal(length(unique(fs)), 4) expect_true(grepl("9", fs[9])) expect_true(grepl("10", fs[10])) expect_equal(getLogFiles(reg, integer(0L)), character(0L)) }) BatchJobs/tests/testthat/test_waitForJobs.R0000644000176200001440000000103612357424573020542 0ustar liggesuserscontext("waitForJobs") test_that("waitForJobs", { f = function(x) { if (x == 5) stop("x == 5") else x } reg = makeTestRegistry() batchMap(reg, f, 1:5) submitJobs(reg) waitForJobs(reg) expect_equal(waitForJobs(reg, 1:5, stop.on.error = FALSE), FALSE) expect_warning(waitForJobs(reg, 1:5, stop.on.error = TRUE)) expect_equal(suppressWarnings(waitForJobs(reg, 1:5, stop.on.error = TRUE)), FALSE) expect_equal(waitForJobs(reg, 1:4, stop.on.error=FALSE), TRUE) expect_equal(waitForJobs(reg, 1:4, stop.on.error=TRUE), TRUE) }) BatchJobs/tests/testthat/test_Registry.R0000644000176200001440000000176512623105424020115 0ustar liggesuserscontext("Registry") test_that("registry with read.only set works", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) reg$read.only = TRUE expect_error(batchMap(reg, function(x) x^2, 1:3), "read-only") expect_error(submitJobs(reg), "read-only") expect_error(removeRegistrySourceFiles(reg), "read-only") expect_error(killJobs(reg), "read-only") expect_error(removeRegistryPackages(reg), "read-only") expect_error(resetJobs(reg), "read-only") expect_error(sweepRegistry(reg), "read-only") expect_data_frame(getStatus(reg)) expect_integer(findJobs(reg)) expect_list(getJobs(reg), types = "Job") expect_class(getJob(reg, 1), "Job") expect_true(sourceRegistryFiles(reg)) expect_numeric(reduceResultsVector(reg)) expect_numeric(reduceResults(reg, ids = 1:3, fun = function(job, res, aggr) c(aggr, res))) expect_numeric(filterResults(reg, fun = function(job, res) TRUE)) expect_numeric(testJob(reg, id = 1, external = FALSE)) }) BatchJobs/tests/testthat/test_batchMap.R0000644000176200001440000000231212413244720020011 0ustar liggesuserscontext("batchMap") test_that("batchMap", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, (1:3)^2, check.attributes = FALSE) reg = makeTestRegistry() batchMap(reg, function(x,y,z) (x+y)*z, 1:3, 4:6, more.args = list(z = 2)) submitJobs(reg) waitForJobs(reg) y = sapply(getJobIds(reg), function(id) loadResult(reg, id)) expect_equal(y, ((1:3) + (4:6))*2, check.attributes = FALSE) reg = makeTestRegistry() expect_equal(batchMap(reg, function(...) 1), integer(0L)) expect_equal(batchMap(reg, function(...) 1, i = integer(0L)), integer(0L)) reg = makeTestRegistry() expect_equal(batchMap(reg, function(...) 1, a = 1:3, b = 1:3, use.names = TRUE), 1:3) # factors reg = makeTestRegistry() ids = batchMap(reg, identity, factor(letters[1:5])) submitJobs(reg) waitForJobs(reg) expect_equal(loadResults(reg, simplify = TRUE), setNames(factor(letters[1:5]), 1:5)) # primitive functions reg = makeTestRegistry() ids = batchMap(reg, sin, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(as.numeric(loadResults(reg, simplify = TRUE)), sin(1:2)) }) BatchJobs/tests/testthat/test_setJobFunction.R0000644000176200001440000000065712357424573021255 0ustar liggesuserscontext("setJobFunction") test_that("setJobFunction", { f = function(x) if(x == 2) stop(2) else x reg = makeTestRegistry() batchMap(reg, f, x = 1:3) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 2L) expect_equal(findDone(reg), c(1L, 3L)) f = function(x) x setJobFunction(reg, fun=f, ids=2L, reset=TRUE, force=TRUE) submitJobs(reg, 2L) waitForJobs(reg) expect_equal(findDone(reg), 1:3) }) BatchJobs/tests/testthat/test_findJobs.R0000644000176200001440000000227712357424573020057 0ustar liggesuserscontext("findJobs") test_that("findJobs", { reg = makeTestRegistry() batchMap(reg, function(x) x, 10:13) ids = findJobs(reg, pars=.arg1 > 10) expect_equal(ids, 2:4) reg = makeTestRegistry() batchMap(reg, function(x) x, x=10:13) ids = findJobs(reg, pars=x > 10) expect_equal(ids, 2:4) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x*y, x=1:2, y=10:11) ids = findJobs(reg, pars=x == 2) expect_true(length(ids) == 2) ids = findJobs(reg, pars=x == 1 && y == 11) expect_true(length(ids) == 1) reg = makeTestRegistry() xi = 3 batchExpandGrid(reg, function(x, y) x+y, x=1:4, y=1:4) expect_equal(findJobs(reg, pars = (x == y)), c(1, 6, 11, 16)) expect_equal(findJobs(reg, pars = (x == xi)), c(3, 7, 11, 15)) }) test_that("findJobs with names", { reg = makeTestRegistry() batchMap(reg, identity, letters[1:5], use.names=TRUE) ids = findJobs(reg, jobnames = c("b", "c", "d")) expect_equal(ids, 2:4) submitJobs(reg, ids) waitForJobs(reg) expect_equal(names(loadResults(reg, ids, use.names = "names")), c("b", "c", "d")) setJobNames(reg, 1:5, sprintf("x%i", 1:5)) expect_equal(names(loadResults(reg, ids, use.names = "names")), c("x2", "x3", "x4")) }) BatchJobs/tests/testthat/test_showStatus.R0000644000176200001440000000153712357424573020503 0ustar liggesuserscontext("showStatus") test_that("showStatus", { reg = makeTestRegistry() f = function(x) { if (x < 3) stop() else { Sys.sleep(2) x } } batchMap(reg, f, 1:3) submitJobs(reg) waitForJobs(reg) expect_output({ showStatus(reg) }, "Status for 3 jobs") expect_output({ showStatus(reg, getJobIds(reg)[2:3]) }, "Status for 2 jobs") }) test_that("showStatus only shows some errors", { reg = makeTestRegistry() batchMap(reg, function(x) stop(), 1:3) submitJobs(reg) waitForJobs(reg) expect_output({ showStatus(reg, errors=1) }, "Showing first 1 errors:") }) test_that("showStatus works with empty id vector", { reg = makeTestRegistry() batchMap(reg, function(x) stop(), 1:3) submitJobs(reg) waitForJobs(reg) expect_output({ showStatus(reg, integer(0L)) }, "Status for 0 jobs") }) BatchJobs/tests/testthat/test_batchReduce.R0000644000176200001440000000134312357424573020523 0ustar liggesuserscontext("batchReduce") test_that("batchReduce", { reg = makeTestRegistry() ids = batchReduce(reg, function(aggr, x) aggr+x, 1:20, init=0, block.size=10) submitJobs(reg) waitForJobs(reg) y = reduceResults(reg, fun=function(aggr, job, res) aggr+res, init=0) expect_equal(y, sum(1:20)) expect_equal(ids, 1:2) reg = makeTestRegistry() ids = batchReduce(reg, function(aggr, x, y) aggr+x+y, 1:20, init=0, block.size=10, more.args=list(y=1)) submitJobs(reg) waitForJobs(reg) y = reduceResults(reg, fun=function(aggr, job, res) aggr+res, init=0) expect_equal(y, sum(1:20)+20) reg = makeTestRegistry() expect_equal(batchReduce(reg, function(aggr, x) aggr+x, integer(0L), init=0, block.size=10), integer(0L)) }) BatchJobs/tests/testthat/test_findStatus.R0000644000176200001440000000063112357424573020435 0ustar liggesuserscontext("findStatus") test_that("findStatus", { reg = makeTestRegistry() f = function(i) { if (i == 2) stop(123) } batchMap(reg, f, 1:3) ids = getJobIds(reg) submitJobs(reg) waitForJobs(reg) j = findDone(reg) expect_equal(j, ids[-2]) j = findErrors(reg) expect_equal(j, ids[2]) j = findNotDone(reg) expect_equal(j, ids[2]) j = findStarted(reg) expect_equal(j, 1:3) }) BatchJobs/tests/testthat/test_batchMapResults.R0000644000176200001440000000163212357424573021414 0ustar liggesuserscontext("batchMapResults") test_that("batchMapResults", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:3) submitJobs(reg) waitForJobs(reg) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir = td) batchMapResults(reg, reg2, fun=function(job, res, y, z) (res+y)*z, 4:6, more.args=list(z=2)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResults(reg2, use.names="none", simplify=TRUE), ((1:3)^2 + (4:6))*2) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir = td) ids = getJobIds(reg) batchMapResults(reg, reg2, function(job, res, y, z) (res+y)*z, 5:6, ids=ids[2:3], more.args=list(z=2)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResults(reg2, use.names="none", simplify=TRUE), ((2:3)^2 + (5:6))*2) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir = td) batchMapResults(reg, reg2, function(job, res, ...) res, ids = integer(0L)) }) BatchJobs/tests/testthat/test_batchReduceResults.R0000644000176200001440000000220012357424573022076 0ustar liggesuserscontext("batchReduceResults") test_that("batchReduceResults", { reg = makeTestRegistry() batchMap(reg, function(x) x^2, 1:5) submitJobs(reg) waitForJobs(reg) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir=td) batchReduceResults(reg, reg2, block.size=3, init=c(), fun=function(aggr, job, res) c(aggr, res)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResult(reg2, getJobIds(reg)[1]), (1:3)^2, check.names=FALSE) expect_equal(loadResult(reg2, getJobIds(reg)[2]), (4:5)^2, check.names=FALSE) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir=td) ids = getJobIds(reg) batchReduceResults(reg, reg2, ids=ids[2:4], block.size=2, init=c(), fun=function(aggr, job, res) c(aggr, res)) submitJobs(reg2) waitForJobs(reg2) expect_equal(loadResult(reg2, getJobIds(reg)[1]), (2:3)^2, check.names=FALSE) expect_equal(loadResult(reg2, getJobIds(reg)[2]), (4)^2, check.names=FALSE) td = getTempDir() reg2 = makeRegistry(id="foo", file.dir=td) expect_equal(batchReduceResults(reg, reg2, ids = integer(0L), block.size=2, init=c(), fun=function(aggr, job, res) c(aggr, res)), integer(0L)) }) BatchJobs/tests/testthat/test_sourceRegistryFiles.R0000644000176200001440000000672712413244720022324 0ustar liggesuserscontext("source registry files") test_that("source registry files", { mywd = file.path(getWorkDir(), "myworkdir") dir.create(mywd, recursive = TRUE, showWarnings = FALSE) setwd(mywd) src.dir.subdir = c(file.path(mywd, "unittest-sources-subdir"), "unittest-sources-subdir-relative") x = lapply(src.dir.subdir, dir.create, recursive = TRUE, showWarnings = FALSE) src.dir.parent = c(file.path(dirname(mywd), "unittest-sources-parent"), "../unittest-sources-parent-relative") x = lapply(src.dir.parent, dir.create, recursive = TRUE, showWarnings = FALSE) for (src.dir in c(src.dir.subdir, src.dir.parent)) { cat("xxx = 123", file = file.path(src.dir, "test.R")) reg = makeTestRegistry(src.dir = src.dir, work.dir = mywd) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) loadRegistry(reg$file.dir) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) batchMap(reg, function(i) i + xxx, i = 1:3) submitJobs(reg) waitForJobs(reg) res = loadResults(reg, simplify = TRUE, use.names = "none") expect_equal(res, 123 + 1:3) } src.files.subdir = file.path(src.dir.subdir, "test.R") src.files.parent = file.path(src.dir.parent, "test.R") for (src.files in c(src.files.subdir, src.files.parent)) { reg = makeTestRegistry(src.files = src.files, work.dir = mywd) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) loadRegistry(reg$file.dir) expect_true(exists("xxx", envir = .GlobalEnv)) rm(list = "xxx", envir = .GlobalEnv) batchMap(reg, function(i) i + xxx, i = 1:3) submitJobs(reg) waitForJobs(reg) res = loadResults(reg, simplify = TRUE, use.names = "none") expect_equal(res, 123 + 1:3) } }) test_that("source registry mutators work", { mywd = file.path(getWorkDir(), "myworkdir") dir.create(mywd, recursive = TRUE, showWarnings = FALSE) setwd(mywd) src.dir.subdir = c(file.path(mywd, "unittest-sources-subdir"), "unittest-sources-subdir-relative") x = lapply(src.dir.subdir, dir.create, recursive = TRUE, showWarnings = FALSE) src.dir.parent = c(file.path(dirname(mywd), "unittest-sources-parent"), "../unittest-sources-parent-relative") x = lapply(src.dir.parent, dir.create, recursive = TRUE, showWarnings = FALSE) src.files.subdir = file.path(src.dir.subdir, "test.R") src.files.parent = file.path(src.dir.parent, "test.R") reg = makeTestRegistry() expect_equal(reg$src.files, character(0L)) expect_equal(reg$src.dirs, character(0L)) reg = addRegistrySourceFiles(reg, src.files.subdir, src.now = FALSE) expect_equal(reg$src.files, sanitizePath(src.files.subdir, make.absolute = FALSE)) expect_equal(reg$src.dirs, character(0L)) reg = addRegistrySourceFiles(reg, src.files.subdir, src.now = FALSE) expect_equal(reg$src.files, sanitizePath(src.files.subdir, make.absolute = FALSE)) expect_equal(reg$src.dirs, character(0L)) reg = addRegistrySourceDirs(reg, src.dir.parent, src.now = FALSE) expect_equal(reg$src.files, sanitizePath(src.files.subdir, make.absolute = FALSE)) expect_equal(reg$src.dirs, sanitizePath(src.dir.parent, make.absolute = FALSE)) reg = removeRegistrySourceFiles(reg, reg$src.files) expect_equal(reg$src.files, character(0L)) expect_equal(reg$src.dirs, sanitizePath(src.dir.parent, make.absolute = FALSE)) reg = removeRegistrySourceDirs(reg, reg$src.dirs) expect_equal(reg$src.files, character(0L)) expect_equal(reg$src.dirs, character(0L)) }) BatchJobs/tests/testthat/test_zzz_cleanup.R0000644000176200001440000000012712357424573020655 0ustar liggesuserscontext("cleanup") test_that("Removing unittest files", { expect_true(cleanup()) }) BatchJobs/tests/testthat/test_getJob.R0000644000176200001440000000165412357424573017531 0ustar liggesuserscontext("getJob") test_that("getJob", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) ids = getJobIds(reg) j = getJob(reg, ids[2]) expect_equal(j$id, ids[2]) expect_equal(j$pars, list(2)) expect_is(j, "Job") js = getJobs(reg, ids[2:3]) expect_is(js, "list") expect_equal(length(js), 2) expect_is(js[[1]], "Job") expect_is(js[[2]], "Job") expect_equal(js[[1]]$id, ids[2]) expect_equal(js[[2]]$id, ids[3]) expect_equal(js[[1]]$pars, list(2)) expect_equal(js[[2]]$pars, list(3)) # empty ids expect_equal(getJobs(reg, integer(0)), list()) # bad id expect_error(getJob(reg, "xxx"), "integer") expect_error(getJobs(reg, -1), "Ids not present in registry: -1") }) test_that("getJob returns jobs in same order as ids", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) ids = getJobIds(reg) ids = rev(ids) js = getJobs(reg, ids) expect_equal(ids, extractSubList(js, "id")) }) BatchJobs/tests/testthat/test_database.R0000644000176200001440000000274212357424573020062 0ustar liggesuserscontext("database") test_that("database", { err.msg = "error with 'quotes'" reg = makeTestRegistry() batchMap(reg, identity, 1) BatchJobs:::dbSendMessage(reg, BatchJobs:::dbMakeMessageError(reg, 1, err.msg=err.msg)) expect_equal(unname(getErrorMessages(reg)), "error with \"quotes\"") }) test_that("dbSelectWithIds works", { reg = makeTestRegistry() batchMap(reg, identity, 1:3) query = sprintf("SELECT job_id from %s_job_status", reg$id) jids = BatchJobs:::dbSelectWithIds(reg, query, where=TRUE, limit=2)$job_id expect_equal(jids, 1:2) submitJobs(reg, 1:2) waitForJobs(reg) query = sprintf("SELECT job_id from %s_job_status where submitted IS NOT NULL", reg$id) jids = BatchJobs:::dbSelectWithIds(reg, query, where=FALSE)$job_id expect_equal(jids, 1:2) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=1:3, where=FALSE)$job_id expect_equal(jids, 1:2) jids = BatchJobs:::dbSelectWithIds(reg, query, where=FALSE, limit=1)$job_id expect_equal(jids, 1) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=1:3, where=FALSE, limit=1)$job_id expect_equal(jids, 1) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=2:3, where=FALSE)$job_id expect_equal(jids, 2) reg = makeTestRegistry() batchMap(reg, identity, 1:4) submitJobs(reg, 3:4) waitForJobs(reg) query = sprintf("SELECT job_id from %s_job_status where submitted IS NOT NULL", reg$id) jids = BatchJobs:::dbSelectWithIds(reg, query, ids=1:4, where=FALSE,)$job_id expect_equal(jids, 3:4) }) BatchJobs/tests/testthat/test_sqlquotes.R0000644000176200001440000000107212357424573020351 0ustar liggesuserscontext("sqlquotes") test_that("sqlquotes", { reg = makeTestRegistry() f = function(x) stop("Quoting is 'on'") batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1:2) reg = makeTestRegistry() f = function(x) stop('Quoting is "on"') batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1:2) reg = makeTestRegistry() f = function(x) stop("Some special chars: \n{}\r\t(),;") batchMap(reg, f, 1:2) submitJobs(reg) waitForJobs(reg) expect_equal(findErrors(reg), 1:2) }) BatchJobs/tests/testthat/test_batchExpandGrid.R0000644000176200001440000000211412357424573021336 0ustar liggesuserscontext("batchExpandGrid") test_that("batchExpandGrid", { reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x*y, 1:3, 5) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, (1:3) * 5) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x*y, 1:3, 2:3) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(sum(y), sum(outer(1:3, 2:3))) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y,z) x*y+z, 1:3, 5, more.args=list(z=10)) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, ((1:3) * 5) + 10) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) paste(x,y), 1, c("a", "b")) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, c("1 a", "1 b")) reg = makeTestRegistry() batchExpandGrid(reg, function(x,y) x^y, y=2, x=1:3) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg, use.names="none") expect_equal(y, (1:3)^2) }) BatchJobs/tests/testthat/test_packages.R0000644000176200001440000000214312623105424020052 0ustar liggesuserscontext("packages") test_that("packages", { reg = makeTestRegistry(packages = "MASS") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS"))) reg = addRegistryPackages(reg, "testthat") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS", "testthat"))) reg = addRegistryPackages(reg, "checkmate") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS", "testthat", "checkmate"))) reg = removeRegistryPackages(reg, "testthat") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS", "checkmate"))) reg = removeRegistryPackages(reg, "MASS") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "checkmate"))) reg = setRegistryPackages(reg, c("MASS", "checkmate")) expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS", "checkmate"))) reg = addRegistryPackages(reg, "BatchJobs") expect_true(is.list(reg$packages) && setequal(names(reg$packages), c("BatchJobs", "MASS", "checkmate"))) }) BatchJobs/tests/testthat/test_makeRegistry.R0000644000176200001440000000242012527173323020746 0ustar liggesuserscontext("makeRegistry") test_that("makeRegistry", { reg = makeTestRegistry() expect_true(inherits(reg, "Registry")) expect_true(is.list(reg)) expect_true(file.exists(reg$file.dir)) expect_true(file.exists(file.path(reg$file.dir, "BatchJobs.db"))) expect_output(print(reg), "Job registry") df = BatchJobs:::dbGetJobStatusTable(reg) expect_true(is.data.frame(df) && nrow(df) == 0L && ncol(df) == 14L) }) test_that("makeRegistry checks id", { expect_error(makeRegistry(id="runit-files"), "comply with") expect_error(makeRegistry(id="-files"), "comply with") }) test_that("makeRegistry checks packs", { makeTestRegistry(packages="base") expect_error(makeTestRegistry(packages="foo"), "Please install the following packages: foo") }) test_that("loadRegistry works", { reg1 = makeTestRegistry() reg2 = loadRegistry(reg1$file.dir) expect_is(reg2, "Registry") }) test_that("torturing makeRegistry/removeRegistry to create registries over and over works", { ## This fails (at least) on Windows if we set the working directory ## to be the current working directory, i.e. ".". This can be achieve ## using Sys.setenv("R_EXPENSIVE_EXAMPLE_OK"=TRUE). for (ii in 1:50) { reg = makeTestRegistry() expect_true(removeRegistry(reg, ask = "no")) } }) BatchJobs/tests/testthat/test_grepLogs.R0000644000176200001440000000070012357424573020070 0ustar liggesusers# FIXME: can we find a way to generate log files in unit tests? context("grepLogs") test_that("grepLogs", { #.... no logs for unittests :( # reg = makeTestRegistry() # f = function(x) { # if (x %% 2 == 0) warning(x) else x # } # batchMap(reg, f, 1:10) # submitJobs(reg) # expect_equal(grepLogs(reg), 1:5*2L) # expect_equal(grepLogs(reg, ids=1:2), 2L) # expect_equal(grepLogs(reg, pattern = "xxx_nomatch_xxx"), integer(0)) }) BatchJobs/tests/testthat/test_doJob.R0000644000176200001440000000325412705156101017334 0ustar liggesuserscontext("doJob") test_that("doJob", { reg = makeTestRegistry() id = 1L batchMap(reg, identity, 123) df = BatchJobs:::dbGetJobStatusTable(reg) expect_true(is.data.frame(df) && nrow(df) == 1L && ncol(df) == 14L) ids = findNotDone(reg) expect_equal(ids, id) BatchJobs:::saveConf(reg) expect_output({ y = BatchJobs:::doJob(reg, id, multiple.result.files=FALSE, disable.mail=TRUE, last=id, array.id=NA_integer_, staged = FALSE) }, "BatchJobs job") waitForJobs(reg) expect_equal(y, TRUE) expect_equal(loadResult(reg, 1L), 123) df = BatchJobs:::dbGetJobStatusTable(reg) expect_true(!is.na(df$started) && !is.na(df$done) && is.na(df$error)) y = loadResult(reg, id) expect_equal(y, 123) ids = findNotDone(reg) expect_equal(length(ids), 0) # test working directory reg = makeTestRegistry() wd.now = getwd() wd.job = reg$work.dir bar = 123 save(bar=bar, file=file.path(wd.job, "foo.RData")) reg$work.dir = wd.job f = function(x) { load("foo.RData") bar+x } batchMap(reg, f, 1) BatchJobs:::saveConf(reg) expect_output({ y = BatchJobs:::doJob(reg, id, multiple.result.files=FALSE, disable.mail=TRUE, last=id, array.id=NA_integer_, staged = FALSE) }, "BatchJobs job") expect_equal(y, TRUE) expect_equal(loadResult(reg, 1L), bar + 1) expect_equal(getwd(), wd.now) unlink(file.path(wd.job, "foo.RData")) # test packages # be sneaky otherwise we get error here due to pack check reg = makeTestRegistry() reg$packages = list(foo="foo") BatchJobs:::saveRegistry(reg) batchMap(reg, identity, 1) expect_error(suppressAll(testJob(reg, 1)), "Please install the following packages: foo") expect_equal(findNotDone(reg), id) }) BatchJobs/NAMESPACE0000644000176200001440000001115513207276646013350 0ustar liggesusers# Generated by roxygen2: do not edit by hand S3method(applyJobFunction,Registry) S3method(copyRequiredJobFiles,Registry) S3method(dbCreateJobDefTable,Registry) S3method(dbGetJobs,Registry) S3method(getJobInfo,Registry) S3method(getJobs,Registry) S3method(getWorkerNumberOfCPUs,WorkerLinux) S3method(getWorkerStatus,WorkerLinux) S3method(killWorkerJob,WorkerLinux) S3method(listWorkerJobs,WorkerLinux) S3method(print,ClusterFunctions) S3method(print,Config) S3method(print,Job) S3method(print,Registry) S3method(print,SubmitJobResult) S3method(setJobFunction,Registry) S3method(startWorkerJob,WorkerLinux) S3method(updateRegistry,Registry) export(addRegistryPackages) export(addRegistrySourceDirs) export(addRegistrySourceFiles) export(applyJobFunction) export(batchExpandGrid) export(batchExport) export(batchMap) export(batchMapQuick) export(batchMapResults) export(batchQuery) export(batchReduce) export(batchReduceResults) export(batchUnexport) export(callFunctionOnSSHWorkers) export(cfBrewTemplate) export(cfHandleUnknownSubmitError) export(cfKillBatchJob) export(cfReadBrewTemplate) export(checkIds) export(copyRequiredJobFiles) export(dbCreateJobDefTable) export(dbGetJobs) export(debugMulticore) export(debugSSH) export(filterResults) export(findDisappeared) export(findDone) export(findErrors) export(findExpired) export(findJobs) export(findMissingResults) export(findNotDone) export(findNotErrors) export(findNotOnSystem) export(findNotRunning) export(findNotStarted) export(findNotSubmitted) export(findNotTerminated) export(findOnSystem) export(findRunning) export(findStarted) export(findSubmitted) export(findTerminated) export(getConfig) export(getErrorMessages) export(getJob) export(getJobIds) export(getJobInfo) export(getJobLocation) export(getJobNr) export(getJobParamDf) export(getJobResources) export(getJobs) export(getLogFiles) export(getResources) export(getSSHWorkersInfo) export(getStatus) export(grepLogs) export(installPackagesOnSSHWorkers) export(killJobs) export(loadConfig) export(loadExports) export(loadRegistry) export(loadResult) export(loadResults) export(makeClusterFunctions) export(makeClusterFunctionsInteractive) export(makeClusterFunctionsLSF) export(makeClusterFunctionsLocal) export(makeClusterFunctionsMulticore) export(makeClusterFunctionsOpenLava) export(makeClusterFunctionsSGE) export(makeClusterFunctionsSLURM) export(makeClusterFunctionsSSH) export(makeClusterFunctionsTorque) export(makeJob) export(makeRegistry) export(makeSSHWorker) export(makeSubmitJobResult) export(reduceResults) export(reduceResultsDataFrame) export(reduceResultsDataTable) export(reduceResultsList) export(reduceResultsMatrix) export(reduceResultsVector) export(removeRegistry) export(removeRegistryPackages) export(removeRegistrySourceDirs) export(removeRegistrySourceFiles) export(resetJobs) export(sanitizePath) export(setConfig) export(setJobFunction) export(setJobNames) export(setRegistryPackages) export(showClusterStatus) export(showLog) export(showStatus) export(sourceRegistryFiles) export(submitJobs) export(sweepRegistry) export(syncRegistry) export(testJob) export(updateRegistry) export(waitForJobs) import(DBI) import(RSQLite) import(checkmate) import(data.table) import(methods) import(stats) import(utils) importFrom(BBmisc,"%nin%") importFrom(BBmisc,catf) importFrom(BBmisc,checkListElementClass) importFrom(BBmisc,chunk) importFrom(BBmisc,clipString) importFrom(BBmisc,collapse) importFrom(BBmisc,convertListOfRowsToDataFrame) importFrom(BBmisc,convertToShortString) importFrom(BBmisc,dropNamed) importFrom(BBmisc,extractSubList) importFrom(BBmisc,filterNull) importFrom(BBmisc,insert) importFrom(BBmisc,is.error) importFrom(BBmisc,isDirectory) importFrom(BBmisc,isProperlyNamed) importFrom(BBmisc,isScalarNA) importFrom(BBmisc,isWindows) importFrom(BBmisc,load2) importFrom(BBmisc,lsort) importFrom(BBmisc,makeFileCache) importFrom(BBmisc,makeProgressBar) importFrom(BBmisc,makeSimpleFileLogger) importFrom(BBmisc,messagef) importFrom(BBmisc,namedList) importFrom(BBmisc,names2) importFrom(BBmisc,requirePackages) importFrom(BBmisc,save2) importFrom(BBmisc,seq_col) importFrom(BBmisc,seq_row) importFrom(BBmisc,setClasses) importFrom(BBmisc,setColNames) importFrom(BBmisc,setRowNames) importFrom(BBmisc,stopf) importFrom(BBmisc,suppressAll) importFrom(BBmisc,system3) importFrom(BBmisc,vcapply) importFrom(BBmisc,viapply) importFrom(BBmisc,vlapply) importFrom(BBmisc,vnapply) importFrom(BBmisc,warningf) importFrom(BBmisc,which.first) importFrom(brew,brew) importFrom(digest,digest) importFrom(sendmailR,sendmail) importFrom(stringi,stri_extract_first_regex) importFrom(stringi,stri_split_fixed) importFrom(stringi,stri_split_regex) importFrom(stringi,stri_trim_both) BatchJobs/NEWS0000644000176200001440000001634012623105424012613 0ustar liggesusersBatchJobs_1.7: - Configuration files are now sourced while loading the namespace, not while attaching the package. - Added OpenLava support - getJobInfo() now also displays time.running until an error occurred in the job, and not NA as before - loadRegisty() now returns a read-only registry if a heuristic detects that the location of the registy changed and the new argument "adjust.paths" is not explicitly set to TRUE. - Fixed a bug where time.started was incorrectly stored if chunking was used. - Packages are now loaded in user provided order instead of alphabetically. - showClusterStatus now returns its information visibly and does not print it per default - The 'max.load' setting for multicore and SSH is now set to a more aggressive default. - New argument 'impute.val' for loadResult() and loadResults() - New config options: "measure.mem" to disable memory measurements on the slave - SSH: makeSSHWorker, debugSSH: ssh.cmd and ssh.args were added to customize the remote login - New functions: -- setRegistryPackages() -- reduceResultsDataTable() -- getJobLocation() -- batchQuery() BatchJobs_1.6: - Added functions: getStatus, removeRegistry, findDisappeared - All find* function now support a 'limit' argument (default unlimited) - We are less strict in arg checks of makeSSHWorker, so you can set a load higher than ncpus - loadExports: added argument "what" - batchApply now works with arrays - filehandling for the registry, especially on Windows, was improved (Thx Henrik Bengtsson) - You can set pragmas via db.options in your config file. - The SQLite pragma "busy_timeout" is now set per default with a value of 5000ms. - The default number of cores is now retrieved from the option "mc.cores" rather - New configuration option BatchJobs.load.config - Fixed some minor things. BatchJobs_1.5: - Updated to use RSQLite >=1.0.0 - Fixed a bug in waitForJobs(). - Fixed a regression parsing array job ids. - Argument 'progressbar' added to multiple functions to suppress displaying the progress bars. - Configuration option 'staged.queries' is now enabled per default. BatchJobs_1.4: - Added functions: batchExport, batchUnexport, loadExports - Added functions: addRegistryPackages, removeRegistryPackages - Added functions: addRegistrySourceFiles, addRegistrySourceDirs, removeRegistrySourceFiles, removeRegistrySourceDirs - Added functions: getJobParamDf - New option: "BatchJobs.clear.function.env". If TRUE the function environment will be erased prior to writing it to the file system. Otherwise a warning is issued if the environment exceeds 10Mb in size. - version 1.3 introduced a mini bug in the SGE interface (via new list.jobs.cmd option). Fixed. - Memory usage is now stored in the database and can be queried using getJobInfo(). BatchJobs_1.3: - Reduced memory consumption and disk i/o if 'more.args' is used. - added option in Registry for absolute paths src.files and src.dirs - Added option "BatchJobs.verbose" for report generators like knitr - added "file.dir" option to batchMapQuick, removed option 'temporary' as this is now more flexible. - package 'methods' is always loaded on slaved to avoid problems with S4 code - some smaller fixes - in makeClusterFunctions* it is now to configurable via what command jobs are listed on the system - default for BatchJobs.check.posix is now: OS != Windows BatchJobs_1.2: - Fixed a bug where too many running jobs were queried on some Torque systems - Fixed a bug where chunking did not work with batchMapQuick - Fixed mail subjects to be more precise on job IDs - Fixed output of running time in testJob() - More robust error handling in syncRegistry - waitForJobs is now more conservative in waiting for termination of jobs. - New argument "impute.val" for reduceResults-family of functions: This allows to conveniently reduce results while some are missing - Experimental framework to support sourcing of multiple files and directories - testJob can now run jobs interactively in the current R session if argument "external" is set to TRUE. BatchJobs_1.1-1135: - Increased minor version number - Added support for SLURM schedulers - Function getErrors renamed to getErrorMessages due to conflicts with RUnit. Interface change: argument "print" dropped - Redefined return value of function "waitForJobs" which should now be much easier to understand. - New option "BatchJobs.check.posix" to enable more relaxed checks on file and directory names - Helper functions to simplify integration into other packages: getConfig, loadConfig and setConfig. showConf was replaced with a print generic. - Improved stability and speed of killJobs - Added a workaround to fix file URI handling on windows - Many minor bug and documentation fixes - The registry now supports an "exports" directory of exported objects, which are always loaded on the slaves, e.g. data objects which are needed in every job This directory can be either managed manually or - more conveniently - via the "fail" package on CRAN. - The registry now supports a list of source directories. All R files in these subdirectories are sourced on the slaves (and also on the master) Nice if you have a lot of helper code in seperate R files which is needed in your jobs. - batchMap and all result functions where this is reasonable support naming results with the names of the elements that were mapped over (see use.names argument) - batchMap now supports mapping over any kind of object (not only lists and vectors) that supports the "length" and "[" / "[["- index method. BatchJobs_1.0-966: - Dropped support for R < 2.13.0 (.find.packages deprecated) - speedups in reduceResults[ReturnValue] - Output of showStatus now contains a time information - fixed a bug in findErrors (argument 'ids' was ignored) - New function sweepRegistry: Remove intermediate/temporary/obsolete files - Better error handling in case of I/O errors - New functions: findStarted() and findNotStarted() BatchJobs_1.0-915: - new option to raise warnings to errors on the slaves - option to set process priority via nice in clusterFunctionsMulticore and clusterFunctionsSSH - new helper functions: getErrors, getJobInfo, loadConfig, callFunctionOnSSHWorkers, getSSHWorkersInfo, installPackagesOnSSHWorkers, waitForJobs, several more find* functions - removed helper getJobTimes -> this is included in getJobInfo - experimental file-based caching mechanism to speed up the processing of queries generated on computational nodes - various smaller fixes - documentation fixes BatchJobs_1.0-606: - many dependencies are now imports - many fixes and improvements for stability in cluster functions - helper functions for your own cluster functions are exported and documented, they all start with cf* - interactive mode now has dummy functions for listing and killing jobs - interactive mode now generates log files - job resources (from submits) are stored and can be queried on the master and the slaves (see getResources and getJobResources) - defaults for job resources can be specified in config file, see config part on web page - multicore and SSH mode now supports r.options, i.e. options to start Rscript and R CMD BATCH with - function grepLogs to search log files for a pattern BatchJobs_1.0-527: - very minor bug fixes: for argument conversion and checks BatchJobs_1.0-485: - First submit to CRAN. BatchJobs/R/0000755000176200001440000000000013207301764012315 5ustar liggesusersBatchJobs/R/getJobInfo.R0000644000176200001440000001035112623105424014462 0ustar liggesusersgetJobInfoInternal = function(reg, ids, select, unit = "seconds", columns) { if (!missing(ids)) ids = checkIds(reg, ids) assertChoice(unit, c("seconds", "minutes", "hours", "days", "weeks")) select.db = c("submitted", "started", "done", "CASE WHEN error IS NULL THEN done - started ELSE error_time - started END AS time_running", "memory", "started - submitted AS time_queued", "error", "error_time", "node", "batch_job_id", "r_pid", "seed") select.cns = c("time.submitted", "time.started", "time.done", "time.running", "memory", "time.queued", "error.msg", "time.error", "nodename", "batch.id", "r.pid", "seed") columns = c(columns, setNames(select.db, select.cns)) if (!missing(select)) { assertSubset(select, c("id", select.cns)) columns = columns[names(columns) %in% c("id", select)] } tab = setNames(dbGetExpandedJobsTable(reg, ids, columns), names(columns)) if (nrow(tab) == 0L) return(tab) # convert times to POSIX if (!is.null(tab$time.submitted)) tab$time.submitted = dbConvertNumericToPOSIXct(tab$time.submitted) if (!is.null(tab$time.started)) tab$time.started = dbConvertNumericToPOSIXct(tab$time.started) if (!is.null(tab$time.done)) tab$time.done = dbConvertNumericToPOSIXct(tab$time.done) if (!is.null(tab$time.error)) tab$time.error = dbConvertNumericToPOSIXct(tab$time.error) # shorten error messages if (!is.null(tab$error.msg)) tab$error.msg = vcapply(tab$error.msg, clipString, len = 30L) # convert time diffs div = setNames(c(1L, 60L, 3600L, 86400L, 604800L), c("seconds", "minutes", "hours", "days", "weeks"))[unit] if (!is.null(tab$time.running)) tab$time.running = as.numeric(tab$time.running) / div if (!is.null(tab$time.queued)) tab$time.queued = as.numeric(tab$time.queued) / div return(tab) } #' @title Get computational information of jobs. #' #' @description #' Returns time stamps (submitted, started, done, error), #' time running, approximate memory usage (in Mb), #' error messages (shortened, see \code{\link{showLog}} for detailed error messages), #' time in queue, hostname of the host the job was executed, #' assigned batch ID, the R PID and the seed of the job. #' #' To estimate memory usage the sum of the last column of \code{\link[base]{gc}} is used. #' #' Column \dQuote{time.running} displays the time until either the job was done, or an error occured; #' it will by \code{NA} in case of time outs or hard R crashes. #' #' @template arg_reg #' @template arg_ids #' @param pars [\code{logical(1)}]\cr #' Include job parameters in the output? #' Default is \code{FALSE}. #' @param prefix.pars [\code{logical(1)}]\cr #' Should a prefix be added to job parameter names (column names) to avoid name clashes? #' Default is \code{FALSE}. #' @param select [\code{character}]\cr #' Select only a subset of columns. #' Usually this is not required and you can subset yourself, #' but in some rare cases it may be advantageous to not query all information. #' Note that the column \dQuote{id} (job id) is always selected. #' If not provided, all columns are queried and returned. #' @param unit [\code{character(1)}]\cr #' Unit to convert execution and queing times to. #' Possible values: \dQuote{seconds}, \dQuote{minutes}, \dQuote{hours}, #' \dQuote{days} and \dQuote{weeks}. #' Default is \dQuote{seconds}. #' @return [\code{data.frame}]. #' @family debug #' @export getJobInfo = function(reg, ids, pars = FALSE, prefix.pars = FALSE, select, unit = "seconds") { UseMethod("getJobInfo") } #' @method getJobInfo Registry #' @export getJobInfo.Registry = function(reg, ids, pars = FALSE, prefix.pars = FALSE, select, unit = "seconds") { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) assertFlag(pars) columns = c(id = "job_id") if (pars) columns = c(columns, c(pars = "pars")) tab = getJobInfoInternal(reg, ids = ids, select = select, unit = unit, columns = columns) if (pars && nrow(tab) > 0L && !is.null(tab$pars)) { pars = deserialize(tab$pars) if (prefix.pars) setnames(pars, sprintf("job.par.%s", names(pars))) tab = cbind(dropNamed(tab, "pars"), pars) } return(tab) } BatchJobs/R/buffer.R0000644000176200001440000000134112623105424013704 0ustar liggesusers# a simple preallocated stack. buffer = function(type = "list", capacity = 0L, value = TRUE, init = NULL, ...) { if (is.null(init)) { st = vector(type, capacity) n = 0L } else { st = init n = length(init) } rm(type) rm(init) force(capacity) force(value) ddd = list(...) clear = function() { if (is.function(value)) ret = do.call(value, c(list(head(st, n)), ddd)) else ret = value n <<- 0L ret } list( get = function() { head(st, n) }, clear = clear, push = function(x) { if (n == capacity) clear() n <<- n + 1L st[[n]] <<- x }, pos = function() { n }, empty = function() { n == 0L } ) } BatchJobs/R/clusterFunctionsInteractive.R0000644000176200001440000000416212413244720020207 0ustar liggesusers#' Create cluster functions for sequential execution in same session. #' #' @description #' All jobs executed under these cluster functions are executed #' sequentially, in the same interactive R process that you currently are. #' That is, \code{submitJob} does not return until the #' job has finished. The main use of this \code{ClusterFunctions} #' implementation is to test and debug programs on a local computer. #' #' Listing jobs returns an empty vector (as no jobs can be running when you call this) #' and \code{killJob} returns at once (for the same reason). #' #' @param write.logs [\code{logical(1)}]\cr #' Sink the output to log files. Turning logging off can increase the speed of #' calculations but makes it next to impossible to debug. #' Default is \code{TRUE}. #' @return [\code{\link{ClusterFunctions}}]. #' @family clusterFunctions #' @export makeClusterFunctionsInteractive = function(write.logs = TRUE) { assertFlag(write.logs) submitJob = if(write.logs) { function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { # open log file for writing fn = file(log.file, open = "wt") sink(fn, type = "output") sink(fn, type = "message") on.exit({ sink(NULL, type = "output") sink(NULL, type = "message") close(fn) }) # sink both output and message streams try(sys.source(rscript, envir = new.env(), keep.source = FALSE)) # return job result (always successful) makeSubmitJobResult(status = 0L, batch.job.id = "cfInteractive", msg = "") } } else { function(conf, reg, job.name, rscript, log.file, job.dir, resources) { suppressAll(try(sys.source(rscript, envir = new.env(), keep.source = FALSE))) makeSubmitJobResult(status = 0L, batch.job.id = "cfInteractive", msg = "") } } killJob = function(conf, reg, batch.job.id) NULL listJobs = function(conf, reg) integer(0L) getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "Interactive", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/clusterFunctions.R0000644000176200001440000001213612413244720016011 0ustar liggesusers#' Create a SubmitJobResult object. #' #' @description #' Use this function in your implementation of \code{\link{makeClusterFunctions}} #' to create a return value for the \code{submitJob} function. #' @param status [\code{integer(1)}]\cr #' Launch status of job. #' 0 means success, codes bewteen 1 and 100 are temporary #' errors and any error greater than 100 is a permanent failure. #' @param batch.job.id [\code{character(1)}]\cr #' Unique id of this job on batch system. Note that this is not the usual job id used in BatchJobs! #' Must be globally unique so that the job can be terminated #' using just this information. #' @param msg [\code{character(1)}]\cr #' Optional error message in case \code{status} is not equal to 0. #' Default is \dQuote{OK}, \dQuote{TEMPERR}, \dQuote{ERROR}, depending on \code{status}. #' @param ... [\code{any}]\cr #' Currently unused. #' @return [\code{\link{SubmitJobResult}}]. A list, containing #' \code{status}, \code{batch.job.id} and \code{msg}. #' @export #' @aliases SubmitJobResult makeSubmitJobResult = function(status, batch.job.id, msg, ...) { if (missing(msg)) { msg = if (status == 0L) "OK" else if (status <= 100L) "TEMPERR" else "ERROR" } setClasses(list(status = status, batch.job.id = batch.job.id, msg = msg), "SubmitJobResult") } #' @export #' @method print SubmitJobResult print.SubmitJobResult = function(x, ...) { cat("Job submission result:\n") catf(" ID : '%s'", x$batch.job.id) catf(" Status : %i", x$status) catf(" Msg : %s", x$msg) } #' Create a ClusterFuntions object. #' #' Use this funtion when you implement a backend for a batch system. #' You must define the functions specified in the arguments. #' @param name [\code{character(1)}]\cr #' Name of cluster functions. #' @param submitJob [\code{function(conf, reg, job.name, rscript, log.file, job.dir, resources, ...)}]\cr #' Function to submit a new job. #' Must return a \code{\link{SubmitJobResult}} object.\cr #' The arguments are:\cr #' conf [\code{environment}]: The user configuration.\cr #' reg [\code{\link{Registry}}]: The registry.\cr #' job.name [\code{character(1)}]: Name of job, used if the job is displayed on the batch system. This is just for display and not an id!\cr #' rscript [\code{character(1)}]: File path to R script that is used to execute the job.\cr #' log.file [\code{character(1)}]: File path where log file (.Rout) has to be placed.\cr #' job.dir [\code{character(1)}]: Directory where all files relating to this job are placed.\cr #' resources [\code{list}]: Freely definable list of required resources for this job, e.g. walltime or memory. #' @param killJob [\code{function(conf, reg, batch.job.id)}]\cr #' Function to kill a job on the batch system. #' Make sure that you definately kill the job! #' Return value is currently ignored.\cr #' The arguments are:\cr #' conf [\code{environment}]: The user configuration.\cr #' reg [\code{\link{Registry}}]: The registry.\cr #' batch.job.id [\code{character(1)}]: Batch job id, as produced by \code{submitJob}.\cr #' Set \code{killJob} to \code{NULL} if killing jobs cannot be supported. #' @param listJobs [\code{function(conf, reg)}]\cr #' List all jobs on the batch system for the current user / registry. #' This includes queued, running, held, idle, etc. jobs. #' Must return an integer vector of batch job ids, same format as they are produced by \code{submitJob}. #' It does not matter if you return a few job ids too many (e.g. all for the current user instead #' of all for the current registry), but you have to include all relevant ones. #' The arguments are:\cr #' conf [\code{environment}]: The user configuration.\cr #' reg [\code{\link{Registry}}]: The registry.\cr #' Set \code{listJobs} to \code{NULL} if listing jobs cannot be supported. #' @param getArrayEnvirName [\code{function()}]\cr #' Returns the name of the environment variable specifying the array ID. #' Should return \code{NA} if not supported. #' @param class [\code{character(1)}]\cr #' Optional class name for cluster functions object. #' Useful to provide a nice print method #' which might show additional information about the workers. #' Default is \code{NULL}. #' @param ... [\code{any}]\cr #' Currently ignored. #' @export #' @aliases ClusterFunctions #' @family clusterFunctions makeClusterFunctions = function(name, submitJob, killJob, listJobs, getArrayEnvirName, class = NULL, ...) { assertString(name) assertFunction(submitJob, c("conf", "reg", "job.name", "rscript", "log.file", "job.dir", "resources")) if (!is.null(killJob)) assertFunction(killJob, c("conf", "reg", "batch.job.id")) if (!is.null(listJobs)) assertFunction(listJobs, c("conf", "reg")) if (!is.null(getArrayEnvirName)) assertFunction(getArrayEnvirName, character(0L)) setClasses(list(name = name, submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName), c("ClusterFunctions", class)) } #' @export #' @method print ClusterFunctions print.ClusterFunctions = function(x, ...) { catf("%s cluster functions.", x$name) } BatchJobs/R/Registry.R0000644000176200001440000002143112623105424014245 0ustar liggesusersmakeRegistryInternal = function(id, file.dir, sharding, work.dir, multiple.result.files, seed, packages, src.dirs, src.files) { checkIdValid(id, allow.minus = FALSE) assertString(file.dir) checkDir(file.dir, create = TRUE, check.empty = TRUE, check.posix = TRUE, msg = TRUE) file.dir = sanitizePath(file.dir, make.absolute = TRUE) if (missing(work.dir)) work.dir = getwd() else assertString(work.dir) checkDir(work.dir, check.posix = TRUE) work.dir = sanitizePath(work.dir, make.absolute = TRUE) assertFlag(sharding) assertFlag(multiple.result.files) seed = if (missing(seed)) getRandomSeed() else asInt(seed) assertCharacter(packages, any.missing = FALSE) packages = unique(c("BatchJobs", packages)) requirePackages(packages, stop = TRUE, suppress.warnings = TRUE, default.method = "attach") assertCharacter(src.dirs, any.missing = FALSE) src.dirs = sanitizePath(src.dirs, make.absolute = FALSE) assertCharacter(src.files, any.missing = FALSE) src.files = sanitizePath(src.files, make.absolute = FALSE) # make paths absolute to be sure. otherwise cfSSH wont work for example # also check the dirs # file dir # job dir job.dir = getJobParentDir(file.dir) checkDir(job.dir, create = TRUE, check.empty = TRUE) # fun dir fun.dir = getFunDir(file.dir) checkDir(fun.dir, create = TRUE, check.empty = TRUE) # resources, pending, exports, work.dir checkDir(getResourcesDir(file.dir), create = TRUE, check.empty = TRUE) checkDir(getPendingDir(file.dir), create = TRUE, check.empty = TRUE) checkDir(getExportDir(file.dir), create = TRUE, check.empty = TRUE) sourceRegistryFilesInternal(work.dir, src.dirs, src.files) packages = setNames(lapply(packages, function(pkg) list(version = packageVersion(pkg))), packages) packages$BatchJobs$mandatory = TRUE conf = getConfig() setClasses(list( id = id, read.only = FALSE, version = R.version, RNGkind = RNGkind(), db.driver = conf$db.driver, db.options = conf$db.options, seed = seed, file.dir = file.dir, sharding = sharding, work.dir = work.dir, src.dirs = src.dirs, src.files = src.files, multiple.result.files = multiple.result.files, packages = packages ), "Registry") } #' Construct a registry object. #' #' Note that if you don't want links in your paths (\code{file.dir}, \code{work.dir}) to get resolved and have #' complete control over the way the path is used internally, pass an absolute path which begins with \dQuote{/}. #' #' Every object is a list that contains the passed arguments of the constructor. # #' @param id [\code{character(1)}]\cr #' Name of registry. Displayed e.g. in mails or in cluster queue. #' @param file.dir [\code{character(1)}]\cr #' Path where files regarding the registry / jobs should be saved. #' Default is \dQuote{-files} in current working directory if \code{id} is set. #' @param sharding [\code{logical(1)}]\cr #' Enable sharding to distribute result files into different subdirectories? #' Important if you have many experiments. #' Default is \code{TRUE}. #' @param work.dir [\code{character(1)}]\cr #' Working directory for R process when experiment is executed. #' Default is the current working directory when registry is created. #' @param multiple.result.files [\code{logical(1)}]\cr #' Should a result file be generated for every list element of the #' returned list of the job function? #' Note that the function provided to \code{\link{batchMap}} or #' \code{\link{batchReduce}} must return a named list if this is set to \code{TRUE}. #' The result file will be named \dQuote{-result-.RData} #' instead of \dQuote{-result.RData}. #' Default is \code{FALSE}. #' @param seed [\code{integer(1)}]\cr #' Start seed for experiments. The first experiment in the registry will use this #' seed, for the subsequent ones the seed is incremented by 1. #' Default is a random number from 1 to \code{.Machine$integer.max/2}. #' @param packages [\code{character}]\cr #' Packages that will always be loaded on each node. #' Default is \code{character(0)}. #' @param src.dirs [\code{character}]\cr #' Directories containing R scripts #' to be sourced on registry load (both on slave and master). #' Files not matching the pattern \dQuote{\\.[Rr]$} are ignored. #' Useful if you have many helper functions that are needed during the execution of your jobs. #' These files should only contain function definitions and no executable code. #' Default is \code{character(0)}. #' @param src.files [\code{character}]\cr #' R scripts files #' to be sourced on registry load (both on slave and master). #' Useful if you have many helper functions that are needed during the execution of your jobs. #' These files should only contain function and constant definitions and no long running, executable code. #' These paths are considered to be relative to your \code{work.dir}. #' As a last remedy in problematic cases you can use absolute paths, by passing paths that #' start with \dQuote{/}, see the comment about \code{file.dir} and \code{work.dir} above, #' where we allow the same thing. #' Note that this is a less portable approach and therefore usually a less good idea. #' Default is \code{character(0)}. #' @param skip [\code{logical(1)}]\cr #' Skip creation of a new registry if a registry is found in \code{file.dir}. #' Defaults to \code{TRUE}. #' @return [\code{\link{Registry}}] #' @aliases Registry #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' print(reg) makeRegistry = function(id, file.dir, sharding = TRUE, work.dir, multiple.result.files = FALSE, seed, packages = character(0L), src.dirs = character(0L), src.files = character(0L), skip = TRUE) { if (missing(file.dir)) file.dir = file.path(getwd(), paste0(id, "-files")) assertFlag(skip) if (skip && isRegistryDir(file.dir)) return(loadRegistry(file.dir = file.dir)) reg = makeRegistryInternal(id, file.dir, sharding, work.dir, multiple.result.files, seed, packages, src.dirs, src.files) dbCreateJobStatusTable(reg) dbCreateJobDefTable(reg) saveRegistry(reg) reg } #' @export print.Registry = function(x, ...) { cat("Job registry: ", x$id, "\n") cat(" Number of jobs: ", dbGetJobCount(x), "\n") cat(" Files dir:", x$file.dir, "\n") cat(" Work dir:", x$work.dir, "\n") cat(" Multiple result files:", x$multiple.result.files, "\n") cat(" Seed:", x$seed, "\n") cat(" Required packages:", collapse(names(x$packages), ", "), "\n") } saveRegistry = function(reg) { fn = getRegistryFilePath(reg$file.dir) info("Saving registry: %s", fn) save(file = fn, reg) reg } isRegistryDir = function(dir) { isDirectory(dir) && file.exists(getRegistryFilePath(dir)) } checkRegistry = function(reg, strict = FALSE, writeable = TRUE) { cl = class(reg) expected = "Registry" if (strict) { if (head(cl, 1L) != expected) stopf("Registry class mismatch: Expected argument with first class '%s'", expected) } else { if (expected %nin% cl) stopf("Registry class mismatch: Expected argument of class '%s'", expected) } if (writeable && isTRUE(reg$read.only)) stop("Registry is read-only. Operation not permitted.") invisible(TRUE) } #' Remove a registry object. #' #' If there are no live/running jobs, the registry will be closed #' and all of its files will be removed from the file system. #' If there are live/running jobs, an informative error is generated. #' The default is to prompt the user for confirmation. #' #' @template arg_reg #' @param ask [\code{character(1)}]\cr #' If \code{"yes"} the user is prompted to confirm the action. #' If trying to prompt the user this way in a non-interactive #' session, then an informative error is generated. #' If \code{"no"}, the registry will be removed without #' further confirmation. #' #' @return [\code{logical[1]}] #' #' @export removeRegistry = function(reg, ask = c("yes", "no")) { checkRegistry(reg, writeable = TRUE) syncRegistry(reg) ask = match.arg(ask) if (ask == "yes") { if (!interactive()) stopf("removeRegistry(..., ask = \"yes\") only works in interactive sessions.") prompt = sprintf("Are you sure you wish to delete BatchJobs registry '%s' and all of it's files in directory '%s'? [y/N]: ", reg$id, reg$file.dir) ans = 2L repeat { ans = tolower(readline(prompt)) ans = gsub("[ ]", "", ans) if (ans == "") ans = "no" ans = pmatch(ans, table=c("yes", "no"), nomatch=0L) if (ans > 0L) break } if (ans != 1L) return(invisible(FALSE)) } running = findOnSystem(reg) if (length(running) > 0L) stopf("Can't remove registry, because there are %d live jobs on the system.", length(running)) removeDirs(reg$file.dir, recursive=TRUE, must.work=TRUE) } BatchJobs/R/filterResults.R0000644000176200001440000000220512623105424015302 0ustar liggesusers#' Find all results where a specific condition is true. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs whose results you want to test for the condition. #' Default is all jobs for which results are available. #' @param fun [\code{fun(job, res)}]\cr #' Predicate function that returns \code{TRUE} or \code{FALSE}. #' @param ... [any]\cr #' Additional arguments to \code{fun}. #' @return [\code{integer}]. Ids of jobs where \code{fun(job, result)} returns \code{TRUE}. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' submitJobs(reg) #' waitForJobs(reg) #' #' # which square numbers are even: #' filterResults(reg, fun = function(job, res) res %% 2 == 0) filterResults = function(reg, ids, fun, ...) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) assertFunction(fun, c("job", "res")) if (missing(ids)) ids = dbFindDone(reg) else ids = checkIds(reg, ids) Filter(function(id) { fun(job = getJob(reg, id, check.id = FALSE), res = getResult(reg, id), ...) }, ids) } BatchJobs/R/reduceResults.R0000644000176200001440000002121112623105424015262 0ustar liggesusers#' Reduce results from result directory. #' #' @description #' The following functions provide ways to reduce result files into either specific R objects (like #' vectors, lists, matrices or data.frames) or to arbitrarily aggregate them, which is a more general #' operation. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of selected jobs. #' Default is all jobs for which results are available. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param fun [\code{function}]\cr #' For \code{reduceResults}, a function \code{function(aggr, job, res, ...)} to reduce things, #' for all others, a function \code{function(job, res, ...)} to select stuff. #' Here, \code{job} is the current job descriptor (see \code{\link{Job}}), \code{result} is the current result object and #' \code{aggr} are the so far aggregated results. When using \code{reduceResults}, #' your function should add the stuff you want to have from \code{job} and #' \code{result} to \code{aggr} and return that. #' When using the other reductions, you should select the stuff you want to have from \code{job} and #' \code{result} and return something that can be coerced to an element of the selected return data structure #' (reasonable conversion is tried internally). #' Default behavior for this argument is to return \code{res}, except for \code{reduceResults} where no #' default is available. #' @param init [\code{ANY}]\cr #' Initial element, as used in \code{\link{Reduce}}. #' Default is first result. #' @param ... [any]\cr #' Additional arguments to \code{fun}. #' @param use.names [\code{character(1)}]\cr #' Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) #' or return a unnamed result (\dQuote{none}). #' Default is \code{ids}. #' @param impute.val [any]\cr #' For \code{reduceResults}: If not missing, the value of \code{impute.val} is passed to function \code{fun} #' as argument \code{res} for jobs with missing results.\cr #' For the specialized reduction functions \code{reduceResults[Type]}: If not missing, \code{impute.val} is #' used as a replacement for the return value of \code{fun} on missing results. #' @template arg_progressbar #' @param rows [\code{logical(1)}]\cr #' Should the selected vectors be used as rows (or columns) in the result matrix? #' Default is \code{TRUE}. #' @param strings.as.factors [\code{logical(1)}] #' Should all character columns in result be converted to factors? #' Default is \code{default.stringsAsFactors()}. #' @return Aggregated results, return type depends on function. If \code{ids} is empty: \code{reduceResults} #' returns \code{init} (if available) or \code{NULL}, \code{reduceResultsVector} returns \code{c()}, #' \code{reduceResultsList} returns \code{list()}, \code{reduceResultsMatrix} returns \code{matrix(0,0,0)}, #' \code{reduceResultsDataFrame} returns \code{data.frame()}. #' @export #' @examples #' # generate results: #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # reduce results to a vector #' reduceResultsVector(reg) #' # reduce results to sum #' reduceResults(reg, fun = function(aggr, job, res) aggr+res) #' #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) list(a = x, b = as.character(2*x), c = x^2) #' batchMap(reg, f, 1:5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # reduce results to a vector #' reduceResultsVector(reg, fun = function(job, res) res$a) #' reduceResultsVector(reg, fun = function(job, res) res$b) #' # reduce results to a list #' reduceResultsList(reg) #' # reduce results to a matrix #' reduceResultsMatrix(reg, fun = function(job, res) res[c(1,3)]) #' reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = TRUE) #' reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = FALSE) #' # reduce results to a data.frame #' print(str(reduceResultsDataFrame(reg))) #' # reduce results to a sum #' reduceResults(reg, fun = function(aggr, job, res) aggr+res$a, init = 0) reduceResults = function(reg, ids, part = NA_character_, fun, init, impute.val, progressbar = TRUE, ...) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) { ids = done = dbFindDone(reg) with.impute = FALSE } else { ids = checkIds(reg, ids) done = dbFindDone(reg, ids) with.impute = !missing(impute.val) if (!with.impute) { if (length(ids) > length(done)) stopf("No results available for jobs with ids: %s", collapse(setdiff(ids, done))) } } assertFunction(fun, c("aggr", "job", "res")) assertFlag(progressbar) n = length(ids) info("Reducing ", n, " results...") if (n == 0L) { if (missing(init)) return(NULL) return(init) } bar = getProgressBar(progressbar, max = n, label = "reduceResults") tryCatch({ if (missing(init)) { # fetch first result as init aggr = getResult(reg, ids[1L], part) ids = tail(ids, -1L) bar$inc(1L) } else { aggr = init } not.done = (ids %nin% done) for (i in seq_along(ids)) { # use lazy evaluation aggr = fun(aggr, job = dbGetJobs(reg, ids[i])[[1L]], res = if (with.impute && not.done[i]) impute.val else getResult(reg, ids[i], part), ...) bar$inc(1L) } }, error = bar$error) return(aggr) } #' @export #' @rdname reduceResults reduceResultsList = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, progressbar = TRUE) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) { ids = done = dbFindDone(reg) with.impute = FALSE } else { ids = checkIds(reg, ids) done = dbFindDone(reg, ids) with.impute = !missing(impute.val) if (!with.impute) { if (length(ids) > length(done)) stopf("No results available for jobs with ids: %s", collapse(setdiff(ids, done))) } } if (missing(fun)) fun = function(job, res) res else assertFunction(fun, c("job", "res")) use.names = convertUseNames(use.names) assertFlag(progressbar) n = length(ids) info("Reducing %i results...", n) if (n == 0L) return(list()) res = vector("list", n) if (with.impute) { res = replace(res, ids %nin% done, impute.val) it = match(done, ids) } else { it = seq_len(n) } bar = getProgressBar(progressbar, max = n, label = "reduceResults") tryCatch({ for (i in it) { # use lazy evaluation! tmp = fun(job = dbGetJobs(reg, ids[i])[[1L]], res = getResult(reg, ids[i], part), ...) if (!is.null(tmp)) res[[i]] = tmp bar$inc(1L) } }, error = bar$error) names(res) = switch(use.names, "none" = NULL, "ids" = as.character(ids), "names" = dbGetJobNames(reg, ids)) return(res) } #' @export #' @rdname reduceResults reduceResultsVector = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val) { unlist(reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val)) } #' @export #' @rdname reduceResults reduceResultsMatrix = function(reg, ids, part = NA_character_, fun, ..., rows = TRUE, use.names = "ids", impute.val) { assertFlag(rows) use.names = convertUseNames(use.names) res = reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val) if (length(res) == 0L) return(matrix(0, nrow = 0L, ncol = 0L)) n = length(res) dn = if (use.names != "none") list(names(res), names(res[[1L]])) else NULL res = unlist(res, use.names = FALSE) if (rows) matrix(res, nrow = n, byrow = TRUE, dimnames = dn) else matrix(res, ncol = n, byrow = FALSE, dimnames = rev(dn)) } #' @export #' @rdname reduceResults reduceResultsDataFrame = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, strings.as.factors = default.stringsAsFactors()) { assertFlag(strings.as.factors) res = reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val) if (!length(res)) return(data.frame()) convertListOfRowsToDataFrame(res, strings.as.factors = strings.as.factors) } #' @export #' @rdname reduceResults reduceResultsDataTable = function(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val) { res = reduceResultsList(reg, ids, part, fun, ..., use.names = use.names, impute.val = impute.val) if (!length(res)) return(data.table()) rbindlist(res, fill = TRUE) } BatchJobs/R/killJobs.R0000644000176200001440000001027112623105424014206 0ustar liggesusers#' Kill some jobs on the batch system. #' #' @description #' Kill jobs which have already been submitted to the batch system. #' If a job is killed its internal state is reset as if it had not been submitted at all. #' #' The function informs if #' (a) the job you want to kill has not been submitted, #' (b) the job has already terminated, #' (c) for some reason no batch job id is available. #' In all 3 cases above, nothing is changed for the state of this job and no call #' to the internal kill cluster function is generated. #' #' In case of an error when killing, the function tries - after a short sleep - to kill the remaining #' batch jobs again. If this fails again for some jobs, the function gives up. Only jobs that could be #' killed are reset in the DB. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs to kill. #' Default is none. #' @template arg_progressbar #' @return [\code{integer}]. Ids of killed jobs. #' @export #' @family debug #' @examples #' \dontrun{ #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) Sys.sleep(x) #' batchMap(reg, f, 1:10 + 5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # kill all jobs currently _running_ #' killJobs(reg, findRunning(reg)) #' # kill all jobs queued or running #' killJobs(reg, findNotTerminated(reg)) #' } killJobs = function(reg, ids, progressbar = TRUE) { checkRegistry(reg, writeable = TRUE) syncRegistry(reg) if (missing(ids)) return(invisible(integer(0L))) else ids = checkIds(reg, ids) assertFlag(progressbar) conf = getBatchJobsConf() killfun = getKillJob("Cannot kill jobs") # FIXME select and order (see below) could be done more efficiently in SQLite data = dbGetJobStatusTable(reg, ids = dbFindOnSystem(reg, ids), cols = c("job_id", "batch_job_id", "submitted", "started", "done", "error")) # print first summary information on jobs to kill info("Trying to kill %i jobs.", length(ids)) info("Jobs on system: %i", nrow(data)) info("Of these: %i not submitted, %i with no batch.job.id, %i already terminated", sum(is.na(data$submitted)), sum(is.na(data$batch_job_id)), sum(!is.na(data$done) | !is.na(data$error))) # subset data: restrict to jobs submitted, not done, no error, has bji # other jobs can be ignored -> overwrite ids data = subset(data, !is.na(data$submitted) & is.na(data$done) & is.na(data$error) & !is.na(data$batch_job_id), select = c("job_id", "batch_job_id", "started")) # kill queued jobs first, otherwise they might get started while killing running jobs data = data[order(data$started, na.last = FALSE),, drop = FALSE] ids = data$job_id bjids = unique(data$batch_job_id) info("Killing real batch jobs: %i", length(bjids)) if (length(bjids) == 0L) { info("No batch jobs to kill.") return(invisible(integer(0L))) } doKill = function(bjids) { n = length(bjids) old.warn = getOption("warn") bar = getProgressBar(progressbar, min = 0L, max = n, label = "killJobs") on.exit({ options(warn = old.warn); bar$kill() }) options(warn = 0L) bar$set() notkilled = logical(n) for (i in seq_len(n)) { ok = try(killfun(conf, reg, bjids[i])) if (is.error(ok)) { notkilled[i] = TRUE warning(as.character(ok)) } bar$set(i) } bjids[notkilled] } # first try to kill info(clipString(collapse(bjids), 200L, ",...")) bjids.notkilled = doKill(bjids) # second try to kill if (length(bjids.notkilled) > 0L) { info("Could not kill %i batch jobs, trying again.", length(bjids.notkilled)) Sys.sleep(2) bjids.notkilled = doKill(bjids.notkilled) } # second try also not successful if (length(bjids.notkilled) > 0L) { fn = file.path(reg$file.dir, sprintf("killjobs_failed_ids_%i", now())) warningf("Could not kill %i batch jobs, kill them manually!\nTheir ids have been saved in %s.", length(bjids.notkilled), fn) writeLines(as.character(bjids.notkilled), con = fn) } # reset killed jobs ids = ids[data$batch_job_id %nin% bjids.notkilled] info("Resetting %i jobs in DB.", length(ids)) dbSendMessage(reg, dbMakeMessageKilled(reg, ids, type = "last")) invisible(ids) } BatchJobs/R/database.R0000644000176200001440000004472513207234653014222 0ustar liggesusers############################################ ### Common database functions ############################################ sqlQuote = function(x) { sprintf("'%s'", x) } dbGetConnection = function(drv, reg, ...) { # method dispatch to support different DBMS UseMethod("dbGetConnection") } dbGetConnection.SQLiteDriver = function(drv, reg, flags = "ro", ...) { flags = switch(flags, "ro" = SQLITE_RO, "rw" = SQLITE_RW, "rwc" = SQLITE_RWC) opts = list(dbname = file.path(reg$file.dir, "BatchJobs.db"), flags = flags, drv = drv) con = do.call(dbConnect, args = c(dropNamed(reg$db.options, "pragmas"), opts)) for (pragma in reg$db.options$pragmas) dbClearResult(dbSendQuery(con, sprintf("PRAGMA %s", pragma))) return(con) } dbConnectToJobsDB = function(reg, flags = "ro") { drv = do.call(reg$db.driver, list()) dbGetConnection(drv, reg, flags) } dbDoQueries = function(reg, queries, flags = "ro", max.retries = 100L, sleep = function(r) 1.025^r) { for (i in seq_len(max.retries)) { con = try(dbConnectToJobsDB(reg, flags), silent = TRUE) if (is.error(con)) { if (!grepl("(lock|i/o|readonly)", tolower(con))) stopf("Error while etablishing the connection: %s", as.character(con)) } else { ok = try ({ dbBegin(con) ress = lapply(queries, dbGetQuery, conn = con) }, silent = TRUE) if (!is.error(ok)) { # this can fail because DB is locked ok2 = dbCommit(con) if (ok2) { dbDisconnect(con) return(ress) } else { dbRollback(con) dbDisconnect(con) } } else { ok = as.character(ok) dbRollback(con) dbDisconnect(con) # catch known temporary errors: # - database is still locked # - disk I/O error # - disk I/O error # - database is only readable if(!grepl("(lock|i/o|readonly)", tolower(ok))) stopf("Error in dbDoQueries. Displaying only 1st query. %s (%s)", ok, queries[1L]) } } # if we reach this here, DB was locked or temporary I/O error Sys.sleep(runif(1L, min = 1, max = sleep(i))) } stopf("dbDoQueries: max retries (%i) reached, database is still locked!", max.retries) } dbDoQuery = function(reg, query, flags = "ro", max.retries = 100L, sleep = function(r) 1.025^r) { for (i in seq_len(max.retries)) { con = try(dbConnectToJobsDB(reg, flags), silent = TRUE) if (is.error(con)) { if (!grepl("(lock|i/o|readonly)", tolower(con))) stopf("Error while etablishing the connection: %s", as.character(con)) } else { res = try(dbGetQuery(con, query), silent = TRUE) dbDisconnect(con) if (!is.error(res)) return(res) res = as.character(res) if(!grepl("(lock|i/o|readonly)", tolower(res))) { stopf("Error in dbDoQuery. %s (%s)", res, query) } } # if we reach this here, DB was locked or temporary I/O error Sys.sleep(runif(1L, min = 1, max = sleep(i))) } stopf("dbDoQuery: max retries (%i) reached, database is still locked!", max.retries) } dbAddData = function(reg, tab, data) { query = sprintf("INSERT INTO %s_%s (%s) VALUES(%s)", reg$id, tab, collapse(colnames(data)), collapse(rep.int("?", ncol(data)))) con = dbConnectToJobsDB(reg, flags = "rw") on.exit(dbDisconnect(con)) dbBegin(con) res = dbSendQuery(con, query) for (i in seq_row(data)) { row = unname(as.list(data[i, ])) dbBind(res, row) ok = try(dbFetch(res)) if(is.error(ok)) { dbClearResult(res) dbRollback(con) stopf("Error in dbAddData: %s", as.character(ok)) } } dbClearResult(res) dbCommit(con) as.integer(dbGetQuery(con, "SELECT total_changes()")) } dbSelectWithIds = function(reg, query, ids, where = TRUE, group.by, limit = NULL, reorder = TRUE) { if(!missing(ids)) query = sprintf("%s %s job_id IN (%s)", query, ifelse(where, "WHERE", "AND"), collapse(ids)) if(!missing(group.by)) query = sprintf("%s GROUP BY %s", query, collapse(group.by)) if(!is.null(limit)) query = sprintf("%s LIMIT %i", query, limit) res = dbDoQuery(reg, query) if(missing(ids) || !reorder) return(res) return(res[na.omit(match(ids, res$job_id)),, drop = FALSE]) } ############################################ ### CREATE ############################################ #' ONLY FOR INTERNAL USAGE. #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @return Nothing. #' @keywords internal #' @export dbCreateJobDefTable = function(reg) { UseMethod("dbCreateJobDefTable") } #' @export dbCreateJobDefTable.Registry = function(reg) { query = sprintf("CREATE TABLE %s_job_def (job_def_id INTEGER PRIMARY KEY, fun_id TEXT, pars TEXT, jobname TEXT)", reg$id) dbDoQuery(reg, query, flags = "rwc") dbCreateExpandedJobsView(reg) } dbCreateJobStatusTable = function(reg, extra.cols = "", constraints = "") { query = sprintf(paste("CREATE TABLE %s_job_status (job_id INTEGER PRIMARY KEY, job_def_id INTEGER,", "first_job_in_chunk_id INTEGER, seed INTEGER, resources_timestamp INTEGER, memory REAL, submitted INTEGER,", "started INTEGER, batch_job_id TEXT, node TEXT, r_pid INTEGER,", "done INTEGER, error TEXT, error_time INTEGER %s %s)"), reg$id, extra.cols, constraints) dbDoQuery(reg, query, flags = "rwc") query = sprintf("CREATE INDEX job_def_id ON %s_job_status(job_def_id)", reg$id) dbDoQuery(reg, query, flags = "rw") return(invisible(TRUE)) } dbCreateExpandedJobsView = function(reg) { query = sprintf("CREATE VIEW %1$s_expanded_jobs AS SELECT * FROM %1$s_job_status LEFT JOIN %1$s_job_def USING(job_def_id)", reg$id) dbDoQuery(reg, query, flags = "rw") } ############################################ ### SELECT ############################################ #' ONLY FOR INTERNAL USAGE. #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of selected jobs. #' @return [list of \code{\link{Job}}]. Retrieved jobs from DB. #' @keywords internal #' @export dbGetJobs = function(reg, ids) { UseMethod("dbGetJobs") } # note that this does not load the job function from disk to increase speed #' @method dbGetJobs Registry #' @export dbGetJobs.Registry = function(reg, ids) { query = sprintf("SELECT job_id, fun_id, pars, jobname, seed FROM %s_expanded_jobs", reg$id) tab = dbSelectWithIds(reg, query, ids) lapply(seq_row(tab), function(i) { makeJob(id = tab$job_id[i], fun.id = tab$fun_id[i], fun = NULL, pars = unserialize(charToRaw(tab$pars[i])), name = tab$jobname[i], seed = tab$seed[i]) }) } dbGetExpandedJobsTable = function(reg, ids, cols = "*") { # Note: job_id must be in cols! query = sprintf("SELECT %s FROM %s_expanded_jobs", collapse(cols), reg$id) tab = dbSelectWithIds(reg, query, ids) setRowNames(tab, tab$job_id) } dbGetJobStatusTable = function(reg, ids, cols = "*") { # Note: job_id must be in cols! query = sprintf("SELECT %s FROM %s_job_status", collapse(cols), reg$id) tab = dbSelectWithIds(reg, query, ids) setRowNames(tab, tab$job_id) } dbGetJobCount = function(reg) { query = sprintf("SELECT COUNT(*) AS count FROM %s_job_status", reg$id) dbDoQuery(reg, query)$count } dbGetJobId = function(reg) { query = sprintf("SELECT job_id FROM %s_job_status LIMIT 1", reg$id) as.integer(dbDoQuery(reg, query)$job_id) } dbGetJobIds = function(reg) { query = sprintf("SELECT job_id FROM %s_job_status", reg$id) dbDoQuery(reg, query)$job_id } dbCheckJobIds = function(reg, ids) { not.found = setdiff(ids, dbGetJobIds(reg)) if (length(not.found) > 0L) stopf("Ids not present in registry: %s", collapse(not.found)) } dbGetJobIdsIfAllDone = function(reg) { query = sprintf("SELECT job_id, done FROM %s_job_status", reg$id) res = dbDoQuery(reg, query) if (all(! is.na(res$done))) return(res$job_id) stop("Not all jobs finished (yet)!") } dbGetLastAddedIds = function(reg, tab, id.col, n) { query = sprintf("SELECT %s AS id_col FROM %s_%s ORDER BY %s DESC LIMIT %i", id.col, reg$id, tab, id.col, n) rev(dbDoQuery(reg, query)$id_col) } dbFindDone = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (done IS NOT NULL)", reg$id, if(negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindErrors = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (error IS NOT NULL)", reg$id, if(negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindTerminated = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (done IS NOT NULL OR error IS NOT NULL)", reg$id, if(negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindSubmitted = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (submitted IS NOT NULL)", reg$id, if (negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindStarted = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (started IS NOT NULL)", reg$id, if (negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindOnSystem = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (batch_job_id IN (%s))", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindSubmittedNotTerminated = function(reg, ids, negate = FALSE, limit = NULL) { query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (submitted IS NOT NULL AND done IS NULL AND error IS NULL)", reg$id, if (negate) "NOT" else "") dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindRunning = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (batch_job_id IN (%s) AND started IS NOT NULL AND done IS NULL AND error IS NULL)", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindExpiredJobs = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") # started, not terminated, not running query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (started IS NOT NULL AND done IS NULL AND error is NULL AND batch_job_id NOT IN (%s))", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbFindDisappeared = function(reg, ids, negate = FALSE, limit = NULL, batch.ids) { if (missing(batch.ids)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") query = sprintf("SELECT job_id FROM %s_job_status WHERE %s (submitted IS NOT NULL AND started IS NULL AND batch_job_id NOT IN (%s))", reg$id, if (negate) "NOT" else "", collapse(sqlQuote(batch.ids))) dbSelectWithIds(reg, query, ids, where = FALSE, limit = limit)$job_id } dbGetFirstJobInChunkIds = function(reg, ids){ query = sprintf("SELECT job_id, first_job_in_chunk_id FROM %s_job_status", reg$id) dbSelectWithIds(reg, query, ids)$first_job_in_chunk_id } dbGetErrorMsgs = function(reg, ids, filter = FALSE, limit = NULL) { query = sprintf("SELECT job_id, error from %s_job_status", reg$id) if (filter) query = sprintf("%s WHERE error IS NOT NULL", query) dbSelectWithIds(reg, query, ids, where = !filter, limit = limit) } dbGetStats = function(reg, ids, running = FALSE, expired = FALSE, times = FALSE, batch.ids) { cols = c(n = "COUNT(job_id)", submitted = "COUNT(submitted)", started = "COUNT(started)", done = "COUNT(done)", error = "COUNT(error)", running = "NULL", expired = "NULL", t_min = "NULL", t_avg = "NULL", t_max = "NULL") if (missing(batch.ids) && (expired || running)) batch.ids = getBatchIds(reg, "Cannot find jobs on system") if(running) cols["running"] = sprintf("SUM(started IS NOT NULL AND done IS NULL AND error IS NULL AND batch_job_id IN (%s))", collapse(sqlQuote(batch.ids))) if(expired) cols["expired"] = sprintf("SUM(started IS NOT NULL AND done IS NULL AND error IS NULL AND batch_job_id NOT IN (%s))", collapse(sqlQuote(batch.ids))) if (times) cols[c("t_min", "t_avg", "t_max")] = c("MIN(done - started)", "AVG(done - started)", "MAX(done - started)") query = sprintf("SELECT %s FROM %s_job_status", collapse(paste(cols, "AS", names(cols)), sep = ", "), reg$id) df = dbSelectWithIds(reg, query, ids, reorder = FALSE) # Convert to correct type. Null has no type and casts tend to not work properly with RSQLite x = c("n", "submitted", "started", "done", "error", "running", "expired") df[x] = lapply(df[x], as.integer) x = c("t_min", "t_avg", "t_max") df[x] = lapply(df[x], as.double) df } dbGetJobNames = function(reg, ids) { query = sprintf("SELECT job_id, jobname FROM %s_expanded_jobs", reg$id) as.character(dbSelectWithIds(reg, query, ids)$jobname) } dbMatchJobNames = function(reg, ids, jobnames) { query = sprintf("SELECT job_id FROM %s_expanded_jobs WHERE jobname IN (%s)", reg$id, collapse(sqlQuote(jobnames))) dbSelectWithIds(reg, query, ids, where = FALSE)$job_id } ############################################ ### Messages ############################################ dbSendMessage = function(reg, msg, staged = useStagedQueries(), fs.timeout = NA_real_) { ## AD HOC/FIXME: Avoid partial matching; some functions pass 'msg' with ## field 'msgs' and some with field 'msg' (e.g. dbMakeMessageError()). msgT <- if ("msgs" %in% names(msg)) msg$msgs else msg$msg if (staged) { fn = getPendingFile(reg, msg$type, msg$ids[1L]) writeSQLFile(msgT, fn) waitForFiles(fn, timeout = fs.timeout) } else { dbDoQuery(reg, msgT, flags = "rw") } } dbSendMessages = function(reg, msgs, max.retries = 200L, sleep = function(r) 1.025^r, staged = useStagedQueries(), fs.timeout = NA_real_) { if (length(msgs) == 0L) return(TRUE) if (staged) { chars = .OrderChars # reorder messages in sublist msgs = split(msgs, extractSubList(msgs, "type")) msgs = msgs[order(match(names(msgs), names(chars)))] fns = vcapply(msgs, function(cur) { first = cur[[1L]] fn = getPendingFile(reg, first$type, first$ids[1L], chars[first$type]) writeSQLFile(extractSubList(cur, "msg"), fn) fn }) waitForFiles(fns, timeout = fs.timeout) } else { ok = try(dbDoQueries(reg, extractSubList(msgs, "msg"), flags = "rw", max.retries, sleep)) if (is.error(ok)) { ok = as.character(ok) if (ok == "dbDoQueries: max retries reached, database is still locked!") { return(FALSE) } else { # throw exception again stopf("Error in dbSendMessages: %s", ok) } } } return(TRUE) } dbMakeMessageSubmitted = function(reg, job.ids, time = now(), batch.job.id, first.job.in.chunk.id = NULL, resources.timestamp, type = "submitted") { if(is.null(first.job.in.chunk.id)) first.job.in.chunk.id = "NULL" updates = sprintf("first_job_in_chunk_id=%s, submitted=%i, batch_job_id='%s', resources_timestamp=%i", first.job.in.chunk.id, time, batch.job.id, resources.timestamp) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageStarted = function(reg, job.ids, time = now(), type = "started") { node = gsub("'", "\"", Sys.info()["nodename"], fixed = TRUE) updates = sprintf("started=%i, node='%s', r_pid=%i, error=NULL, done=NULL", time, node, Sys.getpid()) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageError = function(reg, job.ids, err.msg, time = now(), memory = -1, type = "error") { # FIXME how to escape ticks (')? Just replaced with double quotes for the moment err.msg = gsub("'", "\"", err.msg, fixed = TRUE) err.msg = gsub("[^[:print:]]", " ", err.msg) updates = sprintf("error='%s', error_time=%i, done=NULL, memory='%.4f'", err.msg, time, memory) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageDone = function(reg, job.ids, time = now(), memory = -1, type = "done") { updates = sprintf("done=%i, error=NULL, memory='%.04f'", time, memory) list(msg = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbMakeMessageKilled = function(reg, job.ids, type = "last") { updates = "resources_timestamp=NULL, memory=NULL, submitted=NULL, started=NULL, batch_job_id=NULL, node=NULL, r_pid=NULL, done=NULL, error=NULL" list(msgs = sprintf("UPDATE %s_job_status SET %s WHERE job_id in (%s)", reg$id, updates, collapse(job.ids)), ids = job.ids, type = type) } dbConvertNumericToPOSIXct = function(x) { now = Sys.time() as.POSIXct(x, origin = now - as.integer(now)) } dbSetJobFunction = function(reg, ids, fun.id) { query = sprintf("UPDATE %1$s_job_def SET fun_id = '%2$s' WHERE job_def_id IN (SELECT job_def_id FROM %1$s_job_status WHERE job_id IN (%3$s))", reg$id, fun.id, collapse(ids)) dbDoQuery(reg, query, flags = "rw") } dbSetJobNames = function(reg, ids, jobnames) { queries = sprintf("UPDATE %1$s_job_def SET jobname = '%2$s' WHERE job_def_id IN (SELECT job_def_id FROM %1$s_job_status WHERE job_id IN (%3$i))", reg$id, jobnames, ids) dbDoQueries(reg, queries, flags = "rw") } # this is used in parallelMap :/ dbRemoveJobs = function(reg, ids) { query = sprintf("DELETE FROM %s_job_status WHERE job_id IN (%s)", reg$id, collapse(ids)) dbDoQuery(reg, query, flags = "rw") query = sprintf("DELETE FROM %1$s_job_def WHERE job_def_id NOT IN (SELECT DISTINCT job_def_id FROM %1$s_job_status)", reg$id) dbDoQuery(reg, query, flags = "rw") return(invisible(TRUE)) } BatchJobs/R/callFunctionOnSSHWorkers.R0000644000176200001440000001461712571265761017333 0ustar liggesusers#' Call an arbitrary function on specified SSH workers. #' #' @description #' Calls can be made in parallel or consecutively, #' the function waits until all calls have finished and #' returns call results. #' In consecutive mode the output on the workers can also be shown on the master #' during computation. #' #' Please read and understand the comments for argument \code{dir}. #' #' Note that this function should only be used for short administrative #' tasks or information gathering on the workers, the true work horse for #' real computation is \code{\link{submitJobs}}. #' #' In \code{\link{makeSSHWorker}} various options for load #' management are possible. Note that these will be #' ignored for the current call to execute it immediatly. #' #' @param nodenames [\code{character}]\cr #' Nodenames of workers to call function on. #' Only workers which were specified in your #' \code{\link{makeClusterFunctionsSSH}} configuration can be used. #' @param fun [\code{function}]\cr #' Function to call on workers. #' @param ... [any]\cr #' Arguments for \code{fun}. #' @param consecutive [\code{logical(1)}]\cr #' Do calls consecutively and always wait until each worker is done. #' Default is \code{FALSE}. #' @param show.output [\code{logical(1)}]\cr #' Show output of workers on master during call. #' Can be useful to see what is happening. #' Can only be used in consecutive mode. #' Default is \code{consecutive}. #' @param simplify [\code{logical(1)}]\cr #' Should the result be simplified? See \code{\link{sapply}}. #' Default is \code{TRUE}. #' @param use.names [\code{logical(1)}]\cr #' Name results by \code{nodenames}. #' Default is \code{TRUE}. #' @param dir [\code{character(1)}]\cr #' Directory under which a temporary registry will be #' created in a subdirectory for communication. #' This has to be somewhere on the shared #' filesystem. The created subdirectory will be cleaned up on exit. #' Default is current working directory. #' @return Results of function calls, either a list or simplified. #' @export callFunctionOnSSHWorkers = function(nodenames, fun, ..., consecutive = FALSE, show.output = consecutive, simplify = TRUE, use.names = TRUE, dir = getwd()) { assertCharacter(nodenames, any.missing = FALSE) assertFunction(fun) assertFlag(consecutive) assertFlag(show.output) assertFlag(simplify) assertFlag(use.names) if (!consecutive && show.output) stop("show.output = TRUE can only be used in consecutive mode.") conf = getBatchJobsConf() cf = conf$cluster.functions mail.old = c(conf$mail.start, conf$mail.done, conf$mail.error) if (cf$name != "SSH") stop("callFunctionOnSSHWorkers can only be used in SSH mode!") # create dummy registry, we submit our command from this regid = sprintf("BatchJobs_callFunctionOnSSHWorkers_%i", as.integer(Sys.time())) regdir = file.path(dir, regid) # we will change mailing and cluster funs, reset them on exit # also kill all still running jobs and remove reg dir on.exit({ conf$cluster.functions = cf conf$mail.start = mail.old[1L] conf$mail.done = mail.old[2L] conf$mail.error = mail.old[3L] }) # no mails during for the following jobs, also get the nodenames ssh workers conf$mail.start = conf$mail.done = conf$mail.error = "none" workers = environment(cf$submitJob)$workers d = setdiff(nodenames, names(workers)) if(length(d) > 0L) stopf("For some nodenames no workers exist: %s", collapse(d)) workers = workers[nodenames] # ignore load constraints old.worker.settings = list() wsettings = c("ncpus", "max.jobs", "max.load") for (wn in names(workers)) { w = workers[[wn]] old.worker.settings[[wn]] = mget(wsettings, w) for (ws in wsettings) w[[ws]] = Inf } on.exit({ # reset load settings for (wn in names(old.worker.settings)) { for (ws in wsettings) workers[[wn]][[ws]] = old.worker.settings[[wn]][[ws]] } }, add = TRUE) args = if (consecutive) args = list(fun) else replicate(length(nodenames), fun) suppressMessages({ reg = makeRegistry(regid, file.dir = regdir, sharding = FALSE) more.args = list(...) batchMap(reg, function(fun, ...) { print("###logstart###") res = fun(...) print("###logend###") res }, args, more.args = more.args) }) on.exit({ if (length(findOnSystem(reg)) > 0L) killJobs(reg, getJobIds(reg)) if (file.exists(regdir)) unlink(regdir, recursive = TRUE) }, add = TRUE) # read log as char string, get part between stamps and print only new chars printLog = function(log.old) { log.fn = getLogFiles(reg, 1L) log.new = readChar(log.fn, file.info(log.fn)$size) j = gregexpr("###logstart###", log.new)[[1L]][1L] if (j == -1) { # start not found log.new = "" } else { # start found, clip log.new = substr(log.new, j+15, nchar(log.new)) j = gregexpr("###logend###", log.new)[[1L]][1L] if (j != -1L) { # end found, clip log.new = substr(log.new, 1L, j-7L) } } cat(substr(log.new, nchar(log.old)+1L, nchar(log.new))) log.new } mysubmit = function(nodenames, reg) { messagef("Calling function on: %s.", collapse(nodenames)) conf$cluster.functions = makeClusterFunctionsSSH(workers = workers) capture.output(suppressMessages(submitJobs(reg, getJobIds(reg)))) } # while job not done, sleep and maybe print log waitTillJobsDone = function(reg) { log.old = "" while(length(findOnSystem(reg) > 0L)) { if (show.output) log.old = printLog(log.old) Sys.sleep(1) } if (show.output) { log.old = printLog(log.old) cat("\n\n") } } # if error, throw it on master checkJobErrors = function(reg, nodenames) { errids = findErrors(reg) if (length(errids) > 0L) { j = errids[1L] stopf("Error on %s: %s", nodenames[j], getErrorMessages(reg, j)) } } doit = function(reg, nodenames) { mysubmit(nodenames, reg) waitTillJobsDone(reg) checkJobErrors(reg, nodenames) } if (consecutive) { # loop though nodes individually and call function results = lapply(nodenames, function(nn) { log.old = "" doit(reg, nn) loadResult(reg, 1L) }) } else { doit(reg, nodenames) results = loadResults(reg, simplify = FALSE, use.names = FALSE) } if (use.names) names(results) = nodenames if (simplify) results = simplify2array(results) return(results) } BatchJobs/R/getJobParamDf.R0000644000176200001440000000111612623105424015100 0ustar liggesusers#' @title Retrieve Job Parameters. #' #' @description #' Returns parameters for all jobs as the rows of a data.frame. #' #' @template arg_reg #' @template arg_ids #' @return [\code{data.frame}]. Rows are named with job ids. #' @export #' @examples #' # see batchExpandGrid getJobParamDf = function(reg, ids) { checkRegistry(reg, strict = TRUE, writeable = FALSE) syncRegistry(reg) if (!missing(ids)) ids = checkIds(reg, ids) tab = dbGetExpandedJobsTable(reg, ids, cols = c("job_id", "pars")) res = deserialize(tab$pars) setDF(res, rownames = rownames(tab)) return(res) } BatchJobs/R/helpers.R0000644000176200001440000001027412623105424014102 0ustar liggesuserscheckMoreArgs = function(more.args, reserved) { assertList(more.args, names = "strict") n = names(more.args) if(is.null(n) || missing(reserved)) return(invisible(TRUE)) check = reserved %in% n if (any(check)) stopf("more.args uses element names which are internally reserved: %s", collapse(reserved[check])) return(invisible(TRUE)) } checkPart = function(reg, part) { if (reg$multiple.result.files) { if (!testScalarNA(part) && !testCharacter(part, any.missing = FALSE)) stop("'part' must be NA or a character vector without NAs!") } else { if (!testScalarNA(part)) stop("'part' must be NA because multiple.result.files is FALSE!") } } getListJobs = function(msg = NULL) { conf = getBatchJobsConf() cf = getClusterFunctions(conf) fun = cf$listJobs if (is.null(fun) && !is.null(msg)) stopf("%s because %s cluster functions do not support listing of jobs!", msg, cf$name) return(fun) } getKillJob = function(msg = NULL) { conf = getBatchJobsConf() cf = getClusterFunctions(conf) fun = cf$killJob if (is.null(fun) && !is.null(msg)) stopf("%s because %s cluster functions do not support killing of jobs!", msg, cf$name) return(fun) } getBatchIds = function(reg, msg = NULL) { fun = getListJobs(msg) fun(getBatchJobsConf(), reg) } getRandomSeed = function(n = 1L) { as.integer(runif(n, 1, .Machine$integer.max / 2L)) } seeder = function(reg, seed) { if(!exists(".Random.seed", envir = .GlobalEnv)) runif(1L) prev.seed = get(".Random.seed", envir = .GlobalEnv) prev.kind = RNGkind() set.seed(seed, kind = reg$RNGkind[1L], normal.kind = reg$RNGkind[2L]) return(list( reset = function() { RNGkind(kind = prev.kind[1L], normal.kind = prev.kind[2L]) assign(".Random.seed", prev.seed, envir = .GlobalEnv) })) } switchWd = function(reg) { cur = getwd() message("Setting work dir: ", reg$work.dir) setwd(reg$work.dir) return(list(reset = function() { message("Setting work back to: ", cur) setwd(cur) })) } addIntModulo = function(x, y, mod = .Machine$integer.max) { as.integer((as.double(x) + as.double(y)) %% mod) } isOnSlave = function() { getOption("BatchJobs.on.slave", default = FALSE) } setOnSlave = function(x, resources.path = as.character(NA)) { options(BatchJobs.on.slave = x) options(BatchJobs.resources.path = resources.path) } now = function() { as.integer(Sys.time()) } getArgNames = function(args) { if (!length(args)) return(NULL) if (is.null(names(args[[1L]])) && is.character(args[[1L]])) return(args[[1L]]) return(names(args[[1L]])) } convertUseNames = function(use.names) { if (is.character(use.names) && length(use.names) == 1L && use.names %in% c("none", "ids", "names")) return(use.names) assertFlag(use.names) c("none", "ids")[use.names+1L] } waitForFiles = function(fn, timeout = NA_real_, sleep = 1) { if (is.na(timeout)) return(invisible(TRUE)) fn = fn[!file.exists(fn)] if (length(fn)) { start = now() repeat { Sys.sleep(sleep) fn = fn[!file.exists(fn)] if (!length(fn)) break if (now() - start > timeout) stopf("Error waiting for file system. File '%s' timed out after %.1f seconds", head(fn, 1L), timeout) } } invisible(TRUE) } info = function(...) { if (getOption("BatchJobs.verbose", default = TRUE)) message(sprintf(...)) } getProgressBar = function(condition, ...) { if (condition) { pb = makeProgressBar(...) pb$set() } else { pb = makeProgressBar(style = "off") } pb } checkUserFunction = function(fun) { fun = match.fun(fun) if (getOption("BatchJobs.clear.function.env")) { environment(fun) = .GlobalEnv } else { ee = environment(fun) if (!is.null(ee) && !isNamespace(ee)) { nn = ls(ee, all.names = TRUE) if (sum(vnapply(nn, function(nn) object.size(ee[[nn]])) / 1024^2) > 10) warning("The environment of provided function exceeds 10Mb.") } } fun } deserialize = function(pars) { pars = lapply(pars, function(x) unserialize(charToRaw(x))) if (length(pars) == 0L) return(data.table()) pn = make.names(names2(pars[[1L]], missing.val = ""), unique = TRUE) pars = rbindlist(pars) setnames(pars, pn) pars } BatchJobs/R/clusterFunctionsOpenLava.R0000644000176200001440000000622612527173323017450 0ustar liggesusers#' @title Create cluster functions for OpenLava systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{bsub} command. Jobs are #' killed using the \code{bkill} command and the list of running jobs #' is retrieved using \code{bjobs -u $USER -w}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfOpenLava}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsOpenLava = function(template.file, list.jobs.cmd = c("bjobs", "-u $USER", "-w")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only # when unfinished jobs are found, and 255 when no jobs are found, # or a non-existent job ID is entered. Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = "Y") submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "job") # returns: "Job <128952> is submitted to default queue ." res = runOSCommandLinux("bsub", stdin = outfile, stop.on.exit.code = FALSE) # FIXME filled queues if (res$exit.code > 0L) { cfHandleUnknownSubmitError("bsub", res$exit.code, res$output) } else { # collapse output strings and first number in string is batch.job.id batch.job.id = stri_extract_first_regex(collapse(res$output, sep = " "), "\\d+") makeSubmitJobResult(status = 0L, batch.job.id = batch.job.id) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("bkill", batch.job.id) } listJobs = function(conf, reg) { # JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME # 106560 rogon UNKWN m_amd hpc84 hpc25 QScript Mar 19 12:18 # res = runOSyyCommandLinux("bjobs", c("-u $USER", "-w"), stop.on.exit.code = FALSE) res = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L], stop.on.exit.code = FALSE) if (res$exit.code == 255L && grepl("No unfinished job found", res$output, fixed = TRUE)) return(character(0L)) if (res$exit.code > 0L) stopf("bjobs produced exit code %i; output %s", res$exit.code, res$output) # drop first header line of output out = tail(res$output, -1L) # first number in strings are batch.job.ids stri_extract_first_regex(out, "\\d+") } getArrayEnvirName = function() "LSB_JOBINDEX" makeClusterFunctions(name = "OpenLava", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/Worker.R0000644000176200001440000001204712623105424013711 0ustar liggesusers# ******************** Constructors ******************** # Abstract base class constructor for general workers. # # @param ssh [\code{logical(1)})]\cr # Use ssh for remote node? # @param nodename [\code{character(1)}]\cr # Host name of node. # @param ssh.cmd [\code{character(1)})]\cr # CLI command to ssh into remote node. # @param ssh.args [\code{character}]\cr # CLI args for \code{ssh.cmd}. # @param rhome [\code{character(1)}]\cr # Path to R installation on worker. # \dQuote{} means R installation on the PATH is used. # @param r.options [\code{character}] # Options for R and Rscript, one option per element of the vector, # a la \dQuote{--vanilla}. # @param script [\code{character(1)}]\cr # Path to helper script on worker. # Default means to call \code{\link{findHelperScriptLinux}}. # @param ncpus [\code{integers(1)}]\cr # Number of VPUs of worker. # Default means to query the worker via \code{\link{getWorkerNumberOfCPUs}}. # @param max.jobs [\code{integer(1)}]\cr # Maximal number of jobs that can run concurrently for the current registry. # Default is \code{ncpus}. # @param max.load [\code{numeric(1)}]\cr # Load average (of the last 5 min) at which the worker is considered occupied, # so that no job can be submitted. # Default is \code{ncpus-1}. # @param nice [\code{integer(1)}]\cr # Process priority to run R with set via nice. Integers between -20 and 19 are allowed. # If missing, processes are not nice'd and the system default applies (usually 0). # @param classes [\code{character}]\cr # Extra classes, more specific than dQuote{Worker}. # Will be added to the class attribute of the object. # @return [\code{\link{Worker}}]. makeWorker = function(ssh, nodename, ssh.cmd, ssh.args, rhome, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script, ncpus, max.jobs, max.load, nice, classes) { assertFlag(ssh) assertString(nodename) if (!is.null(ssh.cmd)) assertString(ssh.cmd) if (!is.null(ssh.args)) assertCharacter(ssh.args, any.missing = FALSE) assertString(rhome) assertCharacter(r.options, any.missing = FALSE) if (missing(script)) { # FIXME: dont use linux specific in base class script = findHelperScriptLinux(rhome, r.options, ssh, ssh.cmd, ssh.args, nodename) } else { assertString(script) } # construct object partially so we can query ncpus w = as.environment(list( ssh = ssh, ssh.cmd = ssh.cmd, ssh.args = ssh.args, nodename = nodename, rhome = rhome, r.options = r.options, script = script, last.update = -Inf, available = "A", # worker is available, we can submit, set in update loop in scheduleWorkerJobs.R status = NULL)) class(w) = c(classes, "Worker") if (missing(ncpus)) { ncpus = getWorkerNumberOfCPUs(w) messagef("Setting for worker %s: ncpus=%i", w$nodename, ncpus) } else { ncpus = asCount(ncpus) } if (missing(max.jobs)) { max.jobs = ncpus } else { max.jobs = asCount(max.jobs) } if (missing(max.load)) { max.load = max(getOption("mc.cores", parallel::detectCores()) - 1L, 1L) } else { assertNumber(max.load, lower = 0) } if (missing(nice)) { nice = "" } else { nice = asInt(nice, lower = -20, upper = 19) } w$ncpus = ncpus w$max.jobs = max.jobs w$max.load = max.load w$nice = nice return(w) } # ******************** Interface definition ******************** # Return number of cores on worker. # @param worker [\code{\link{Worker}}]. # Worker. # @return [\code{integer(1)}]. getWorkerNumberOfCPUs = function(worker) { UseMethod("getWorkerNumberOfCPUs") } # Return 4 numbers to describe worker status. # - load average of last 1 min, as given by e.g. uptime # - number of R processes by _all_ users # - number of R processes by _all_ users which have a load of >= 50% # - number of R processes by current user which match $FILEDIR/jobs in the cmd call of R # @param worker [\code{\link{Worker}}]. # Worker. # @param file.dir [\code{character(1)}}]. # File dir of registry. # @return [named \code{list} of \code{numeric(1)}]. getWorkerStatus = function(worker, file.dir) { UseMethod("getWorkerStatus") } # Start a job on worker, probably with R CMD BATCH. # @param worker [\code{\link{Worker}}]. # Worker. # @param rfile [\code{character(1)}]. # Path to R file to execute. # @param outfile [\code{character(1)}]. # Path to log file for R process. # @return [\code{character(1)}]. Relevant process id. startWorkerJob = function(worker, rfile, outfile) { UseMethod("startWorkerJob") } # Kill a job on worker. Really do it. # @param worker [\code{\link{Worker}}]. # Worker. # @param pid [\code{character(1)}]. # Process id from DB/batch.job.id to kill. # @return Nothing. killWorkerJob = function(worker, pid) { UseMethod("killWorkerJob") } # List all jobs on worker belonging to the current registry. # @param worker [\code{\link{Worker}}]. # Worker. # @param file.dir [\code{character(1)}}]. # File dir of registry. # @return [\code{character}]. Vector of process ids. listWorkerJobs = function(worker, file.dir) { UseMethod("listWorkerJobs") } BatchJobs/R/Job.R0000644000176200001440000000302712623105424013150 0ustar liggesusers#' Creates a job description. #' #' Usually you will not do this manually. #' Every object is a list that contains the passed arguments of the constructor. #' #' @param id [\code{integer(1)}]\cr #' Job id, determined by DB autoincrement. #' Default is \code{NA}. #' @param fun [\code{function}]\cr #' Job function to apply on parameters. #' @param fun.id [\code{character(1)}]\cr #' Id used to store function on disk. #' Default is \code{digest(fun)}. #' @param pars [\code{list}]\cr #' Parameter list for job function. #' @param name [\code{character(1)}]\cr #' Alias name for this job. #' @param seed [\code{integer(1)}]\cr #' Random seed for job. #' @aliases Job #' @export makeJob = function(id = NA_integer_, fun, fun.id = digest(fun), pars, name, seed) { setClasses(list(id = id, fun = fun, fun.id = fun.id, pars = pars, name = name, seed = seed), "Job") } #' Get number of jobs in registry. #' @template arg_reg #' @return [\code{integer(1)}]. #' @export getJobNr = function(reg) { checkRegistry(reg, writeable = FALSE) dbGetJobCount(reg) } #' Get ids of jobs in registry. #' @template arg_reg #' @return [\code{character}]. #' @export getJobIds = function(reg) { checkRegistry(reg, writeable = FALSE) dbGetJobIds(reg) } #' @export print.Job = function(x, ...) { cat("BatchJobs job:\n") catf(" Job id: %s", x$id) catf(" Fun id: %s", x$fun.id) catf(" Fun formals: %s", collapse(names(formals(x$fun)))) catf(" Name: %s", x$name) catf(" Seed: %i", x$seed) catf(" Pars: %s", convertToShortString(x$pars)) } BatchJobs/R/conf.R0000644000176200001440000001754313207234653013401 0ustar liggesusers#' BatchJobs configuration. #' #' In order to understand how the package should be configured #' please read #' \url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}. #' #' @name configuration #' @rdname configuration #' @family conf #' @aliases .BatchJobs.R NULL # sources 1 config file and returns the envir sourceConfFile = function(conffile) { assertFileExists(conffile) conf = new.env() x = try(sys.source(conffile, envir = conf)) if (is.error(x)) stopf("There was an error in sourcing your configuration file '%s': %s!", conffile, as.character(x)) checkConf(conf) do.call(checkConfElements, as.list(conf)) return(conf) } # sources multiple config files, the later overwrite the first, and returns the envir sourceConfFiles = function(conffiles) { conf = new.env() for (cf in conffiles) { conf2 = sourceConfFile(cf) lapply(ls(conf2), function(x) assign(x, conf2[[x]], envir = conf)) } return(conf) } # assigns a conf to namespace assignConf = function(conf) { conf.in.ns = getBatchJobsConf() lapply(ls(conf), function(x) assign(x, conf[[x]], envir = conf.in.ns)) } # locates package conf, userhome conf, working dir conf findConfigs = function(path = find.package("BatchJobs")) { fn.pack = file.path(path, "etc", "BatchJobs_global_config.R") fn.user = path.expand("~/.BatchJobs.R") fn.wd = suppressWarnings(normalizePath(".BatchJobs.R")) Filter(file.exists, unique(c(fn.pack, fn.user, fn.wd))) } # reads available config files and assigns them to namespace readConfs = function(path = find.package("BatchJobs")) { conffiles = findConfigs(path) if (length(conffiles) == 0L) { warning("No configuation found at all. Not in package, not in user.home, not in work dir!") assignConfDefaults() return(character(0L)) } # really do this in 2 steps # otherwise weird things might happen due to lazy eval combined with envirs # and we might not see the error msg triggered in the checking of the config file conf = sourceConfFiles(conffiles) assignConf(conf) invisible(conffiles) } assignConfDefaults = function() { conf = getBatchJobsConf() conf$cluster.functions = makeClusterFunctionsInteractive() conf$mail.start = "none" conf$mail.done = "none" conf$mail.error = "none" conf$db.driver = "SQLite" conf$db.options = list(pragmas = "busy_timeout=5000") conf$default.resources = list() conf$debug = FALSE conf$raise.warnings = FALSE conf$staged.queries = TRUE conf$max.concurrent.jobs = Inf conf$fs.timeout = NA_real_ conf$measure.mem = TRUE } # loads conf into namespace on slave loadConf = function(reg) { fn = getConfFilePath(reg) info("Loading conf: %s", fn) loaded = load2(fn, simplify = FALSE)$conf # assign all stuff to conf in namespace conf = getBatchJobsConf() lapply(ls(loaded), function(x) assign(x, loaded[[x]], envir = conf)) invisible(conf) } getBatchJobsConf = function() { get(".BatchJobs.conf", envir = getNamespace("BatchJobs")) } saveConf = function(reg) { fn = getConfFilePath(reg) info("Saving conf: %s", fn) conf = getBatchJobsConf() save(file = fn, conf) } getConfNames = function() { c("cluster.functions", "mail.start", "mail.done", "mail.error", "mail.from", "mail.to", "mail.control", "db.driver", "db.options", "default.resources", "debug", "raise.warnings", "staged.queries", "max.concurrent.jobs", "fs.timeout", "measure.mem") } checkConf = function(conf) { ns = if (is.list(conf)) names(conf) else ls(conf, all.names = TRUE) ns2 = getConfNames() if (any(ns %nin% ns2)) stopf("You are only allowed to define the following R variables in your config:\n%s\nBut you also had:\n%s", collapse(ns2, sep = ", "), collapse(setdiff(ns, ns2), sep = ", ")) } checkConfElements = function(cluster.functions, mail.to, mail.from, mail.start, mail.done, mail.error, mail.control, db.driver, db.options, default.resources, debug, raise.warnings, staged.queries, max.concurrent.jobs, fs.timeout, measure.mem) { mail.choices = c("none", "first", "last", "first+last", "all") if (!missing(cluster.functions)) assertClass(cluster.functions, "ClusterFunctions") if (!missing(mail.from)) assertString(mail.from) if (!missing(mail.to)) assertString(mail.to) if (!missing(mail.start)) assertChoice(mail.start, mail.choices) if (!missing(mail.done)) assertChoice(mail.done, mail.choices) if (!missing(mail.error)) assertChoice(mail.error, mail.choices) if (!missing(mail.control)) assertList(mail.control) if (!missing(db.driver)) assertString(db.driver) if (!missing(db.options)) assertList(db.options, names = "named") if (!missing(default.resources)) assertList(default.resources, names = "named") if (!missing(debug)) assertFlag(debug) if (!missing(raise.warnings)) assertFlag(raise.warnings) if (!missing(staged.queries)) assertFlag(staged.queries) if (!missing(max.concurrent.jobs)) assertCount(max.concurrent.jobs) if (!missing(fs.timeout)) assertNumber(fs.timeout) if (!missing(measure.mem)) assertFlag(measure.mem) } getClusterFunctions = function(conf) { conf$cluster.functions } # Function which returns a printable string describing the config # Used in packageStartupMessage and in print.Config printableConf = function(conf) { x = as.list(conf) # This gem here is for R CMD check running examples # where we get an empty config for some reasons? if (length(x) == 0L) { assignConfDefaults() x = as.list(getBatchJobsConf()) } x[setdiff(getConfNames(), names(x))] = "" fmt = paste( "BatchJobs configuration:", " cluster functions: %s", " mail.from: %s", " mail.to: %s", " mail.start: %s", " mail.done: %s", " mail.error: %s", " default.resources: %s", " debug: %s", " raise.warnings: %s", " staged.queries: %s", " max.concurrent.jobs: %s", " fs.timeout: %s", " measure.mem: %s\n", sep = "\n") sprintf(fmt, x$cluster.functions$name, x$mail.from, x$mail.to, x$mail.start, x$mail.done, x$mail.error, convertToShortString(x$default.resources), x$debug, x$raise.warnings, x$staged.queries, x$max.concurrent.jobs, x$fs.timeout, x$measure.mem) } #' @export print.Config = function(x, ...) { cat(printableConf(x)) } #' Load a specific configuration file. #' #' @param conffile [\code{character(1)}]\cr #' Location of the configuration file to load. #' Default is \dQuote{.BatchJobs.conf} in the current working directory. #' @return Invisibly returns a list of configuration settings. #' @family conf #' @export loadConfig = function(conffile = ".BatchJobs.R") { # checks are done in sourceConfFile conf = sourceConfFile(conffile) assignConf(conf) invisible(setClasses(as.list(conf), "Config")) } #' Set and overwrite configuration settings #' #' @param conf [\code{Config} or \code{list}]\cr #' List of configuration parameters as returned by \code{\link{loadConfig}} or \code{\link{getConfig}}. #' @param ... [\code{ANY}]\cr #' Named configuration parameters. Overwrites parameters in \code{conf}, if provided. #' @return Invisibly returns a list of configuration settings. #' @family conf #' @export setConfig = function(conf = list(), ...) { if (!is.list(conf) && !inherits(conf, "Config")) stopf("Argument 'conf' must be of class 'list' or 'Config', not %s", head(conf, 1L)) overwrites = insert(conf, list(...)) if (length(overwrites) == 0L) return(invisible(getConfig())) if (!isProperlyNamed(overwrites)) stopf("All configuration arguments in '...' must be properly named") checkConf(overwrites) conf = insert(as.list(getBatchJobsConf()), overwrites) assignConf(as.environment(conf)) invisible(setClasses(conf, "Config")) } #' Returns a list of BatchJobs configuration settings #' #' @return \code{list} of current configuration variables with classs \dQuote{Config}. #' @family conf #' @export getConfig = function() { setClasses(as.list(getBatchJobsConf()), "Config") } BatchJobs/R/writeFiles.R0000644000176200001440000000257212571265761014574 0ustar liggesuserswriteFiles = function(reg, cf, ids, chunks.as.arrayjobs, resources.timestamp, disable.mail, staged, delay) { ### write r script files template = paste( "Sys.sleep(%f)", "options(BatchJobs.on.slave = TRUE, BatchJobs.resources.path = '%s')", "library(BatchJobs)", "res = BatchJobs:::doJob(", "\treg = loadRegistry('%s'),", "\tids = c(%s),", "\tmultiple.result.files = %s,", "\tstaged = %s,", "\tdisable.mail = %s,", "\tfirst = %iL,", "\tlast = %iL,", "\tarray.id = %s)", "BatchJobs:::setOnSlave(FALSE)", sep = "\n") first = head(ids, 1L) last = tail(ids, 1L) # print the constant arguments (of length 1) into the template resources.path = getResourcesFilePath(reg, resources.timestamp) array.str = if (chunks.as.arrayjobs) sprintf("Sys.getenv(\"%s\", NA)", cf$getArrayEnvirName()) else NA template = sprintf(template, delay, resources.path, reg$file.dir, collapse(paste0(ids, "L")), reg$multiple.result.files, staged, disable.mail, first, last, array.str) r.file = getRScriptFilePath(reg, first) cat(template, file = r.file) ### if staged is FALSE, also write jobs to file system if (staged) { job.file = getJobFile(reg, first) saveRDS(getJobs(reg, ids, check.ids = FALSE), file = job.file) } else { job.file = character(0L) } invisible(c(r.file, job.file)) } BatchJobs/R/debugMulticore.R0000644000176200001440000000527312413244720015415 0ustar liggesusers#' Helper function to debug multicore mode. #' #' @description #' Useful in case of severe errors. #' Tries different operations of increasing difficulty #' and provides debug output on the console #' #' @param r.options [\code{list}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @return Nothing. #' @family debug #' @export debugMulticore = function(r.options) { if (missing(r.options)) r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file") conf = getBatchJobsConf() conf$debug = TRUE conf$mail.start = conf$mail.done = conf$mail.error = "none" rhome = R.home() messagef("*** System info: ***") print(Sys.info()) catf("\n") messagef("*** which R: ***") res = runOSCommandLinux("which", "R") messagef("which R result: %s", res$output) catf("\n") messagef("*** Find helper script: ***") script = findHelperScriptLinux(rhome = rhome, r.options = r.options) messagef("Find helper script result: %s", script) catf("\n") messagef("*** Auto-detecting ncpus: ***") worker = makeWorkerLocalLinux(r.options = r.options, script = script, ncpus = 1) ncpus = runWorkerCommand(worker, "number-of-cpus") messagef("Auto-detecting ncpus result: %s", ncpus) catf("\n") messagef("*** Query worker status: ***") res = runWorkerCommand(worker, "status", args = "") messagef("Query worker status result: %s", res) catf("\n") messagef("*** Submitting 1 job: ***") conf$cluster.functions = makeClusterFunctionsMulticore() fd = tempfile() reg = makeRegistry(id = "debug_multicore", file.dir = fd, sharding = FALSE) batchMap(reg, identity, 1) submitJobs(reg) Sys.sleep(3) messagef("Submitting 1 job result: %i", loadResult(reg, 1)) messagef("Query worker status:") res = runWorkerCommand(worker, "status", args = reg$file.dir) messagef("Query worker status result: %s", res) catf("\n") messagef("*** Killing 2 jobs: ***") fd = tempfile() reg = makeRegistry(id = "debug_multicore", file.dir = fd, sharding = FALSE) f = function(i) if(i <= 1) i else f(i-1) + f(i-2) xs = 50 + seq(1,2) ids = 1:2 batchMap(reg, f, xs) submitJobs(reg) Sys.sleep(3) messagef("Query worker status:") res = runWorkerCommand(worker, "status", args = reg$file.dir) messagef("Query worker status result: %s", res) messagef("Running jobs: %s", collapse(findRunning(reg))) killJobs(reg, ids) messagef("Query worker status:") res = runWorkerCommand(worker, "status", args = reg$file.dir) messagef("Query worker status result: %s", res) messagef("Running jobs: %s", collapse(findRunning(reg))) catf("\n") } BatchJobs/R/clusterFunctionsHelpers.R0000644000176200001440000001143613207234653017344 0ustar liggesusers#' Cluster functions helper: Read in your brew template file. #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Simply reads your template and returns it as a character vector. #' If you do this in the constructor of your cluster functions once, you can avoid this #' repeated file access later on. #' #' @param template.file [\code{character(1)}]\cr #' File path. #' @return [\code{character}]. #' @export cfReadBrewTemplate = function(template.file) { assertFileExists(template.file, "r") tmpl = readLines(template.file) if (length(tmpl) == 0L) stopf("Error reading template '%s' or empty template", template.file) collapse(tmpl, "\n") } #' Cluster functions helper: Brew your template into a job description file. #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Calls brew silently on your template, any error will lead to an exception. #' If debug mode is turned on in the configuration, the file is stored at the same place as the #' corresponding R script in the \dQuote{jobs}-subdir of your files directory, #' otherwise in the temp dir via \code{\link{tempfile}}. #' #' @param conf [\code{environment}]\cr #' BatchJobs configuration. #' @param template [\code{character(1)}]\cr #' Job desfription template as a char vecrtor, #' possibly read in via \code{\link{cfReadBrewTemplate}}. #' @param rscript [\code{character(1)}]\cr #' File path to you your corresponding R script for the job. #' @param extension [\code{character(1)}]\cr #' Extension for the job description file, e.g. \dQuote{pbs}. #' @return [\code{character(1)}]. File path of result. #' @export cfBrewTemplate = function(conf, template, rscript, extension) { assertEnvironment(conf) assertString(template) assertString(rscript) assertString(extension) if (conf$debug) { # if debug, place in jobs dir outfile = sub("\\.R$", sprintf(".%s", extension), rscript) } else { outfile = tempfile("template") } pf = parent.frame() old = getOption("show.error.messages") on.exit(options(show.error.messages = old)) options(show.error.messages = FALSE) z = suppressAll(try(brew(text = template, output = outfile, envir = pf), silent = TRUE)) if (is.error(z)) stopf("Error brewing template: %s", as.character(z)) waitForFiles(outfile, conf$fs.timeout) return(outfile) } #' Cluster functions helper: Handle an unknown error during job submission. #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Simply constructs a \code{\link{SubmitJobResult}} object with status code 101, #' NA as batch job id and an informative error message containing the output of the OS command in \code{output}. #' #' @param cmd [\code{character(1)}]\cr #' OS command used to submit the job, e.g. qsub. #' @param exit.code [\code{integer(1)}]\cr #' Exit code of the OS command, should not be 0. #' @param output [\code{character}]\cr #' Output of the OS command, hopefully an informative error message. #' If these are mutiple lines in a vector, they are automatically pasted together. #' @return [\code{\link{SubmitJobResult}}]. #' @export cfHandleUnknownSubmitError = function(cmd, exit.code, output) { assertString(cmd) exit.code = asInt(exit.code) assertCharacter(output, any.missing = FALSE) msg = sprintf("%s produced exit code %i; output %s", cmd, exit.code, collapse(output, sep = "\n")) makeSubmitJobResult(status = 101L, batch.job.id = NA_character_, msg = msg) } #' Cluster functions helper: Kill a batch job via OS command #' #' @description #' This function is only intended for use in your own cluster functions implementation. #' #' Calls the OS command to kill a job via \code{system} like this: \dQuote{cmd batch.job.id}. #' If the command returns an exit code > 0, the command is repeated #' after a 1 second sleep \code{max.tries-1} times. #' If the command failed in all tries, an exception is generated. #' #' @param cmd [\code{character(1)}]\cr #' OS command, e.g. \dQuote{qdel}. #' @param batch.job.id [\code{character(1)}]\cr #' Id of the batch job on the batch system. #' @param max.tries [\code{integer(1)}]\cr #' Number of total times to try execute the OS command in cases of failures. #' Default is \code{3}. #' @return Nothing. #' @export cfKillBatchJob = function(cmd, batch.job.id, max.tries = 3L) { assertString(cmd) assertString(batch.job.id) max.tries = asCount(max.tries) assertCount(max.tries) for (tmp in seq_len(max.tries)) { res = runOSCommandLinux(cmd, batch.job.id, stop.on.exit.code = FALSE) if (res$exit.code == 0L) return() Sys.sleep(1) } stopf("Really tried to kill job, but could not do it. batch job id is %s.\nMessage: %s", batch.job.id, collapse(res$output, sep = "\n")) } BatchJobs/R/showLog.R0000644000176200001440000000511512623105424014060 0ustar liggesusers#' Display the contents of a log file. #' #' @description #' Display the contents of a log file, useful in case of errors. #' #' Note this rare special case: When you use chunking, submit some jobs, some jobs fail, #' then you resubmit these jobs again in different chunks, the log files will contain the log #' of the old, failed job as well. \code{showLog} tries to jump to the correct part #' of the new log file with a supported pager. #' #' @template arg_reg #' @param id [\code{integer(1)}]\cr #' Id of selected job. #' Default is first id in registry. #' @param pager [\code{any}]\cr #' Pager to use to display the log. Defaults to \code{getOption("pager")}. #' This option is passed to \code{file.show} and is highly OS dependant and GUI dependant. #' If either R's pager or the environment variable \dQuote{PAGER} is set to \dQuote{less} #' or \dQuote{vim}, the correct part of the log file will be shown. #' Otherwise you find information about the correct part in the beginning of the displayed file. #' @return [\code{character(1)}]. Invisibly returns path to log file. #' @family debug #' @export showLog = function(reg, id, pager = getOption("pager")) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(id)) { id = dbGetJobId(reg) if (length(id) == 0L) stop("No jobs in registry!") } else { id = checkIds(reg, id, len = 1L) } fn = getLogFiles(reg, id) if (!file.exists(fn)) stopf("Log file does not exist: %s", fn) exact1 = sprintf("Executing jid=%i", id) exact2 = sprintf("########## %s ##########", exact1) header = c(sprintf("Showing log file for job with id=%i", id), sprintf("The string '%s' indicates start of requested job", exact1)) if (!is.function(pager)) { sys.pager = Sys.getenv("PAGER") if ((grepl("/pager$", pager) || pager == "internal") && sys.pager != "") { # prefer the system pager pager = sys.pager } bn = basename(stri_split_regex(pager, "\\s+")[[1L]][1L]) # check for less or vim as pager # if we find the pattern, we jump to the matching line if (bn %in% c("less", "vim")) { pos = grep(exact2, readLines(fn), fixed = TRUE) if (length(pos) == 1L) { pos = pos + length(header) + 1L if (pager == "less") pager = function(files, header, title, delete.file) system2("less", c(sprintf("+%ig", pos), files)) else pager = function(files, header, title, delete.file) system2("vim", c(sprintf("+%i", pos), files)) } } } file.show(fn, pager = pager, header = collapse(header, sep = "\n")) invisible(fn) } BatchJobs/R/debugSSH.R0000644000176200001440000000772712623105424014115 0ustar liggesusers#' @title Helper function to debug SSH mode. #' #' @description #' Useful in case of configuration problems. #' Tries different operations of increasing difficulty #' and provides debug output on the console. #' #' Note that this function does not access nor use information specified for #' your cluster functions in your configuration. #' #' @param nodename [\code{character(1)}]\cr #' Node on which worker should be constructed for the test. #' @param ssh.cmd [\code{character(1)})]\cr #' CLI command to ssh into remote node. #' Default is \dQuote{ssh}. #' @param ssh.args [\code{character}]\cr #' CLI args for \code{ssh.cmd}. #' Default is none. #' @param rhome [\code{character(1)}]\cr #' Path to R installation on the worker. #' \dQuote{} means R installation on the PATH is used, #' of course this implies that it must be on the PATH #' (also for non-interactive shells)! #' Default is \dQuote{}. #' @param r.options [\code{list}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @param dir [\code{character(1)}]\cr #' Path where internally used test registries can be created. #' Note that this must be shared for the worker. #' Default is current working directory. #' @return Nothing. #' @family debug #' @export debugSSH = function(nodename, ssh.cmd = "ssh", ssh.args = character(0L), rhome = "", r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), dir = getwd()) { assertString(ssh.cmd) assertCharacter(ssh.args, any.missing = FALSE) assertString(nodename) assertString(rhome) assertString(dir) conf = getBatchJobsConf() conf$debug = TRUE conf$mail.start = conf$mail.done = conf$mail.error = "none" wd = dir messagef("*** System info on master: ***") print(Sys.info()) catf("\n") messagef("*** which R on slave: ***") res = runOSCommandLinux(cmd = "which", args = "R", ssh = TRUE, ssh.cmd = ssh.cmd, ssh.args = ssh.args, nodename = nodename, stop.on.exit.code = TRUE) messagef("which R result:") print(res) catf("\n") messagef("*** Find helper script on slave: ***") res = findHelperScriptLinux(rhome, r.options, ssh = TRUE, ssh.cmd = ssh.cmd, ssh.args = ssh.args, nodename = nodename) messagef("Find helper script result:") print(res) catf("\n") messagef("*** Auto-detecting ncpus for slave: ***") worker = makeWorkerRemoteLinux(ssh.cmd = ssh.cmd, ssh.args = ssh.args, nodename = nodename, rhome = rhome, r.options = r.options, ncpus = 1) res = runWorkerCommand(worker = worker, command = "number-of-cpus") messagef("Auto-detecting ncpus result:") print(res) catf("\n") queryWorkerStatus = function() { messagef("*** Query worker status: ***") res = runWorkerCommand(worker = worker, command = "status", args = "") messagef("Query worker status result:") message("load n.rprocs n.rprocs.50 n.jobs") print(res) catf("\n") } queryWorkerStatus() messagef("*** Submitting 1 job: ***") ssh.workers = list(makeSSHWorker(ssh.cmd = ssh.cmd, ssh.args = ssh.args, nodename = nodename, rhome = rhome, r.options = r.options)) conf$cluster.functions = do.call(makeClusterFunctionsSSH, ssh.workers) id = "debug_ssh_1" reg = makeRegistry(id = id, file.dir = file.path(dir, id), work.dir = wd, sharding = FALSE) batchMap(reg, identity, 1) submitJobs(reg) Sys.sleep(3) messagef("Submitting 1 job result: %i", loadResult(reg, 1)) queryWorkerStatus() messagef("*** Killing 2 jobs: ***") id = "debug_ssh_2" reg = makeRegistry(id = id, file.dir = file.path(dir, id), work.dir = wd, sharding = FALSE) f = function(i) if(i <= 1) i else f(i-1) + f(i-2) xs = 50 + seq(1,2) ids = 1:2 batchMap(reg, f, xs) submitJobs(reg) Sys.sleep(3) queryWorkerStatus() messagef("Running jobs: %s", collapse(findRunning(reg))) killJobs(reg, ids) queryWorkerStatus() messagef("Running jobs: %s", collapse(findRunning(reg))) catf("\n") } BatchJobs/R/batchApply.R0000644000176200001440000000556212571265761014550 0ustar liggesusers# @title Apply a function on maticies or arrays # # A simple wrapper to batchMap to define jobs as an application of a function. # on margins of matrices or arrays. # # @param reg [\code{\link{Registry}}]\cr # Empty Registry that will store jobs for the mapping. # @param X [\code{\link[base]{matrix}} | \code{\link[base]{array}}]\cr # A matrix or an array. # @param margin [\code{integer(1)}]\cr # Margin of the matrix. 1 for rows, 2 for columns. # @param fun [\code{function}]\cr # Function to map over \code{...}. # @param chunk.size [\code{integer(1)}]\cr # Preferred number of jobs in each chunk. # Can not be used in combination with \code{n.chunks}. # Default is \code{chunk.size = 1} if \code{n.chunks} is also not provided and # results in \code{\link{nrow}} or \code{\link{ncol}} jobs, respectively. # @param n.chunks [\code{integer(1)}]\cr # Preferred number of chunks. # Can not be used in combination with \code{chunk.size}. # @param ... [any]\cr # Additional arguments passed to \code{fun}. # @return [\code{integer}]. Ids of created jobs. # @examples # reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) # X = matrix(1:16, 4) # # Define two jobs to calculate the row sums: # batchApply(reg, X, 1, sum, n.chunks = 2) # submitJobs(reg) # waitForJobs(reg) # reduceResultsVector(reg, use.names = FALSE) == rowSums(X) batchApply = function(reg, X, margin, fun, chunk.size, n.chunks, ..., use.names = FALSE) { checkRegistry(reg, strict = TRUE, writeable = TRUE) assert(checkMatrix(X), checkArray(X)) dX = dim(X) margin = asInt(margin, lower = 1L, upper = length(dX)) fun = checkUserFunction(fun) more.args = list(...) checkMoreArgs(more.args, c(".X", ".inds", ".user.fun")) assertFlag(use.names) inds = chunk(seq_len(dX[margin]), chunk.size = chunk.size, n.chunks = n.chunks, shuffle = FALSE) if (use.names && !is.null(dimnames(X)[[margin]])) names(inds) = dimnames(X)[[margin]] more.args = c(more.args, list(.X = aperm(X, c(margin, seq_along(dX)[-margin])), .user.fun = fun)) batchMap(reg, batchApplyWrapper, .inds = inds, more.args = more.args, use.names = use.names) } batchApplyWrapper = function(.inds, .X, .user.fun, ...) { apply(.X[.inds,, drop = FALSE], 1L, .user.fun, ...) } # FIXME: we should split the matrix on the master first. # here is a piece of code which allows this, but there is no sufficient # mechanism in BJ to find these files again on the slave? # maybe store in work.dir? # splitX = function(X, margin, chunks) { # n = length(chunks) # X = aperm(X, c(margin, seq_along(dim(X))[-margin])) # dn = file.path(reg$file.dir, "data-chunks") # dir = checkDir(dn, create = TRUE) # dest = file.path(dn, sprintf("chunk-%i.RData", seq_len(n))) # # info("Splitting input into %i files ...", n) # for (i in seq_len(n)) { # save2(.X = X[chunks[[i]],, drop = FALSE], file = dest[i]) # } # dest # } BatchJobs/R/batchExpandGrid.R0000644000176200001440000000405012571265761015477 0ustar liggesusers#' @title Map function over all combinations. #' #' @description #' Maps an n-ary-function over a list of all combinations which are given by some vectors. #' Internally \code{\link{expand.grid}} is used to compute the combinations, then #' \code{\link{batchMap}} is called. #' #' @param reg [\code{\link{Registry}}]\cr #' Empty Registry that will store jobs for the mapping. #' @param fun [\code{function}]\cr #' Function to map over the combinations. #' @param ... [any]\cr #' Vectors that are used to compute all combinations. #' If the arguments are named, these names are used to bind to arguments of \code{fun}. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return [\code{data.frame}]. Expanded grid of combinations produced by \code{\link{expand.grid}}. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x, y, z) x * y + z #' # lets store the param grid #' grid = batchExpandGrid(reg, f, x = 1:2, y = 1:3, more.args = list(z = 10)) #' submitJobs(reg) #' waitForJobs(reg) #' y = reduceResultsVector(reg) #' # later, we can always access the param grid like this #' grid = getJobParamDf(reg) #' cbind(grid, y = y) batchExpandGrid = function(reg, fun, ..., more.args = list()) { checkRegistry(reg, strict = TRUE, writeable = TRUE) assertFunction(fun) args = list(...) ns = names(args) if (length(args) == 0L) return(invisible(integer(0L))) if(!all(vlapply(args, is.vector))) stop("All args in '...' must be vectors!") checkMoreArgs(more.args) reserved = c("KEEP.OUT.ATTRS", "stringsAsFactors") if (any(reserved %in% ns)) stopf("You cannot use the reserved arg names %s in ... args!", collapse(reserved)) args$KEEP.OUT.ATTRS = FALSE args$stringsAsFactors = FALSE grid = do.call(expand.grid, args) if (is.null(ns)) colnames(grid) = NULL do.call(batchMap, c(as.list(grid), list(reg = reg, fun = fun, more.args = more.args))) return(setRowNames(grid, as.character(getJobIds(reg)))) } BatchJobs/R/filenames.R0000644000176200001440000001307712571265761014424 0ustar liggesuserscheckDir = function(path, create = FALSE, check.empty = FALSE, check.posix = FALSE, msg = FALSE) { if (create) { if (file.exists(path)) { if (!isDirectory(path)) stop("File in place where dir should be created: ", path) } else { if (msg) info("Creating dir: %s", path) if (!dir.create(path, recursive = TRUE)) stopf("Could not create dir: %s", path) } } if (!isDirectory(path)) stopf("Directory '%s' does not exists", path) if (!is.accessible(path)) stopf("Directory '%s' is not readable/writable!", path) if (check.empty && any(list.files(path, all.files = TRUE) %nin% c(".", ".."))) stopf("Directory '%s' does not seem to be empty: %s", path, paste(setdiff(list.files(path, all.files = TRUE), c(".", "..")), collapse=", ")) if (check.posix && getOption("BatchJobs.check.posix", TRUE)) { path.abs = sanitizePath(path, make.absolute = TRUE) if(! grepl("^[[:alnum:]:/_.-]+$", path.abs)) stopf("Directory '%s' contains characters that are not fully portable according to POSIX standards. Allowed: a-z A-Z 0-9 : / . - _", path.abs) } } checkDirs = function(paths, ...) { vcapply(paths, checkDir) } createShardedDirs = function(reg, ids) { if (reg$sharding) { lapply(getJobDirs(reg, ids, unique = TRUE), checkDir, create = TRUE) } } # tests a directory for read and write permissions # uses a heuristic for windows is.accessible = function(path) { if (isWindows()) { # Workaround: No POSIX file system informations available, use a heuristic rnd = basename(tempfile("")) tf1 = file.path(path, sprintf("test_write_access_file_%s", rnd)) td1 = file.path(path, sprintf("test_write_access_dir_%s", rnd)) tf2 = file.path(td1, "test_write_access_subfile") td2 = file.path(td1, "test_write_access_subdir") # on exit, try to clean up the mess we might have caused on.exit(try(removeDirs(c(tf2, td2, tf1, td1), recursive = TRUE))) fileCreate = function(pathname) { if (!file.create(pathname)) return(FALSE) msg0 = "Hello world!" cat(file=pathname, msg0) msg = readLines(pathname, warn=FALSE) if (identical(msg, msg0)) return(TRUE) stop(sprintf("Created test file does not contain expected content ('%s'): '%s'", msg, msg0)) } # perform the checks ok = try({ fileCreate(tf1) && dir.create(td1) && dir.create(td2) && length(list.files(td1)) == 1L && fileCreate(tf2) && removeDirs(c(tf2, td2, tf1, td1), recursive = TRUE) }) if (is.error(ok) || !isTRUE(ok)) return(FALSE) # we don't need the on exit handler anymore, everything should be fine on.exit(NULL) return(TRUE) } return(file.access(path, mode = c(2L, 4L)) == 0L) } # a more trusty version than file.exists() fileExists = function(path) { if (file.exists(path)) return(TRUE) ## Double check with what dir() reports, because (at least) on Windows ## there seems to be a delay between a file/directory being removed ## and the directory listings being updated. /HB 2015-02-08 filename = basename(path) dirs = dir(path = dirname(path), all.files = TRUE, include.dirs = TRUE) return(filename %in% dirs) } # a more robust and insisting version than unlink() removeDirs = function(paths, recursive=FALSE, ..., must.work=TRUE, max.tries=30L, interval=0.2) { exists = vlapply(paths, fileExists) for (ii in which(exists)) { path = paths[ii] for (kk in seq_len(max.tries)) { unlink(path, recursive=recursive, ...) exists[ii] = fileExists(path) if (!exists[ii]) break if (kk < max.tries) Sys.sleep(interval) } } exists = vlapply(paths, fileExists) failed = path[exists] if (must.work && length(failed) > 0L) stop("Failed to remove files/directories: ", paste(sQuote(path), collapse=", ")) return(!exists) } isPathFromRoot = function(path) { (isWindows() & grepl("^[[:alpha:]]:", path)) | grepl("^[/\\]", path) } getJobDirs = function(reg, ids, unique = FALSE) { if (reg$sharding) { shards = sprintf("%02i", ids %% 100L) if(unique) shards = unique(shards) return(file.path(reg$file.dir, "jobs", shards)) } file.path(reg$file.dir, "jobs") } getFilePaths = function(reg, id, suffix, ext) { fn = sprintf("%i%s.%s", id, ifelse(is.null(suffix), "", paste0("-", suffix)), ext) file.path(getJobDirs(reg, id), fn) } getJobParentDir = function(file.dir) file.path(file.dir, "jobs") getFunDir = function(file.dir) file.path(file.dir, "functions") getFunFilePath = function(reg, fun.id) file.path(getFunDir(reg$file.dir), sprintf("%s.RData", fun.id)) getJobFile = function(reg, id) getFilePaths(reg, id, NULL, "rds") getConfFilePath = function(reg) file.path(reg$file.dir, "conf.RData") getRegistryFilePath = function(file.dir) file.path(file.dir, "registry.RData") getRScriptFilePath = function(reg, id) getFilePaths(reg, id, NULL, "R") getLogFilePath = function(reg, id) getFilePaths(reg, id, NULL, "out") getResultFilePath = function(reg, id, part = NA_character_) { s = if (is.na(part)) "result" else paste0("result-", part) getFilePaths(reg, id, s, "RData") } getResourcesDir = function(file.dir) file.path(file.dir, "resources") getResourcesFilePath = function(reg, timestamp) file.path(getResourcesDir(reg$file.dir), sprintf("resources_%i.RData", timestamp)) getPendingDir = function(file.dir) file.path(file.dir, "pending") getExportDir = function(file.dir) file.path(file.dir, "exports") getPendingFile = function(reg, type, id, char = .OrderChars[type]) { file.path(getPendingDir(reg$file.dir), sprintf("pending_%s_%s_%i.sql", char, type, id)) } BatchJobs/R/Packages.R0000644000176200001440000000360712623105424014160 0ustar liggesusers#' @title Add packages to registry. #' #' @description #' Mutator function for \code{packages} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param packages [\code{character}]\cr #' Packages to add to registry. #' @template ret_reg_mut #' @family exports #' @export addRegistryPackages = function(reg, packages) { checkRegistry(reg, writeable = TRUE) assertCharacter(packages, any.missing = FALSE) packages = setdiff(packages, names(reg$packages)) packages = setNames(lapply(packages, function(pkg) list(version = packageVersion(pkg))), packages) reg$packages = c(reg$packages, packages) saveRegistry(reg) } #' @title Remove packages from registry. #' #' @description #' Mutator function for \code{packages} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param packages [\code{character}]\cr #' Packages to remove from registry. #' @template ret_reg_mut #' @family exports #' @export removeRegistryPackages = function(reg, packages) { checkRegistry(reg, writeable = TRUE) assertCharacter(packages, any.missing = FALSE) mandatory = names(filterNull(extractSubList(reg$packages, "mandatory"))) reg$packages = reg$packages[names(reg$packages) %nin% setdiff(packages, mandatory)] saveRegistry(reg) } #' @title Set packages for a registry. #' #' @description #' Mutator function for \code{packages} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param packages [\code{character}]\cr #' Character vector of package names to load. #' @template ret_reg_mut #' @family exports #' @export setRegistryPackages = function(reg, packages) { checkRegistry(reg, writeable = TRUE) assertCharacter(packages, any.missing = FALSE) mandatory = names(filterNull(extractSubList(reg$packages, "mandatory"))) packages = unique(c(mandatory, packages)) reg$packages = setNames(lapply(packages, function(pkg) list(version = packageVersion(pkg))), packages) saveRegistry(reg) } BatchJobs/R/getErrorMessages.R0000644000176200001440000000107112623105424015714 0ustar liggesusers#' Get error messages of jobs. #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs with errors. #' @return [\code{character}]. Error messages for jobs as character vector\cr #' \code{NA} if job has terminated successfully. #' @family debug #' @export getErrorMessages = function(reg, ids) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) ids = dbFindErrors(reg) else ids = checkIds(reg, ids) tab = dbGetErrorMsgs(reg, ids, filter = FALSE) setNames(tab$error, tab$job_id) } BatchJobs/R/loadResults.R0000644000176200001440000000623112623105424014737 0ustar liggesusers#' Loads result files for id vector. #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all done jobs. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param simplify [\code{logical(1)}]\cr #' Should the result be simplified to a vector, matrix or higher dimensional array if possible? #' Default is \code{TRUE}. #' @param use.names [\code{character(1)}]\cr #' Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) #' or return a unnamed result (\dQuote{none}). #' Default is \code{ids}. #' @param missing.ok [\code{logical(1)}]\cr #' If \code{FALSE} an error is thrown if the results are not found. #' Otherwise missing results are imputed to \code{NULL}. #' Default is \code{FALSE}. #' @param impute.val [any]\cr #' The value to return when no result is available. #' Defaults to \code{NULL} (the previous behavior) #' @return [\code{list}]. Results of jobs as list, possibly named by ids. #' @seealso \code{\link{reduceResults}} #' @export loadResults = function(reg, ids, part = NA_character_, simplify = FALSE, use.names = "ids", missing.ok = FALSE, impute.val = NULL) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) { ids = dbFindDone(reg) } else { ids = checkIds(reg, ids) } checkPart(reg, part) assertFlag(simplify) use.names = convertUseNames(use.names) assertFlag(missing.ok) res = getResults(reg, ids, part, missing.ok, impute.val = impute.val) names(res) = switch(use.names, "none" = NULL, "ids" = as.character(ids), "names" = dbGetJobNames(reg, ids)) if(simplify && length(res) > 0L) res = simplify2array(res, higher = (simplify=="array")) return(res) } getResults = function(reg, ids, part = NA_character_, missing.ok = FALSE, impute.val = NULL) { if (reg$multiple.result.files) { read.files = function(id, dir, pattern) { fns = list.files(dir, pattern, full.names = TRUE) found.parts = sub(".+-(.+)\\.RData$", "\\1", basename(fns)) if(length(found.parts) == 0L) { if (missing.ok) return(list()) stop("No partial result files found for job with id ", id) } setNames(lapply(fns, load2, "result"), found.parts) } dirs = getJobDirs(reg, ids) if(length(part) == 1L && is.na(part)) { patterns = sprintf("^%i-result-.+\\.RData$", ids) } else { patterns = sprintf("^%i-result-(%s)\\.RData$", ids, collapse(part, "|")) } return(mapply(read.files, id = ids, dir = dirs, pattern = patterns, SIMPLIFY = FALSE, USE.NAMES = FALSE)) } fns = getResultFilePath(reg, ids, part) miss = !file.exists(fns) if (any(miss)) { if (!missing.ok) stopf("Some job result files do not exist, showing up to first 10:\n%s", collapse(head(fns[miss], 10L), "\n")) ret = rep.int(list(impute.val), times = length(ids)) ret = replace(ret, !miss, lapply(fns[!miss], load2, "result")) } else { ret = lapply(fns, load2, "result") } ret } BatchJobs/R/setJobNames.R0000644000176200001440000000164112571265761014665 0ustar liggesusers#' Set job names. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @param jobnames [\code{character}]\cr #' Character vector with length equal to \code{length(ids)}. #' \code{NA} removes the names stored in the registry. #' A single \code{NA} is replicated to match the length of ids provided. #' @return Named vector of job ids. #' @export setJobNames = function(reg, ids, jobnames) { checkRegistry(reg, strict = TRUE, writeable = TRUE) if (missing(ids)) ids = dbGetJobIds(reg) else ids = checkIds(reg, ids) if (isScalarNA(jobnames)) jobnames = rep.int(NA_character_, length(ids)) else assertCharacter(jobnames, any.missing = FALSE) if (length(ids) != length(jobnames)) stop("Arguments 'ids' and 'jobnames' must have same length") dbSetJobNames(reg, ids, jobnames) setNames(ids, jobnames) } BatchJobs/R/sanitizePath.R0000644000176200001440000000166612413244720015110 0ustar liggesusers#' Sanitize a path #' #' @description #' Replaces backward slashes with forward slashes and optionally #' normalizes the path. #' #' @param path [\code{character}]\cr #' Vector of paths to sanitize. #' @param make.absolute [\code{logical}]\cr #' If \code{TRUE} convert to an absolute path. #' @param normalize.absolute [\code{logical}]\cr #' Also call \code{\link[base]{normalizePath}} on absolute paths? #' This will immediately resolve symlinks. #' @return \code{character} with sanitized paths. #' @export sanitizePath = function(path, make.absolute = TRUE, normalize.absolute = FALSE) { assertCharacter(path, any.missing = FALSE) assertFlag(make.absolute) assertFlag(normalize.absolute) if (make.absolute) { normalize = if (normalize.absolute) rep.int(TRUE, length(path)) else !isPathFromRoot(path) path[normalize] = normalizePath(path[normalize], mustWork = FALSE, winslash = "/") } gsub("\\", "/", path, fixed = TRUE) } BatchJobs/R/getJobLocation.R0000644000176200001440000000037212623105424015341 0ustar liggesusers#' Get the physical location of job files on the hard disk. #' @template arg_reg #' @template arg_ids #' @export #' @return [\code{character}] Vector of directories. getJobLocation = function(reg, ids) { checkRegistry(reg) getJobDirs(reg, ids) } BatchJobs/R/findStatus.R0000644000176200001440000001065712623105424014571 0ustar liggesusersfindState = function(reg, ids, fun, negate, limit) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (!missing(ids)) ids = checkIds(reg, ids) if (!is.null(limit)) assertCount(limit) fun(reg, ids, negate, limit) } #' Find jobs depending on computional state. #' #' \code{findDone}: Find jobs which succesfully terminated. #' @template arg_reg #' @param ids [\code{integer}]\cr #' Subset of job ids to restrict the result to. #' Default is all jobs. #' @param limit [\code{integer(1)}]\cr #' Limit the number of returned ids. #' Default is all ids. #' @return [\code{integer}]. Ids of jobs. #' @export #' @rdname findState findDone = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindDone, FALSE, limit) } #' \code{findNotDone}: Find jobs for which results are still missing. #' @export #' @rdname findState findNotDone = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindDone, TRUE, limit) } #' \code{findMissingResults}: Deprecated. Alias for findNotDone. #' @export #' @rdname findState findMissingResults = function(reg, ids, limit = NULL) { findNotDone(reg, ids, limit) } #' \code{findErrors}: Find jobs where errors occured. #' @export #' @rdname findState findErrors = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindErrors, FALSE, limit) } #' \code{findNotErrors}: Find jobs where no errors occured. #' @export #' @rdname findState findNotErrors = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindErrors, TRUE, limit) } #' \code{findTerminated}: Find jobs which have terminated (done / error). #' @export #' @rdname findState findTerminated = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindTerminated, FALSE, limit) } #' \code{findNotTerminated}: Find jobs which have not terminated (not done / no error). #' @export #' @rdname findState findNotTerminated = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindTerminated, TRUE, limit) } #' \code{findSubmitted}: Find jobs which have been submitted. #' @export #' @rdname findState findSubmitted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindSubmitted, FALSE, limit) } #' \code{findNotSubmitted}: Find jobs which have not been submitted. #' @export #' @rdname findState findNotSubmitted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindSubmitted, TRUE, limit) } #' \code{findOnSystem}: Find jobs which are present on the batch system at the moment. #' @export #' @rdname findState findOnSystem = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindOnSystem, FALSE, limit) } #' \code{findNotOnSystem}: Find jobs which are not present on the batch system at the moment. #' @export #' @rdname findState findNotOnSystem = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindOnSystem, TRUE, limit) } #' \code{findRunning}: Find jobs which are running. #' @export #' @rdname findState findRunning = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindRunning, FALSE, limit) } #' \code{findNotRunning}: Find jobs which are not running. #' @export #' @rdname findState findNotRunning = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindRunning, TRUE, limit) } #' \code{findStarted}: Find jobs which have been started on the batch system. #' @export #' @rdname findState findStarted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindStarted, FALSE, limit) } #' \code{findStarted}: Find jobs which have not been started on the batch system. #' \code{findNotRunning}: Find jobs which are not running. #' @export #' @rdname findState findNotStarted = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindStarted, TRUE, limit) } #' \code{findExpired}: Find jobs where walltime was probably hit. #' Right now the heuristic is as follows: #' Find all jobs that have started, did not abort with an error, #' did not complete with a result and are not submitted or running anymore. #' Note that this heuristic does not include jobs the scheduler looses before starting. #' @export #' @rdname findState findExpired = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindExpiredJobs, FALSE, limit) } #' \code{findDisappeared}: Find jobs which disappeared from the system. #' Right now the heuristic is as follows: #' Find all jobs that are submitted but not started nor on the system anymore. #' @export #' @rdname findState findDisappeared = function(reg, ids, limit = NULL) { findState(reg, ids, dbFindDisappeared, FALSE, limit) } BatchJobs/R/submitJobs.R0000644000176200001440000002555212623105424014566 0ustar liggesusers#' @title Submit jobs or chunks of jobs to batch system via cluster function. #' #' @description #' If the internal submit cluster function completes successfully, the \code{retries} #' counter is set back to 0 and the next job or chunk is submitted. #' If the internal submit cluster function returns a fatal error, the submit process #' is completely stopped and an exception is thrown. #' If the internal submit cluster function returns a temporary error, the submit process #' waits for a certain time, which is determined by calling the user-defined #' \code{wait}-function with the current \code{retries} counter, the counter is #' increased by 1 and the same job is submitted again. If \code{max.retries} is #' reached the function simply terminates. #' #' Potential temporary submit warnings and errors are logged inside your file #' directory in the file \dQuote{submit.log}. #' To keep track you can use \code{tail -f [file.dir]/submit.log} in another #' terminal. #' #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Vector for job id or list of vectors of chunked job ids. #' Only corresponding jobs are submitted. Chunked jobs will get executed #' sequentially as a single job for the scheduler. #' Default is all jobs which were not yet submitted to the batch system. #' @param resources [\code{list}]\cr #' Required resources for all batch jobs. The elements of this list #' (e.g. something like \dQuote{walltime} or \dQuote{nodes} are defined by your template job file. #' Defaults can be specified in your config file. #' Default is empty list. #' @param wait [\code{function(retries)}]\cr #' Function that defines how many seconds should be waited in case of a temporary error. #' Default is exponential back-off with \code{10*2^retries}. #' @param max.retries [\code{integer(1)}]\cr #' Number of times to submit one job again in case of a temporary error #' (like filled queues). Each time \code{wait} is called to wait a certain #' number of seconds. #' Default is 10 times. #' @param chunks.as.arrayjobs [\code{logical(1)}]\cr #' If ids are passed as a list of chunked job ids, execute jobs in a chunk #' as array jobs. Note that your scheduler and your template must be adjusted to #' use this option. Default is \code{FALSE}. #' @param job.delay [\code{function(n, i)} or \code{logical(1)}]\cr #' Function that defines how many seconds a job should be delayed before it starts. #' This is an expert option and only necessary to change when you want submit #' extremely many jobs. We then delay the jobs a bit to write the submit messages as #' early as possible to avoid writer starvation. #' \code{n} is the number of jobs and \code{i} the number of the ith job. #' The default function used with \code{job.delay} set to \code{TRUE} is no delay for #' 100 jobs or less and otherwise \code{runif(1, 0.1*n, 0.2*n)}. #' If set to \code{FALSE} (the default) delaying jobs is disabled. #' @template arg_progressbar #' @return [\code{integer}]. Vector of submitted job ids. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' submitJobs(reg) #' waitForJobs(reg) #' #' # Submit the 10 jobs again, now randomized into 2 chunks: #' chunked = chunk(getJobIds(reg), n.chunks = 2, shuffle = TRUE) #' submitJobs(reg, chunked) submitJobs = function(reg, ids, resources = list(), wait, max.retries = 10L, chunks.as.arrayjobs = FALSE, job.delay = FALSE, progressbar = TRUE) { ### helper function to calculate the delay getDelays = function(cf, job.delay, n) { if (is.logical(job.delay)) { if (job.delay && n > 100L && cf$name %nin% c("Interactive", "Multicore", "SSH")) { return(runif(n, n*0.1, n*0.2)) } return(delays = rep.int(0, n)) } vnapply(seq_along(ids), job.delay, n = n) } ### argument checks on registry and ids checkRegistry(reg, writeable = TRUE) syncRegistry(reg) if (missing(ids)) { ids = dbFindSubmitted(reg, negate = TRUE) if (length(ids) == 0L) { info("All jobs submitted, nothing to do!") return(invisible(integer(0L))) } } else { if (is.list(ids)) { ids = lapply(ids, checkIds, reg = reg, check.present = FALSE) dbCheckJobIds(reg, unlist(ids)) } else if(is.numeric(ids)) { ids = checkIds(reg, ids) } else { stop("Parameter 'ids' must be a integer vector of job ids or a list of chunked job ids (list of integer vectors)!") } } ### initialization of some helping vars conf = getBatchJobsConf() cf = getClusterFunctions(conf) limit.concurrent.jobs = is.finite(conf$max.concurrent.jobs) n = length(ids) ### argument checks for other parameters assertList(resources) resources = resrc(resources) if (missing(wait)) wait = function(retries) 10 * 2^retries else assertFunction(wait, "retries") if (is.finite(max.retries)) max.retries = asCount(max.retries) assertFlag(chunks.as.arrayjobs) if (chunks.as.arrayjobs && is.na(cf$getArrayEnvirName())) { warningf("Cluster functions '%s' do not support array jobs, falling back on chunks", cf$name) chunks.as.arrayjobs = FALSE } assert(checkFlag(job.delay), checkFunction(job.delay, c("n", "i"))) assertFlag(progressbar) if (!is.null(cf$listJobs)) { ### check for running jobs ids.intersect = intersect(unlist(ids), dbFindOnSystem(reg, unlist(ids))) if (length(ids.intersect) > 0L) { stopf("Some of the jobs you submitted are already present on the batch system! E.g. id=%i.", ids.intersect[1L]) } } if (limit.concurrent.jobs && (cf$name %in% c("Interactive", "Local", "Multicore", "SSH") || is.null(cf$listJobs))) { warning("Option 'max.concurrent.jobs' is enabled, but your cluster functions implementation does not support the listing of system jobs.\n", "Option disabled, sleeping 5 seconds for safety reasons.") limit.concurrent.jobs = FALSE Sys.sleep(5) } ### quick sanity check if (n > 5000L) { warningf(collapse(c("You are about to submit %i jobs.", "Consider chunking them to avoid heavy load on the scheduler.", "Sleeping 5 seconds for safety reasons."), sep = "\n"), n) Sys.sleep(5) } ### save config, start the work saveConf(reg) is.chunked = is.list(ids) info("Submitting %i chunks / %i jobs.", n, if(is.chunked) sum(viapply(ids, length)) else n) info("Cluster functions: %s.", cf$name) info("Auto-mailer settings: start=%s, done=%s, error=%s.", conf$mail.start, conf$mail.done, conf$mail.error) # use staged queries on master if fs.timeout is set # -> this way we are relatively sure that db transactions are performed in the intended order fs.timeout = conf$fs.timeout staged = conf$staged.queries || !is.na(fs.timeout) interrupted = FALSE submit.msgs = buffer(type = "list", capacity = 1000L, value = dbSendMessages, reg = reg, max.retries = 10000L, sleep = function(r) 5, staged = staged, fs.timeout = fs.timeout) logger = makeSimpleFileLogger(file.path(reg$file.dir, "submit.log"), touch = FALSE, keep = 1L) resources.timestamp = saveResources(reg, resources) ### set on exit handler to avoid inconsistencies caused by user interrupts on.exit({ # we need the second case for errors in brew (e.g. resources) if(interrupted && exists("batch.result", inherits = FALSE)) { submit.msgs$push(dbMakeMessageSubmitted(reg, id, time = submit.time, batch.job.id = batch.result$batch.job.id, first.job.in.chunk.id = if(is.chunked) id1 else NULL, resources.timestamp = resources.timestamp)) } # send remaining msgs now info("Sending %i submit messages...\nMight take some time, do not interrupt this!", submit.msgs$pos()) submit.msgs$clear() # message the existance of the log file if (logger$getSize()) messagef("%i temporary submit errors logged to file '%s'.\nFirst message: %s", logger$getSize(), logger$getLogfile(), logger$getMessages(1L)) }) ### reset status of jobs: delete errors, done, ... dbSendMessage(reg, dbMakeMessageKilled(reg, unlist(ids), type = "first"), staged = staged, fs.timeout = fs.timeout) ### initialize progress bar bar = getProgressBar(progressbar, max = n, label = "SubmitJobs") bar$set() delays = getDelays(cf, job.delay, n) tryCatch({ for (i in seq_along(ids)) { id = ids[[i]] id1 = id[1L] retries = 0L ### write R scripts files = writeFiles(reg, cf, id, chunks.as.arrayjobs, resources.timestamp, disable.mail = FALSE, staged = staged, delay = delays[i]) waitForFiles(files, timeout = fs.timeout) repeat { # max.retries may be Inf if (limit.concurrent.jobs && length(cf$listJobs(conf, reg)) >= conf$max.concurrent.jobs) { # emulate a temporary erroneous batch result batch.result = makeSubmitJobResult(status = 10L, batch.job.id = NA_character_, "Max concurrent jobs exhausted") } else { # try to submit the job interrupted = TRUE submit.time = now() batch.result = cf$submitJob( conf = conf, reg = reg, job.name = sprintf("%s-%i", reg$id, id1), rscript = files[1], log.file = getLogFilePath(reg, id1), job.dir = getJobDirs(reg, id1), resources = resources, arrayjobs = if(chunks.as.arrayjobs) length(id) else 1L ) } ### validate status returned from cluster functions if (batch.result$status == 0L) { submit.msgs$push(dbMakeMessageSubmitted(reg, id, time = submit.time, batch.job.id = batch.result$batch.job.id, first.job.in.chunk.id = if(is.chunked) id1 else NULL, resources.timestamp = resources.timestamp)) interrupted = FALSE bar$inc(1L) break } ### submitJob was not successful, handle the return status interrupted = FALSE if (batch.result$status > 0L && batch.result$status <= 100L) { if (is.finite(max.retries) && retries > max.retries) stopf("Retried already %i times to submit. Aborting.", max.retries) # temp error, wait and increase retries, then submit again in next iteration Sys.sleep(wait(retries)) # log message to file logger$log(batch.result$msg) retries = retries + 1L } else if (batch.result$status > 100L && batch.result$status <= 200L) { # fatal error, abort at once stopf("Fatal error occured: %i. %s", batch.result$status, batch.result$msg) } else { # illeagal status code stopf("Illegal status code %s returned from cluster functions!", batch.result$status) } } } }, error = bar$error) ### return ids (on.exit handler kicks now in to submit the remaining messages) return(invisible(ids)) } BatchJobs/R/batchReduceResults.R0000644000176200001440000000631312571265761016247 0ustar liggesusers#' Reduces results via a binary function and adds jobs for this to a registry. #' #' @description #' Each jobs reduces a certain number of results on one slave. #' You can then submit these jobs to the batch system. #' Later, you can do a final reduction with \code{\link{reduceResults}} on the master. #' @param reg [\code{\link{Registry}}]\cr #' Registry whose results should be reduced by \code{fun}. #' @param reg2 [\code{\link{Registry}}]\cr #' Empty registry that should store the job for the mapping. #' @param fun [\code{function(aggr, job, res, ...)}]\cr #' Function to reduce results with. #' @param ids [\code{integer}]\cr #' Ids of jobs whose results should be reduced with \code{fun}. #' Default is all jobs. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param init [any]\cr #' Initial object for reducing. #' @param block.size [\code{integer(1)}]\cr #' Number of results reduced in one job. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return Vector of type \code{integer} with job ids. #' @export #' @examples #' # generating example results: #' reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg1, f, 1:20) #' submitJobs(reg1) #' waitForJobs(reg1) #' #' # define function to reduce on slave, we want to sum the squares #' myreduce = function(aggr, job, res) aggr + res #' #' # sum 5 results on each slave process, i.e. 4 jobs #' reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) #' batchReduceResults(reg1, reg2, fun = myreduce, init = 0, block.size = 5) #' submitJobs(reg2) #' waitForJobs(reg2) #' #' # now reduce one final time on master #' reduceResults(reg2, fun = myreduce) batchReduceResults = function(reg, reg2, fun, ids, part = NA_character_, init, block.size, more.args = list()) { checkRegistry(reg, writeable = FALSE) checkRegistry(reg2, writeable = TRUE) syncRegistry(reg) syncRegistry(reg2) assertFunction(fun, c("aggr", "job", "res")) if (missing(ids)) { ids = dbGetJobIdsIfAllDone(reg) } else { ids = checkIds(reg, ids) if (length(dbFindDone(reg, ids, negate = TRUE, limit = 1L)) > 0L) stop("Not all jobs with corresponding ids finished (yet)!") } block.size = asCount(block.size) checkMoreArgs(more.args, reserved = c("..reg", "..fun", "..part")) if (dbGetJobCount(reg2) > 0L) stop("Registry 'reg2' is not empty!") if(reg$file.dir == reg2$file.dir) stop("Both registries cannot point to the same file dir. Files would get overwritten!") reg2$packages = insert(reg2$packages, reg$packages) saveRegistry(reg2) batchReduce(reg2, batchReduceResultsWrapper, ids, init = init, block.size = block.size, more.args = c(more.args, list(..reg = reg, ..fun = fun, ..part = part))) } batchReduceResultsWrapper = function(aggr, x, ..reg, ..fun, ..part) { # x is id # use lazy evaluation, if fun doesn't access job or res (unlikely) ..fun(aggr = aggr, job = getJob(..reg, x, check.id = FALSE), res = getResult(..reg, x, ..part)) } BatchJobs/R/clusterFunctionsMulticore.R0000644000176200001440000000653212623105424017700 0ustar liggesusers#' Use multiple cores on local Linux machine to spawn parallel jobs. #' #' @description #' Jobs are spawned by starting multiple R sessions on the commandline #' (similar like on true batch systems). #' Packages \code{parallel} or \code{multicore} are not used in any way. #' #' @param ncpus [\code{integer(1)}]\cr #' Number of VPUs of worker. #' Default is to use all cores but one, where total number of cores #' "available" is given by option \code{\link[base:options]{mc.cores}} #' and if that is not set it is inferred by #' \code{\link[parallel]{detectCores}}. #' @param max.jobs [\code{integer(1)}]\cr #' Maximal number of jobs that can run concurrently for the current registry. #' Default is \code{ncpus}. #' @param max.load [\code{numeric(1)}]\cr #' Load average (of the last 5 min) at which the worker is considered occupied, #' so that no job can be submitted. #' Default is inferred by \code{\link[parallel]{detectCores}}, cf. argument \code{ncpus}. #' @param nice [\code{integer(1)}]\cr #' Process priority to run R with set via nice. Integers between -20 and 19 are allowed. #' If missing, processes are not nice'd and the system default applies (usually 0). #' @param r.options [\code{character}] #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @param script [\code{character(1)}]\cr #' Path to helper bash script which interacts with the worker. #' You really should not have to touch this, as this would imply that we have screwed up and #' published an incompatible version for your system. #' This option is only provided as a last resort for very experienced hackers. #' Note that the path has to be absolute. #' This is what is done in the package: #' \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} #' Default means to take it from package directory. #' @return [\code{\link{ClusterFunctions}}]. #' @family clusterFunctions #' @export makeClusterFunctionsMulticore = function(ncpus = max(getOption("mc.cores", parallel::detectCores()) - 1L, 1L), max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) { if (isWindows()) stop("ClusterFunctionsMulticore do not work in Windows") worker = makeWorkerLocalLinux(r.options, script, ncpus, max.jobs, max.load, nice) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { updateWorker(worker, reg$file.dir, tdiff = 0L) s = worker$available if (s != "A") { makeSubmitJobResult(status = 1L, batch.job.id = NULL, msg = sprintf("Multicore busy: %s", s)) } else { pid = try(startWorkerJob(worker, rscript, log.file)) if (is.error(pid)) makeSubmitJobResult(status = 101L, batch.job.id = NULL, msg = "Submit failed.") else makeSubmitJobResult(status = 0L, batch.job.id = pid) } } killJob = function(conf, reg, batch.job.id) { killWorkerJob(worker, batch.job.id) } listJobs = function(conf, reg) { listWorkerJobs(worker, reg$file.dir) } getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "Multicore", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/Exports.R0000644000176200001440000000660313207234653014113 0ustar liggesusers#' @title Load exported R data objects. #' #' @description #' Loads exported \code{RData} object files in the \dQuote{exports} subdirectory of your \code{file.dir} #' and assigns the objects to the global environment. #' #' @template arg_reg #' @param what [\code{character}]\cr #' Names of objects to load. Defaults to all objects exported. #' @return [\code{character}]. Invisibly returns a character vector of loaded objects. #' @family exports #' @export loadExports = function(reg, what = NULL) { checkRegistry(reg, writeable = FALSE) if (!is.null(what)) assertCharacter(what, any.missing = FALSE) path = getExportDir(reg$file.dir) fns = list.files(path, pattern = "\\.RData$", all.files = TRUE) keys = gsub("\\.RData$", "", fns) if (!is.null(what)) { i = which(keys %in% what) fns = fns[i] keys = keys[i] } if (length(fns) > 0L) { messagef("Loading RData files: %s", collapse(keys)) for (i in seq_along(fns)) assign(keys[i], load2(file.path(path, fns[i]), envir = .GlobalEnv)) } invisible(keys) } #' @title Export R object to be available on the slaves. #' #' @description #' Saves objects as \code{RData} files in the \dQuote{exports} subdirectory of your \code{file.dir} #' to be later loaded on the slaves. #' #' @template arg_reg #' @param ... [any]\cr #' Objects to export. You must provide a valid name. #' @param li [\code{list}]\cr #' More objects to export provided as a named list. #' @param overwrite [\code{logical(1)}]\cr #' If set to \code{FALSE} (default), exported objects are protected from being overwritten #' by multiple calls of this function. Setting this to \code{TRUE} disables this check. #' @return [\code{character}]. Invisibly returns a character vector of exported objects. #' @family exports #' @export batchExport = function(reg, ..., li = list(), overwrite = FALSE) { checkRegistry(reg, writeable = FALSE) ddd = list(...) assertList(li, names = "strict") assertList(ddd, names = "strict") assertFlag(overwrite) keys = c(names(li), names(ddd)) dup = anyDuplicated(keys) if (dup > 0L) stopf("Object for export provided more than once: '%s'", keys[dup]) path = getExportDir(reg$file.dir) if (!overwrite) { fns = list.files(path, pattern = "\\.RData$", all.files = TRUE) old.keys = gsub("\\.RData$", "", fns) collision = which.first(keys %in% old.keys) if (length(collision) > 0L) stopf("Object named '%s' already exported and 'overwrite' is set to FALSE", keys[collision]) } objs = list2env(c(li, ddd)) for (i in seq_along(keys)) save(list = keys[i], envir = objs, file = file.path(path, sprintf("%s.RData", keys[i]))) invisible(keys) } #' @title Unload exported R objects. #' #' @description #' Removes \code{RData} files from the \dQuote{exports} subdirectory of your \code{file.dir} #' and thereby prevents loading on the slave. #' #' @template arg_reg #' @param what [\code{character}]\cr #' Names of objects to remove. #' @return [\code{character}]. Invisibly returns a character vector of unexported objects. #' @family exports #' @export batchUnexport = function(reg, what) { checkRegistry(reg, writeable = FALSE) assertCharacter(what, any.missing = FALSE) path = getExportDir(reg$file.dir) fns = list.files(path, pattern = "\\.RData$", all.files = TRUE) keys = gsub("\\.RData$", "", fns) i = which(keys %in% what) file.remove(file.path(path, fns[i])) invisible(keys[i]) } BatchJobs/R/syncRegistry.R0000644000176200001440000000320412623105424015140 0ustar liggesusers#' Syncronize staged queries into the registry. #' #' @description #' If the option \dQuote{staged.queries} is enabled, all communication from the nodes #' to the master is done via files in the subdirectory \dQuote{pending} of the \code{file.dir}. #' This function checks for such files and merges the information into the database. #' Usually you do not have to call this function yourself. #' #' @template arg_reg #' @return Invisibly returns \code{TRUE} on success. #' @export syncRegistry = function(reg) { conf = getBatchJobsConf() if (conf$staged.queries) { if (conf$debug && isOnSlave()) stop("SQL query sent from Worker") fns = lsort(list.files(getPendingDir(reg$file.dir), full.names = TRUE)) if (length(fns) == 0L) return(invisible(TRUE)) info("Syncing registry ...") queries = lapply(fns, readSQLFile) ok = !vlapply(queries, isFALSE) tryCatch(dbDoQueries(reg, unlist(queries[ok]), "rw"), error = function(e) stopf("Error syncing registry (%s)", e)) fns = fns[ok] ok = file.remove(fns) if (!all(ok)) warningf("Some pending result sets could not be removed, e.g. '%s'", head(fns[ok], 1L)) } invisible(TRUE) } readSQLFile = function(con) { x = try(readLines(con), silent = TRUE) n = length(x) if (is.error(x) || n == 0L || x[n] != "--EOF--") return(FALSE) x = x[-n] substr(x, 1L, nchar(x) - 1L) } writeSQLFile = function(x, con) { writeLines(c(paste0(x, ";"), "--EOF--"), con = con) } useStagedQueries = function() { getBatchJobsConf()$staged.queries } .OrderChars = setNames(letters[1L:6L], c("first", "submitted", "started", "done", "error", "last")) BatchJobs/R/getLogFiles.R0000644000176200001440000000076412623105424014647 0ustar liggesusers#' Get log file paths for jobs. #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @return [\code{character}]. Vector of file paths to log files. #' @family debug #' @export getLogFiles = function(reg, ids) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) ids = getJobIds(reg) else ids = checkIds(reg, ids) fids = dbGetFirstJobInChunkIds(reg, ids) getLogFilePath(reg, ifelse(is.na(fids), ids, fids)) } BatchJobs/R/sourceRegistryFiles.R0000644000176200001440000000763112623105424016457 0ustar liggesusers#' Source registry files #' #' @description #' Sources all files found in \code{src.dirs} and specified via \code{src.files}. #' #' @template arg_reg #' @param envir [\code{environment}]\cr #' Environment to source the files into. Default is the global environment. #' @return Nothing. #' @export sourceRegistryFiles = function(reg, envir = .GlobalEnv) { checkRegistry(reg, writeable = FALSE) assertEnvironment(envir) sourceRegistryFilesInternal(reg$work.dir, reg$src.dirs, reg$src.files) } sourceRegistryFilesInternal = function(work.dir, dirs, files, envir = .GlobalEnv) { # add work dir if not /foo/bar path w = !isPathFromRoot(files) files[w] = file.path(work.dir, files[w]) w = which.first(!file.exists(files)) if (length(w)) stopf("Files to source not found, e.g. %s", files[w]) w = !isPathFromRoot(dirs) dirs[w] = file.path(work.dir, dirs[w]) w = which.first(!isDirectory(dirs)) if (length(w)) stopf("Directories to source not found, e.g. %s", dirs[w]) lapply(c(getRScripts(dirs), files), sys.source, envir = envir) invisible(TRUE) } getRScripts = function(dirs) { if (length(dirs)) { ok = isDirectory(dirs) if (any(!ok)) stopf("Directories not found: %s", collapse(dirs[!ok])) unlist(lapply(dirs, list.files, pattern = "\\.[Rr]$", full.names = TRUE)) } else { character(0L) } } #' @title Add source files to registry. #' #' @description #' Mutator function for \code{src.files} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.files [\code{character}]\cr #' Paths to add to registry. #' See \code{\link{makeRegistry}}. #' @param src.now [\code{logical(1)}] #' Source files now on master? #' Default is \code{TRUE}. #' @template ret_reg_mut #' @family exports #' @export addRegistrySourceFiles = function(reg, src.files, src.now = TRUE) { checkRegistry(reg, writeable = TRUE) assertCharacter(src.files, any.missing = FALSE) assertFlag(src.now) src.files = sanitizePath(src.files, make.absolute = FALSE) if (src.now) sourceRegistryFilesInternal(reg$work.dir, character(0L), src.files) reg$src.files = union(reg$src.files, src.files) saveRegistry(reg) } #' @title Add source dirs to registry. #' #' @description #' Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.dirs [\code{character}]\cr #' Paths to add to registry. #' See \code{\link{makeRegistry}}. #' @param src.now [\code{logical(1)}] #' Source files now on master? #' Default is \code{TRUE}. #' @template ret_reg_mut #' @family exports #' @export addRegistrySourceDirs = function(reg, src.dirs, src.now = TRUE) { checkRegistry(reg, writeable = TRUE) assertCharacter(src.dirs, any.missing = FALSE) assertFlag(src.now) src.dirs = sanitizePath(src.dirs, make.absolute = FALSE) if (src.now) sourceRegistryFilesInternal(reg$work.dir, src.dirs, character(0L)) reg$src.dirs = c(reg$src.dirs, src.dirs) saveRegistry(reg) } #' @title Remove source files from registry. #' #' @description #' Mutator function for \code{src.files} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.files [\code{character}]\cr #' Paths to remove from registry. #' @template ret_reg_mut #' @family exports #' @export removeRegistrySourceFiles = function(reg, src.files) { checkRegistry(reg, writeable = TRUE) assertCharacter(src.files, any.missing = FALSE) reg$src.files = setdiff(reg$src.files, src.files) saveRegistry(reg) } #' @title Remove packages from registry. #' #' @description #' Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. #' #' @template arg_reg #' @param src.dirs [\code{character}]\cr #' Paths to remove from registry. #' @template ret_reg_mut #' @family exports #' @export removeRegistrySourceDirs = function(reg, src.dirs) { checkRegistry(reg, writeable = TRUE) assertCharacter(src.dirs, any.missing = FALSE) reg$src.dirs = setdiff(reg$src.dirs, src.dirs) saveRegistry(reg) } BatchJobs/R/showStatus.R0000644000176200001440000000562512623105424014630 0ustar liggesusers#' @title Retrieve or show status information about jobs. #' #' @description #' E.g.: How many there are, how many are done, any errors, etc. #' \code{showStatus} displays on the console, \code{getStatus} returns an informative result #' without console output. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of selected jobs. #' Default is all jobs. #' @param errors [\code{integer(1)}]\cr #' How many of the error messages should be displayed if any errors occured in the jobs? #' Default is 10. #' @param run.and.exp [\code{logical(1)}]\cr #' Show running and expired jobs? #' Requires to list the job on the batch system. If not possible, because #' that cluster function is not avilable, this option is ignored anyway. #' Default is \code{TRUE}. #' @return [\code{list}]. List of absolute job numbers. \code{showStatus} returns them #' invisibly. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' submitJobs(reg) #' waitForJobs(reg) #' #' # should show 10 submitted jobs, which are all done. #' showStatus(reg) showStatus = function(reg, ids, run.and.exp = TRUE, errors = 10L) { checkRegistry(reg, writeable = FALSE) if (!missing(ids)) ids = checkIds(reg, ids) assertFlag(run.and.exp) errors = asCount(errors) stats = getStatus(reg, ids = ids, run.and.exp = run.and.exp) procent = function(x, n) { if(is.na(x)) return("") p = ifelse(n == 0L, 0, x / n * 100) sprintf("(%6.2f%%)", p) } output = collapse(c("Status for %%i jobs at %%s", "Submitted: %%%1$ii %%s", "Started: %%%1$ii %%s", "Running: %%%1$ii %%s", "Done: %%%1$ii %%s", "Errors: %%%1$ii %%s", "Expired: %%%1$ii %%s", "Time: min=%%.2fs avg=%%.2fs max=%%.2fs"), "\n") output = sprintf(output, min(4L, nchar(sprintf("%i", stats$n + 1L)))) with(stats, catf(output, n, Sys.time(), submitted, procent(submitted, n), started, procent(started, n), running, procent(running, n), done, procent(done, n), error, procent(error, n), expired, procent(expired, n), t_min, t_avg, t_max)) m = min(errors, stats$error) if(m > 0L) { msgs = dbGetErrorMsgs(reg, ids, filter = TRUE, limit = m) catf("\nShowing first %i errors:", m) cat(sprintf("Error in %i: %s", msgs$job_id, msgs$error), sep = "\n") } return(invisible(stats)) } #' @rdname showStatus #' @export getStatus getStatus = function(reg, ids, run.and.exp = TRUE) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) assertFlag(run.and.exp) if (!missing(ids)) ids = checkIds(reg, ids) run.and.exp = run.and.exp && !is.null(getListJobs()) dbGetStats(reg, ids, running = run.and.exp, expired = run.and.exp, times = TRUE) } BatchJobs/R/sendMail.R0000644000176200001440000000346612357424573014216 0ustar liggesuserssendMail = function(reg, ids, result.str, extra.msg = "", disable.mail, condition, first, last, conf) { if (disable.mail) return(invisible(NULL)) conf = getBatchJobsConf() ischunk = (length(ids) > 1L) first.id = if(ischunk) ids[[1L]] else ids # should we mail mail.conds = list(start = conf$mail.start, done = conf$mail.done, error = conf$mail.error) mail.cond = mail.conds[[condition]] if (mail.cond == "all" || (mail.cond %in% c("first", "first+last") && first.id == first) || (mail.cond %in% c("last", "first+last") && first.id == last)) { myformat = function(title, lines) { collapse(c(sprintf("### %s", title), lines, "", ""), "\n") } cstr = switch(condition, "start" = "started", "done" = "finished", "error" = "terminated with exception") subj = sprintf("[%s]: %s %s has %s", reg$id, ifelse(ischunk, "Chunk with first job ", "Job"), first.id, cstr) msg = myformat("Ids", ids) # append result and status information if (condition != "start") { if (extra.msg != "") msg = paste0(msg, myformat("Message", extra.msg)) msg = paste0(msg, myformat("Results", result.str)) if(first.id == last) msg = paste0(msg, myformat("Status", capture.output(showStatus(reg, run.and.exp = FALSE)))) } # if a mail problem occurs, we only warn but do not terminate ok = try (sendmail(conf$mail.from, conf$mail.to, subj, msg, control = conf$mail.control)) if (is.error(ok)) { warningf("Could not send mail to signal condition '%s'!\nFrom: %s\nTo: %s\nControl: %s\nError message: %s", condition, conf$mail.from, conf$mail.to, convertToShortString(conf$mail.control), as.character(ok)) } else { messagef("Mail signaling condition '%s' send to %s", condition, conf$mail.to) } } invisible(NULL) } BatchJobs/R/clusterFunctionsSLURM.R0000644000176200001440000000534212623105424016635 0ustar liggesusers#' @title Create cluster functions for SLURM-based systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{sbatch} command. Jobs are #' killed using the \code{scancel} command and the list of running jobs #' is retrieved using \code{squeue}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSLURM}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsSLURM = function(template.file, list.jobs.cmd = c("squeue", "-h", "-o %i", "-u $USER")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "sb") res = runOSCommandLinux("sbatch", outfile, stop.on.exit.code = FALSE) max.jobs.msg = "sbatch: error: Batch job submission failed: Job violates accounting policy (job submit limit, user's size and/or time limits)" temp.error = "Socket timed out on send/recv operation" output = collapse(res$output, sep = "\n") if (grepl(max.jobs.msg, output, fixed = TRUE)) { makeSubmitJobResult(status = 1L, batch.job.id = NA_character_, msg = max.jobs.msg) } else if (grepl(temp.error, output, fixed = TRUE)) { # another temp error we want to catch makeSubmitJobResult(status = 2L, batch.job.id = NA_character_, msg = temp.error) } else if (res$exit.code > 0L) { cfHandleUnknownSubmitError("sbatch", res$exit.code, res$output) } else { makeSubmitJobResult(status = 0L, batch.job.id = stri_trim_both(stri_split_fixed(output, " ")[[1L]][4L])) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("scancel", batch.job.id) } listJobs = function(conf, reg) { # Result is lines of fully quantified batch.job.ids jids = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L])$output stri_extract_first_regex(jids, "[0-9]+") } getArrayEnvirName = function() "SLURM_ARRAY_TASK_ID" makeClusterFunctions(name = "SLURM", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/installPackagesOnSSHWorkers.R0000644000176200001440000000263012430505510017766 0ustar liggesusers#' Install packages on SSH workers. #' #' @description #' Installation is done via \code{\link{callFunctionOnSSHWorkers}} #' and \code{\link{install.packages}}. #' #' Note that as usual the function tries to install #' the packages into the first path of \code{.libPaths()} #' of each each worker. #' #' @param nodenames [\code{character}]\cr #' Nodenames of workers. #' @param pkgs [\code{character}]\cr #' See \code{\link{install.packages}}. #' @param repos [\code{character}]\cr #' See \code{\link{install.packages}}. #' If the user must be queried this is of course done on the master. #' @param consecutive [\code{logical(1)}]\cr #' See \code{\link{callFunctionOnSSHWorkers}}. #' Default is \code{TRUE}. #' @param show.output [\code{logical(1)}]\cr #' See \code{\link{callFunctionOnSSHWorkers}}. #' Default is \code{consecutive}. #' @param ... [any]\cr #' Passed to \code{\link{install.packages}}. #' @return Nothing. #' @export #' @seealso \code{\link{callFunctionOnSSHWorkers}} installPackagesOnSSHWorkers = function(nodenames, pkgs, repos = getOption("repos"), consecutive = TRUE, show.output = consecutive, ...) { assertCharacter(pkgs, min.len = 1L, any.missing = FALSE) if (repos == "@CRAN@") repos = chooseCRANmirror() callFunctionOnSSHWorkers(nodenames, fun = install.packages, pkgs = pkgs, repos = repos, consecutive = consecutive, show.output = show.output, ...) invisible(NULL) } BatchJobs/R/getJob.R0000644000176200001440000000177112623105424013654 0ustar liggesusers#' Get job from registry by id. #' @template arg_reg #' @param id [\code{integer(1)}]\cr #' Id of job. #' @param check.id [\code{logical(1)}]\cr #' Check the job id? #' Default is \code{TRUE}. #' @return [\code{\link{Job}}]. #' @export getJob = function(reg, id, check.id = TRUE) { if (check.id) id = checkIds(reg, id, len = 1L) getJobs(reg, id, check.ids = FALSE)[[1L]] } #' Get jobs from registry by id. #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @param check.ids [\code{logical(1)}]\cr #' Check the job ids? #' Default is \code{TRUE}. #' @return [list of \code{\link{Job}}]. #' @export getJobs = function(reg, ids, check.ids = TRUE) { checkRegistry(reg, writeable = FALSE) # syncRegistry(reg) NOT! assertFlag(check.ids) UseMethod("getJobs") } #' @method getJobs Registry #' @export getJobs.Registry = function(reg, ids, check.ids = TRUE) { if (!missing(ids) && check.ids) ids = checkIds(reg, ids) dbGetJobs(reg, ids) } BatchJobs/R/batchQuery.R0000644000176200001440000000142612623105424014546 0ustar liggesusers#' Manually query the BatchJobs database #' #' @template arg_reg #' @param query [\code{character(1)}]\cr #' SQL query to send to the database. #' @param flags [\code{character(1)}]\cr #' One of \dQuote{ro}, \dQuote{rw} or \dQuote{rwc} which is translated #' to \code{SQLITE_RO}, \code{SQLITE_RW} or \code{SQLITE_RWC}, respectively. #' See \link[RSQLite]{SQLITE_RO} for more info. #' @return [\code{data.frame}] Result of the query. #' @export #' @examples #' reg = makeRegistry("test", file.dir = tempfile()) #' batchMap(reg, identity, i = 1:10) #' batchQuery(reg, "SELECT * FROM test_job_status") batchQuery = function(reg, query, flags = "ro") { assertString(query) assertChoice(flags, c("ro", "rw", "rwc")) dbDoQuery(reg, query = query, flags = flags, max.retries = 3L) } BatchJobs/R/clusterFunctionsSGE.R0000644000176200001440000000542712527173323016363 0ustar liggesusers#' @title Create cluster functions for Sun Grid Engine systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{qsub} command. Jobs are #' killed using the \code{qdel} command and the list of running jobs #' is retrieved using \code{qselect}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSGE}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsSGE = function(template.file, list.jobs.cmd = c("qstat", "-u $USER")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "job") # returns: "Your job 240933 (\"sleep 60\") has been submitted" res = runOSCommandLinux("qsub", outfile, stop.on.exit.code = FALSE) # FIXME filled queues if (res$exit.code > 0L) { cfHandleUnknownSubmitError("qsub", res$exit.code, res$output) } else { # collapse output strings and first number in string is batch.job.id batch.job.id = stri_extract_first_regex(collapse(res$output, sep = " "), "\\d+") makeSubmitJobResult(status = 0L, batch.job.id = batch.job.id) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("qdel", batch.job.id) } listJobs = function(conf, reg) { # looks like this # job-ID prior name user state submit/start at queue slots ja-task-ID #----------------------------------------------------------------------------------------------------------------- # 240935 0.00000 sleep 60 matthias qw 04/03/2012 15:45:54 1 # res = runOSCommandLinux("qstat", "-u $USER") res = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L]) # drop first 2 header lines out = tail(res$output, -2L) # first number in strings are batch.job.ids stri_extract_first_regex(out, "\\d+") } getArrayEnvirName = function() "SGE_TASK_ID" makeClusterFunctions(name = "SGE", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/loadResult.R0000644000176200001440000000220212623105424014546 0ustar liggesusers#' Loads a specific result file. #' @template arg_reg #' @param id [\code{integer(1)}]\cr #' Id of job. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param missing.ok [\code{logical(1)}]\cr #' If \code{FALSE} an error is thrown if no result file is found. #' Otherwise \code{NULL} is returned. #' Default is \code{FALSE}. #' @param impute.val [any]\cr #' The value to return when no result is available. #' Defaults to \code{NULL} (the previous behavior) #' @return [any]. Result of job. #' @seealso \code{\link{reduceResults}} #' @export loadResult = function(reg, id, part = NA_character_, missing.ok = FALSE, impute.val = NULL) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) id = checkIds(reg, id, len = 1L) checkPart(reg, part) assertFlag(missing.ok) getResult(reg, id, part, missing.ok, impute.val) } getResult = function(reg, id, part = NA_character_, missing.ok = FALSE, impute.val = NULL) { getResults(reg, id, part, missing.ok, impute.val)[[1L]] } BatchJobs/R/doJob.R0000644000176200001440000001010012623105424013461 0ustar liggesusersdoJob = function(reg, ids, multiple.result.files, staged, disable.mail, first, last, array.id) { saveOne = function(result, name) { fn = getResultFilePath(reg, job$id, name) message("Writing result file: ", fn) save2(file = fn, result = result) } # Get the conf conf = loadConf(reg) # Say hi. messagef("%s: Starting job on node %s.", Sys.time(), Sys.info()["nodename"]) messagef("Auto-mailer settings: start=%s, done=%s, error=%s.", conf$mail.start, conf$mail.done, conf$mail.error) # We need to see all warnings immediatly if (conf$cluster.functions$name != "Testing") options(warning.length = 8170L, warn = 1L + conf$raise.warnings) # Go to the work.dir wd = switchWd(reg) on.exit({ wd$reset() message("Memory usage according to gc:") print(gc()) }) if (!is.na(array.id)) { # FIXME better send error to database here, we don't see those errors on the master :( array.id = asInt(as.integer(array.id), lower = 1L, upper = length(ids)) messagef("Processing array id %s", array.id) ids = ids[array.id] } n = length(ids) results = character(n) error = logical(n) mail.extra.msg = "" cache = makeFileCache(use.cache = n > 1L) # notify status sendMail(reg, ids, results, "", disable.mail, condition = "start", first, last) # create buffer of started messages msg.buf = buffer(capacity = 2L * n) next.flush = 0L if (staged) { fn = getJobFile(reg, first) messagef("Loading jobs from file '%s'", fn) jobs = readRDS(fn) } else { jobs = getJobs(reg, ids, check.ids = FALSE) } for (i in seq_len(n)) { job = jobs[[i]] messagef("########## Executing jid=%s ##########", job$id) started = Sys.time() msg.buf$push(dbMakeMessageStarted(reg, ids[i], time = as.integer(started))) messagef("Timestamp: %s" , started) print(job) if (now() > next.flush) { if (dbSendMessages(reg, msg.buf$get(), staged = staged)) msg.buf$clear() next.flush = now() + as.integer(runif(1L, 1200L, 24001L)) } message("Setting seed: ", job$seed) seed = seeder(reg, job$seed) if (conf$measure.mem) gc(reset = TRUE) result = try(applyJobFunction(reg, job, cache), silent = TRUE) mem.used = if (conf$measure.mem) sum(gc()[, 6L]) else -1 seed$reset() catf("Result:") print(str(result, max.level = 1L, list.len = 5L)) error[i] = is.error(result) if (error[i]) { results[i] = as.character(result) } else if (multiple.result.files) { if (!is.list(result)) { results[i] = "multiple.result.files is TRUE, but your algorithm did not return a list!" error[i] = TRUE } else if (!isProperlyNamed(result)) { results[i] = "multiple.result.files is TRUE, but some the returned lists is not fully, distinctly named!" error[i] = TRUE } } if (error[i]) { msg.buf$push(dbMakeMessageError(reg, job$id, err.msg = results[i], memory = mem.used)) message("Error occurred: ", results[i]) } else { results[i] = paste0(capture.output(str(result)), collapse = "\n") msg.buf$push(dbMakeMessageDone(reg, job$id, memory = mem.used)) if (multiple.result.files) { Map(saveOne, result = result, name = names(result)) } else { saveOne(result, NA_character_) } } } # try to flush the remaining msgs at the end for (i in seq_len(10L)) { if (dbSendMessages(reg, msg.buf$get(), staged = staged)) { msg.buf$clear() break } Sys.sleep(round(runif(1L, 30, 120))) } # check if there are still remaining messages if (!msg.buf$empty()) { mail.extra.msg = paste("Some DB messages could not be flushed.", "This indicates some DB problem or too much communication with the DB.", "Everything should still be ok, you only might have to resubmit some jobs as they are not recorded as 'done'.", sep = "\n") warningf(mail.extra.msg) } sendMail(reg, ids, results, mail.extra.msg, disable.mail, condition = ifelse(any(error), "error", "done"), first, last) messagef("%s: All done.", Sys.time()) return(!any(error)) } BatchJobs/R/testJob.R0000644000176200001440000001002412623105424014043 0ustar liggesusers#' Tests a job by running it with Rscript in a new process. #' #' @description #' Useful for debugging. #' Note that neither the registry, database or file directory are changed. #' @template arg_reg #' @param id [\code{integer(1)}]\cr #' Id of job to test. #' Default is first job id of registry. #' @param external [\code{logical(1)}]\cr #' Run test in an independent external R session instead of current. #' The former allows to uncover missing variable definitions (which may #' accidentially be defined in the current global environment) and the latter #' is useful to get traceable execeptions. #' Default is \code{TRUE}. #' @param resources [\code{list}]\cr #' Usually not needed, unless you call the function \code{\link{getResources}} in your job. #' See \code{\link{submitJobs}}. #' Default is empty list. #' @return [any]. Result of job. If the job did not complete because of an error, NULL is returned. #' @family debug #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) if (x==1) stop("oops") else x #' batchMap(reg, f, 1:2) #' testJob(reg, 2) testJob = function(reg, id, resources = list(), external = TRUE) { checkRegistry(reg, writeable = FALSE) #syncRegistry(reg) if (missing(id)) { id = dbGetJobId(reg) if (length(id) == 0L) stop("Registry is empty!") messagef("Testing job with id=%i ...", id) } else { id = checkIds(reg, id, len = 1L) } assertList(resources) resources = resrc(resources) assertFlag(external) if (external) { # we dont want to change anything in the true registry / file dir / DB # so we have to copy stuff a little bit r = reg # get a unique, unused tempdir. tmpdir() always stays the same per session td = tempfile(pattern = "") construct = sprintf("make%s", class(r)[1L]) # copy reg reg = do.call(construct, list(id = reg$id, seed = r$seed, file.dir = td, work.dir = r$work.dir, sharding = FALSE, multiple.result.files = r$multiple.result.files, packages = names(reg$packages), src.dirs = reg$src.dirs, src.files = reg$src.files)) # copy DB file.copy(from = file.path(r$file.dir, "BatchJobs.db"), to = file.path(td, "BatchJobs.db"), overwrite = TRUE) # copy conf saveConf(reg) # copy job stuff copyRequiredJobFiles(r, reg, id) # copy exports file.copy(from = getExportDir(r$file.dir), to = td, recursive = TRUE) # write r script resources.timestamp = saveResources(reg, resources) writeFiles(reg, getClusterFunctions(getBatchJobsConf()), id, chunks.as.arrayjobs = FALSE, resources.timestamp = resources.timestamp, disable.mail = TRUE, staged = FALSE, delay = 0) # execute now = Sys.time() message("### Output of new R process starts here ###") r.interp = file.path(R.home("bin"), sprintf("%s%s", "Rscript", ifelse(isWindows(), ".exe", ""))) system3(r.interp, getRScriptFilePath(reg, id), wait = TRUE) message("### Output of new R process ends here ###") dt = difftime(Sys.time(), now) messagef("### Approximate running time: %.2f %s", as.double(dt), units(dt)) res = try(getResult(reg, id)) if (is.error(res)) return(NULL) } else { setOnSlave(TRUE) on.exit(setOnSlave(FALSE)) # FIXME: stuff we might need to store before: resources saveConf(reg) # trigger loadExports, sourceRegistryFiles, ... loadRegistry(reg$file.dir) res = applyJobFunction(reg, getJob(reg, id), makeFileCache(FALSE)) } return(res) } #' ONLY FOR INTERNAL USAGE. #' @param reg1 [\code{\link{Registry}}]\cr #' Source registry. #' @param reg2 [\code{\link{Registry}}]\cr #' Detination registry. #' @param id [\code{character(1)}]\cr #' Job id. #' @return Nothing. #' @keywords internal #' @export copyRequiredJobFiles = function(reg1, reg2, id) { UseMethod("copyRequiredJobFiles") } #' @export copyRequiredJobFiles.Registry = function(reg1, reg2, id) { job = getJob(reg1, id, check.id = FALSE) file.copy(getFunFilePath(reg1, job$fun.id), getFunFilePath(reg2, job$fun.id)) } BatchJobs/R/waitForJobs.R0000644000176200001440000000732112623105424014670 0ustar liggesusers#' Wait for termination of jobs on the batch system. #' #' @description #' Waits for termination of jobs while displaying a progress bar #' containing summarizing informations of the jobs. #' The following abbreviations are used in the progress bar: #' \dQuote{S} for number of jobs on system, \dQuote{D} for number of #' jobs successfully terminated, \dQuote{E} for number ofjobs terminated #' with an R exception and \dQuote{R} for number of jobs currently running #' on the system. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Vector of job ids. #' Default is all submitted jobs not yet terminated. #' @param sleep [\code{numeric(1)}]\cr #' Seconds to sleep between status updates. Default is \code{10}. #' @param timeout [\code{numeric(1)}]\cr #' After waiting \code{timeout} seconds, show a message and return \code{FALSE}. #' This argument may be required on some systems where, e.g., expired jobs or jobs on hold #' are problematic to detect. If you don't want a timeout, set this to \code{Inf}. #' Default is \code{604800} (one week). #' @param stop.on.error [\code{logical(1)}]\cr #' Immediately return if a job terminates with an error? Default is \code{FALSE}. #' @template arg_progressbar #' @return [\code{logical(1)}]. Returns \code{TRUE} if all jobs terminated successfully #' and \code{FALSE} if either an error occurred or the timeout is reached. #' @export waitForJobs = function(reg, ids, sleep = 10, timeout = 604800, stop.on.error = FALSE, progressbar = TRUE) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) { ids = dbFindSubmittedNotTerminated(reg) } else { ids = checkIds(reg, ids) not.submitted = dbFindSubmitted(reg, ids, negate = TRUE, limit = 1L) if (length(not.submitted) > 0L) stopf("Not all jobs have been submitted, e.g. job with id %i", not.submitted) } assertNumber(sleep, lower = 1) if (is.infinite(sleep)) stop("Argument 'sleep' must be finite") assertNumber(timeout, lower = sleep) assertFlag(stop.on.error) assertFlag(progressbar) n = length(ids) if (n == 0L) return(TRUE) timeout = now() + timeout batch.ids = getBatchIds(reg, "Cannot find jobs on system") i = 1L bar = getProgressBar(progressbar, min = 0L, max = n, label = "Waiting ") on.exit(bar$kill()) repeat { stats = dbGetStats(reg, ids, running = TRUE, expired = FALSE, times = FALSE, batch.ids = batch.ids) n.sys = n - stats$done - stats$error bar$set(n - n.sys, msg = sprintf("Waiting [S:%i D:%i E:%i R:%i]", n.sys, stats$done, stats$error, stats$running)) if (stop.on.error && stats$error) { err = dbGetErrorMsgs(reg, ids, filter = TRUE, limit = 1L) warningf("Job %i terminated with an error: %s", err$job_id, err$error) return(FALSE) } if (n.sys == 0L) return(stats$error == 0L) if (i %% 5L == 0L) { # update batch ids batch.ids = getBatchIds(reg, "Cannot find jobs on system") # check if there are still jobs on the system and none has mystically disappeared # NOTE it seems like some schedulers are "laggy", we should not do this operation # in the first loop w/o a sleep if (length(dbFindOnSystem(reg, ids, limit = 1L, batch.ids = batch.ids)) == 0L) { if (length(dbFindDisappeared(reg, ids, limit = 1L, batch.ids = batch.ids)) > 0L) bar$error(stop("Some jobs disappeared, i.e. were submitted but are now gone. Check your configuration and template file.")) return(stats$error == 0L) } } if (is.finite(timeout) && now() > timeout) { warningf("Timeout reached. %i jobs still on system.", n.sys) return(FALSE) } Sys.sleep(sleep) i = i + 1L suppressMessages(syncRegistry(reg)) } } BatchJobs/R/loadRegistry.R0000644000176200001440000000460512623105424015111 0ustar liggesusers#' @title Load a previously saved registry. #' #' @description #' Loads a previously created registry from the file system. #' The \code{file.dir} is automatically updated upon load if \code{adjust.paths} is set to #' \code{TRUE}, so be careful if you use the registry on multiple machines simultaneously, #' e.g. via sshfs or a samba share. #' #' There is a heuristic included which tries to detect if the location of the registry #' has changed and returns a read-only registry if necessary. #' #' @param file.dir [\code{character(1)}]\cr #' Location of the file.dir to load the registry from. #' @param work.dir [\code{character(1)}]\cr #' Location of the work. Unchanged if missing. #' @param adjust.paths [\code{logical(1)}]\cr #' If set to \code{FALSE} (default), the paths for the \code{file.dir} and \code{work.dir} #' will not be updated. Set to \code{TRUE} if you moved the directoy to another system #' \emph{after} all computations finished. #' @return [\code{\link{Registry}}]. #' @export loadRegistry = function(file.dir, work.dir, adjust.paths = FALSE) { assertString(file.dir) assertFlag(adjust.paths) fn = getRegistryFilePath(file.dir) if (!file.exists(fn)) stopf("No registry found in '%s'", file.dir) info("Loading registry: %s", fn) reg = load2(fn, "reg") requirePackages(names(reg$packages), why = sprintf("registry %s", reg$id), default.method = "attach") if (!isOnSlave()) { save.reg = FALSE read.only = FALSE adjusted = adjustRegistryPaths(reg, file.dir, work.dir) if (!isFALSE(adjusted)) { if (!adjust.paths) { warning("It seems like you've moved the registry to a new location or system. ", "To update the paths, call 'loadRegistry' with option 'adjust.paths' set to TRUE. ", "But make sure that there are no jobs running on the system. ", "Returning a read-only registry, and not updating the database to the latest layout, ", "i.e. your registry may be defunct.") read.only = TRUE } reg = adjusted save.reg = TRUE } if (!read.only) { updated = updateRegistry(reg) if (!isFALSE(updated)) { reg = updated save.reg = TRUE } } if (save.reg) { if (read.only) reg$read.only = TRUE else saveRegistry(reg) } } loadExports(reg) sourceRegistryFiles(reg) return(reg) } BatchJobs/R/setJobFunction.R0000644000176200001440000000316412623105424015374 0ustar liggesusers#' Sets the job function for already existing jobs. #' #' @description #' Use this function only as last measure when there is a bug #' in a part of your job function and you have already computed a large number #' of (unaffected) results. This function allows you to fix the error and to #' associate the jobs with the corrected function. #' #' Note that by default the computational state of the affected jobs is also reset. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all jobs. #' @param fun [\code{function}]\cr #' Replacement function. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @param reset [\code{logical(1)}]\cr #' Reset job status via \code{\link{resetJobs}}. #' Default is \code{TRUE}. #' @param force [\code{logical(1)}]\cr #' See \code{\link{resetJobs}}. #' Default is \code{FALSE}. #' @return Nothing. #' @family debug #' @export setJobFunction = function(reg, ids, fun, more.args = list(), reset = TRUE, force = FALSE) { checkRegistry(reg, strict = FALSE, writeable = TRUE) syncRegistry(reg) assertFunction(fun) checkMoreArgs(more.args) assertFlag(reset) assertFlag(force) UseMethod("setJobFunction") } #' @export setJobFunction.Registry = function(reg, ids, fun, more.args = list(), reset = TRUE, force = FALSE) { if (missing(ids)) { # nothing to do ... return(invisible(NULL)) } else { ids = checkIds(reg, ids) } fun.id = saveFunction(reg, fun, more.args) dbSetJobFunction(reg, ids, fun.id) if (reset) resetJobs(reg, ids, force = force) invisible(NULL) } BatchJobs/R/checkIds.R0000644000176200001440000000135012623105424014150 0ustar liggesusers#' Check job ids. #' #' @description #' Simply checks if probided vector of job ids is valid and #' throws an error if something is odd. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Vector of job ids. #' @param check.present [\code{logical(1)}]\cr #' Check if the ids are present in the database? Default is \code{TRUE}. #' @param len [\code{integer(1)}]\cr #' Expected length of id vector. Passed to \code{\link[checkmate]{asInteger}}. #' @return Invisibly returns the vector of ids, converted to integer. #' @export checkIds = function(reg, ids, check.present = TRUE, len = NULL) { ids = asInteger(ids, any.missing = FALSE, unique = TRUE, len = len) if (check.present) dbCheckJobIds(reg, ids) invisible(ids) } BatchJobs/R/clusterFunctionsSSH.R0000644000176200001440000001357712623105424016401 0ustar liggesusers#' Create SSH worker for SSH cluster functions. #' #' @param nodename [\code{character(1)}]\cr #' Host name of node. #' @param ssh.cmd [\code{character(1)})]\cr #' CLI command to ssh into remote node. #' Default is \dQuote{ssh}. #' @param ssh.args [\code{character}]\cr #' CLI args for \code{ssh.cmd}. #' Default is none. #' @param rhome [\code{character(1)}]\cr #' Path to R installation on worker. #' \dQuote{} means R installation on the PATH is used, #' of course this implies that it must be on the PATH #' (also for non-interactive shells)! #' Default is \dQuote{}. #' @param ncpus [\code{integers(1)}]\cr #' Number of VPUs of worker. #' Default means to query the worker via \dQuote{/proc/cpuinfo}. #' @param max.jobs [\code{integer(1)}]\cr #' Maximal number of jobs that can run concurrently for the current registry. #' Default is \code{ncpus}. #' @param max.load [\code{numeric(1)}]\cr #' Load average (of the last 5 min) at which the worker is considered occupied, #' so that no job can be submitted. #' Default is \code{ncpus-1}. #' @param nice [\code{integer(1)}]\cr #' Process priority to run R with set via nice. Integers between -20 and 19 are allowed. #' If missing, processes are not nice'd and the system default applies (usually 0). #' Default is no niceing. #' @param r.options [\code{character}]\cr #' Options for R and Rscript, one option per element of the vector, #' a la \dQuote{--vanilla}. #' Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}. #' @param script [\code{character(1)}]\cr #' Path to helper bash script which interacts with the worker. #' You really should not have to touch this, as this would imply that we have screwed up and #' published an incompatible version for your system. #' This option is only provided as a last resort for very experienced hackers. #' Note that the path has to be absolute. #' This is what is done in the package: #' \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} #' Default means to take it from package directory. #' @return [\code{\link{SSHWorker}}]. #' @export #' @aliases SSHWorker makeSSHWorker = function(nodename, ssh.cmd = "ssh", ssh.args = character(0L), rhome = "", ncpus, max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) { worker = makeWorkerRemoteLinux(nodename, ssh.cmd, ssh.args, rhome, r.options, script, ncpus, max.jobs, max.load, nice) class(worker) = c("SSHWorker", class(worker)) return(worker) } #' Create an SSH cluster to execute jobs. #' #' Worker nodes must share the same file system and be accessible by ssh #' without manually entering passwords (e.g. by ssh-agent or passwordless pubkey). #' Note that you can also use this function to parallelize on multiple cores on your local machine. #' But you still have to run an ssh server and provide passwordless access to #' localhost. #' #' @param ... [\code{\link{SSHWorker}}]\cr #' Worker objects, all created with \code{\link{makeSSHWorker}}. #' @param workers [list of \code{\link{SSHWorker}}]\cr #' Alternative way to pass workers. #' @return [\code{ClusterFunctions}]. #' #' @examples \dontrun{ #' #' # Assume you have three nodes larry, curley and moe. All have 6 #' # cpu cores. On curley and moe R is installed under #' # "/opt/R/R-current" and on larry R is installed under #' # "/usr/local/R/". larry should not be used extensively because #' # somebody else wants to compute there as well. #' # Then a call to 'makeClusterFunctionsSSH' #' # might look like this: #' #' cluster.functions = makeClusterFunctionsSSH( #' makeSSHWorker(nodename = "larry", rhome = "/usr/local/R", max.jobs = 2), #' makeSSHWorker(nodename = "curley", rhome = "/opt/R/R-current"), #' makeSSHWorker(nodename = "moe", rhome = "/opt/R/R-current")) #' } #' @export #' @family clusterFunctions #' @seealso \code{\link{makeSSHWorker}} makeClusterFunctionsSSH = function(..., workers) { args = list(...) if (!xor(length(args) > 0L, !missing(workers))) stop("You must use exactly only 1 of: '...', 'workers'!") if (missing(workers)) workers = args checkListElementClass(workers, "SSHWorker") if (length(workers) == 0L) stop("You must pass at least 1 SSH worker!") nodenames = extractSubList(workers, "nodename") dup = duplicated(nodenames) if (any(dup)) stopf("Multiple definitions for worker nodenames: %s!", collapse(nodenames[dup])) names(workers) = nodenames rm(nodenames, dup) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { worker = findWorker(workers, reg$file.dir, tdiff = 5L) if (is.null(worker)) { states = collapse(extractSubList(workers, "available", simplify = TRUE), sep = "") makeSubmitJobResult(status = 1L, batch.job.id = NULL, msg = sprintf("Workers busy: %s", states)) } else { pid = try(startWorkerJob(worker, rscript, log.file)) if (is.error(pid)) makeSubmitJobResult(status = 101L, batch.job.id = NULL, msg = "Submit failed.") else makeSubmitJobResult(status = 0L,batch.job.id = paste0(worker$nodename, "#", pid)) } } killJob = function(conf, reg, batch.job.id) { parts = stri_split_fixed(batch.job.id, "#")[[1L]] nodename = parts[1L] pid = parts[2L] worker = workers[[nodename]] if (is.null(worker)) stopf("Unknown worker node '%s'.", nodename) killWorkerJob(worker, pid) } listJobs = function(conf, reg) { res = NULL for (worker in workers) { nodename = worker[["nodename"]] pids = listWorkerJobs(worker, reg$file.dir) if (length(pids) > 0L) { res = c(res, paste0(nodename, "#", pids)) } } res } getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "SSH", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/checkIdValid.R0000644000176200001440000000101412413244720014742 0ustar liggesusers# check for a valid registry id: # must start with an alphabetic # then alpha-numerics and underscores are allowed # pattern = "^[0-9a-zA-Z]+[0-9a-zA-Z_-]*$" # we must be quite restrictive here because we prefix # the table name with the registry name checkIdValid = function(id, allow.minus = TRUE) { assertString(id) if (allow.minus) pattern = "^[a-zA-Z]+[0-9a-zA-Z_-]*$" else pattern = "^[a-zA-Z]+[0-9a-zA-Z_]*$" if (!grepl(pattern, id)) stopf("Id does not comply with pattern %s: %s", pattern, id) } BatchJobs/R/showClusterStatus.R0000644000176200001440000000216312623105424016164 0ustar liggesusers#' Show information about available computational resources on cluster. #' #' @description #' Currently only supported for multicore and SSH mode. #' Displays: Name of node, current load, number of running R processes, number of R processes #' with more than 50% load, number of BatchJobs jobs running. #' The latter counts either jobs belonging to \code{reg} or all BatchJobs jobs if reg was not passed. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry. #' Must not be passed and this is the default. #' @return [\code{data.frame}]. #' @export showClusterStatus = function(reg) { if (missing(reg)) { file.dir = "" } else { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) file.dir = reg$file.dir } conf = getBatchJobsConf() cf = conf$cluster.functions if (cf$name %nin% c("Multicore", "SSH")) stop("showWorkerStatus can only be used in multicore or SSH mode!") workers = environment(cf$submitJob)$workers data = rbindlist(lapply(workers, getWorkerStatus, file.dir = file.dir)) data$ncpus = extractSubList(workers, "ncpus") setDF(data, rownames = names(workers)) return(data) } BatchJobs/R/batchReduce.R0000644000176200001440000000371212571265761014665 0ustar liggesusers#' Reduces via a binary function over a list adding jobs to a registry. #' #' @description #' Each jobs reduces a certain number of elements on one slave. #' You can then submit these jobs to the batch system. #' @param reg [\code{\link{Registry}}]\cr #' Empty Registry. #' @param fun [\code{function(aggr, x, ...)}]\cr #' Function to reduce \code{xs} with. #' @param xs [\code{vector}]\cr #' Vector to reduce. #' @param init [any]\cr #' Initial object for reducing. #' @param block.size [\code{integer(1)}]\cr #' Number of elements of \code{xs} reduced in one job. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return Vector of type \code{integer} with job ids. #' @export #' @examples #' # define function to reduce on slave, we want to sum a vector #' f = function(aggr, x) aggr + x #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' #' # sum 20 numbers on each slave process, i.e. 5 jobs #' batchReduce(reg, fun = f, 1:100, init = 0, block.size = 5) #' submitJobs(reg) #' waitForJobs(reg) #' #' # now reduce one final time on master #' reduceResults(reg, fun = function(aggr,job,res) f(aggr, res)) batchReduce = function(reg, fun, xs, init, block.size, more.args = list()) { checkRegistry(reg, strict = TRUE, writeable = FALSE) syncRegistry(reg) assertFunction(fun, c("aggr", "x")) if (!is.vector(xs)) stop("Argument xs must be a vector") block.size = asCount(block.size, positive = TRUE) if (dbGetJobCount(reg) > 0L) stop("Registry is not empty!") checkMoreArgs(more.args, reserved = c(".fun", ".init")) xs.blocks = chunk(xs, chunk.size = block.size, shuffle = FALSE) more.args = c(more.args, list(.fun = fun, .init = init)) batchMap(reg, batchReduceWrapper, xs.blocks, more.args = more.args) } batchReduceWrapper = function(xs.block, .fun, .init, ...) { fun = function(aggr, x) .fun(aggr, x, ...) Reduce(fun, xs.block, init = .init) } BatchJobs/R/batchMapQuick.R0000644000176200001440000000552512413244720015157 0ustar liggesusers#' Combination of makeRegistry, batchMap and submitJobs. #' #' @description #' Combination of \code{\link{makeRegistry}}, \code{\link{batchMap}} #' and \code{\link{submitJobs}} #' for quick computations on the cluster. #' Should only be used by skilled users who know what they are doing. #' Creates the file.dir, maps function, potentially chunks jobs and submits them. #' #' @param fun [\code{function}]\cr #' Function to map over \code{...}. #' @param ... [any]\cr #' Arguments to vectorize over (list or vector). #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @param file.dir [\code{character}]\cr #' See \code{\link{makeRegistry}}. #' Default is \code{NULL}, which means that it is created in the current directory under the name #' \dQuote{bmq_[random alphanumerics]}. #' @param packages [\code{character}]\cr #' See \code{\link{makeRegistry}}. #' @param chunk.size [\code{integer(1)}]\cr #' Preferred number of jobs in each chunk. #' Can not be used in combination with \code{n.chunks}. #' Note that the ids will get shuffled to balance out possible run time differences. #' Default is not to use chunking. #' @param n.chunks [\code{integer(1)}]\cr #' Preferred number chunks. #' Can not be used in combination with \code{chunk.size}. #' Note that the ids will get shuffled to balance out possible run time differences. #' Default is not to use chunking. #' @param chunks.as.arrayjobs [\code{logical(1)}]\cr #' Submit chunks as array jobs? #' Default is \code{FALSE}. #' @param inds [\code{integer}]\cr #' Indices of ids / chunks to submit. #' Default is all. If ids get chunked, this subsets the list of shuffled ids. #' @param resources [\code{list}]\cr #' Required resources for all batch jobs. #' Default is empty list. #' @return [\code{\link{Registry}}] #' @export batchMapQuick = function(fun, ..., more.args = list(), file.dir = NULL, packages = character(0L), chunk.size, n.chunks, chunks.as.arrayjobs = FALSE, inds, resources = list()) { if (is.null(file.dir)) { # create name for temp file dir in current dir file.dir = tempfile(pattern = "bmq_", tmpdir = getwd()) } else { assertString(file.dir) } id = basename(file.dir) reg = makeRegistry(id = id, file.dir = file.dir, packages = packages) on.exit(messagef("Interrupted. You can find your registry in %s.", reg$file.dir)) # we want to return the reg in any case # otherwise we cannot look at it / do anything with it in case of errors try({ ids = batchMap(reg, fun, ..., more.args = more.args) if (!missing(chunk.size) || !missing(n.chunks)) ids = chunk(ids, chunk.size = chunk.size, n.chunks = n.chunks, shuffle = TRUE) if (!missing(inds)) ids = ids[inds] submitJobs(reg, ids, resources = resources) }, silent = FALSE) on.exit(NULL) return(reg) } BatchJobs/R/clusterFunctionsLSF.R0000644000176200001440000000620212527173323016361 0ustar liggesusers#' @title Create cluster functions for LSF systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{bsub} command. Jobs are #' killed using the \code{bkill} command and the list of running jobs #' is retrieved using \code{bjobs -u $USER -w}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfLSF}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsLSF = function(template.file, list.jobs.cmd = c("bjobs", "-u $USER", "-w")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only # when unfinished jobs are found, and 255 when no jobs are found, # or a non-existent job ID is entered. Sys.setenv(LSB_BJOBS_CONSISTENT_EXIT_CODE = "Y") submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "job") # returns: "Job <128952> is submitted to default queue ." res = runOSCommandLinux("bsub", stdin = outfile, stop.on.exit.code = FALSE) # FIXME filled queues if (res$exit.code > 0L) { cfHandleUnknownSubmitError("bsub", res$exit.code, res$output) } else { # collapse output strings and first number in string is batch.job.id batch.job.id = stri_extract_first_regex(collapse(res$output, sep = " "), "\\d+") makeSubmitJobResult(status = 0L, batch.job.id = batch.job.id) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("bkill", batch.job.id) } listJobs = function(conf, reg) { # JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME # 106560 rogon UNKWN m_amd hpc84 hpc25 QScript Mar 19 12:18 # res = runOSyyCommandLinux("bjobs", c("-u $USER", "-w"), stop.on.exit.code = FALSE) res = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L], stop.on.exit.code = FALSE) if (res$exit.code == 255L && grepl("No unfinished job found", res$output, fixed = TRUE)) return(character(0L)) if (res$exit.code > 0L) stopf("bjobs produced exit code %i; output %s", res$exit.code, res$output) # drop first header line of output out = tail(res$output, -1L) # first number in strings are batch.job.ids stri_extract_first_regex(out, "\\d+") } getArrayEnvirName = function() "LSB_JOBINDEX" makeClusterFunctions(name = "LSF", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/Resources.R0000644000176200001440000000412412623105424014407 0ustar liggesusersresrc = function(res.new) { res.old = getBatchJobsConf()$default.resources if(!isProperlyNamed(res.new) || !isProperlyNamed(res.old)) stop("Resources must be all be uniquely named!") insert(res.old, res.new) } saveResources = function(reg, resources, timestamp = now()) { fn = getResourcesFilePath(reg, timestamp) save2(file = fn, resources = resources) return(timestamp) } #' Function to get job resources in job function. #' #' Return the list passed to \code{\link{submitJobs}}, e.g. #' nodes, walltime, etc. #' #' Can only be called in job function during job execution on slave. #' #' @return [\code{list}]. #' @export getResources = function() { if (!isOnSlave()) stop("getResources can only be called during job execution on slave!") load2(getOption("BatchJobs.resources.path")) } #' Function to get the resources that were submitted for some jobs. #' #' Throws an error if call it for unsubmitted jobs. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs. #' Default is all submitted jobs. #' @param as.list [\code{integer(1)}]\cr #' If \code{FALSE} a data.frame will be returned. #' Default is \code{TRUE}. #' @return [\code{list} | \code{data.frame}]. List (or data.frame) of resource lists as passed to \code{\link{submitJobs}}. #' @export getJobResources = function(reg, ids, as.list = TRUE) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) { ids = dbFindSubmitted(reg) } else { ids = checkIds(reg, ids) nsub = dbFindSubmitted(reg, ids, negate = TRUE, limit = 1L) if (length(nsub) > 0L) stopf("Some of your jobs have not been submitted, so no resources are available, e.g. for id=%i", nsub) } query = sprintf("SELECT job_id, resources_timestamp FROM %s_job_status", reg$id) df = dbSelectWithIds(reg, query, ids) res = namedList(df$job_id) for(ts in unique(df$resources_timestamp)) { res[df$resources_timestamp == ts] = load2(getResourcesFilePath(reg, ts), simplify = FALSE) } if (!as.list) { res = rbindlist(res) setDF(res, rownames = as.character(df$job_id)) } return(res) } BatchJobs/R/clusterFunctionsTorque.R0000644000176200001440000000470612527173323017223 0ustar liggesusers#' @title Create cluster functions for torque-based systems. #' #' @description #' Job files are created based on the brew template #' \code{template.file}. This file is processed with brew and then #' submitted to the queue using the \code{qsub} command. Jobs are #' killed using the \code{qdel} command and the list of running jobs #' is retrieved using \code{qselect}. The user must have the #' appropriate privileges to submit, delete and list jobs on the #' cluster (this is usually the case). #' #' The template file can access all arguments passed to the #' \code{submitJob} function, see here \code{\link{ClusterFunctions}}. #' It is the template file's job to choose a queue for the job #' and handle the desired resource allocations. #' Examples can be found on #' \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfTorque}. #' #' @template arg_template #' @template arg_list_jobs_cmd #' @template ret_cf #' @family clusterFunctions #' @export makeClusterFunctionsTorque = function(template.file, list.jobs.cmd = c("qselect", "-u $USER", "-s EHQRTW")) { assertCharacter(list.jobs.cmd, min.len = 1L, any.missing = FALSE) template = cfReadBrewTemplate(template.file) submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { outfile = cfBrewTemplate(conf, template, rscript, "pbs") res = runOSCommandLinux("qsub", outfile, stop.on.exit.code = FALSE) max.jobs.msg = "Maximum number of jobs already in queue" output = collapse(res$output, sep = "\n") if (grepl(max.jobs.msg, output, fixed = TRUE)) { makeSubmitJobResult(status = 1L, batch.job.id = NA_character_, msg = max.jobs.msg) } else if (res$exit.code > 0L) { cfHandleUnknownSubmitError("qsub", res$exit.code, res$output) } else { makeSubmitJobResult(status = 0L, batch.job.id = stri_trim_both(output)) } } killJob = function(conf, reg, batch.job.id) { cfKillBatchJob("qdel", batch.job.id) } listJobs = function(conf, reg) { # Result is lines of fully quantified batch.job.ids batch.ids = runOSCommandLinux(list.jobs.cmd[1L], list.jobs.cmd[-1L])$output # simplify batch ids of array jobs, i.e. remove the array id from the batch id unique(gsub("\\[[[:digit:]]\\]", "[]", batch.ids)) } getArrayEnvirName = function() "PBS_ARRAYID" makeClusterFunctions(name = "Torque", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/resetJobs.R0000644000176200001440000000367212623105424014404 0ustar liggesusers#' Reset computational state of jobs. #' #' @description #' Reset state of jobs in the database. Useful under two circumstances: #' Either to re-submit them because of changes in e.g. external #' data or to resolve rare issues when jobs are killed in an unfortunate state #' and therefore blocking your registry. #' #' The function internally lists all jobs on the batch system and #' if those include some of the jobs you want to reset, it informs you to kill them first by raising #' an exception. #' If you really know what you are doing, you may set \code{force} to \code{TRUE} to omit this sanity check. #' Note that this is a dangerous operation to perform which may harm #' the database integrity. In this case you HAVE to make externally sure that none of the jobs #' you want to reset are still running. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs to kill. #' Default is none. #' @param force [\code{logical(1)}]\cr #' Reset jobs without checking whether they are currently running. #' READ THE DETAILS SECTION! #' Default is \code{FALSE}. #' @return Vector of reseted job ids. #' @family debug #' @export resetJobs = function(reg, ids, force = FALSE) { checkRegistry(reg, writeable = TRUE) syncRegistry(reg) if (missing(ids) || length(ids) == 0L) return(integer(0L)) ids = checkIds(reg, ids) assertFlag(force) if (!force) { if(is.null(getListJobs()) || is.null(getKillJob())) { stop("Listing or killing of jobs not supported by your cluster functions\n", "You need to set force = TRUE to reset jobs, but see the warning in ?resetJobs") } running = dbFindOnSystem(reg, ids, limit = 10L) if (length(running) > 0L) stopf("Can't reset jobs which are live on system. You have to kill them first!\nIds: %s", collapse(running)) } info("Resetting %i jobs in DB.", length(ids)) dbSendMessage(reg, dbMakeMessageKilled(reg, ids), staged = FALSE) invisible(ids) } BatchJobs/R/sweepRegistry.R0000644000176200001440000000561212571265761015331 0ustar liggesusers#' @title Sweep obsolete files from the file system. #' #' @description #' Removes R scripts, log files, resource informations and temporarily stored configuration files #' from the registry's file directory. Assuming all your jobs completed successfully, none of these are needed #' for further work. This operation potentially releases quite a lot of disk space, depending on the number of your jobs. #' BUT A HUGE WORD OF WARNING: #' IF you later notice something strange and need to determine the reason for it, you are at a huge disadvantage. #' Only do this at your own risk and when you are sure that you have successfully completed a project and only #' want to archive your produced experiments and results. #' #' @template arg_reg #' @param sweep [\code{character}]\cr #' Possible choices: #' Temporary R scripts of jobs, #' really not needed for anything else then execution (\dQuote{scripts}), #' log file of jobs, #' think about whether you later want to inspect them (\dQuote{logs}), #' BatchJobs configuration files which are temporarily stored on submit, #' really not needed for anything else then execution (\dQuote{conf}), #' resource lists of \code{\link{submitJobs}} which are temporarily stored on submit, #' think about whether you later want to inspect them (\dQuote{resources}), #' Default is \code{c("scripts", "conf")}. #' @return [\code{logical}]. Invisibly returns \code{TRUE} on success and \code{FALSE} #' if some files could not be removed. #' @export sweepRegistry = function(reg, sweep = c("scripts", "conf")) { checkRegistry(reg, writeable = TRUE) syncRegistry(reg) assertSubset(sweep, c("scripts", "logs", "resources", "conf")) if (length(dbFindRunning(reg, limit = 1L)) > 0L) stop("Can't sweep registry while jobs are running") fd = reg$file.dir jd = getJobParentDir(fd) rd = getResourcesDir(fd) # failed kill ids are always obsolete because no jobs are running anymore files = list.files(fd, pattern = "^killjobs_failed_ids_*", full.names = TRUE) # sweep configuration if ("conf" %in% sweep) files = c(files, list.files(fd, pattern = "^conf.RData$", full.names = TRUE)) # sweep resources if ("resources" %in% sweep) files = c(files, list.files(rd, full.names = TRUE)) # sweep logs and scripts (in one go if possible) if (all(c("logs", "scripts") %in% sweep)) { files = c(files, list.files(jd, pattern = "^[0-9]+\\.[out|R]$", recursive = TRUE, full.names = TRUE)) } else { if ("logs" %in% sweep) files = c(files, list.files(jd, pattern = "^[0-9]+\\.out$", recursive = TRUE, full.names = TRUE)) if ("scripts" %in% sweep) files = c(files, list.files(jd, pattern = "^[0-9]+\\.R$", recursive = TRUE, full.names = TRUE)) } info("Removing %i files ...", length(files)) ok = all(file.remove(files)) if (!ok) warning("Not all files could be deleted. Check file permissions and try again") return(invisible(ok)) } BatchJobs/R/clusterFunctionsLocal.R0000644000176200001440000000245512413244720016767 0ustar liggesusers#' Create cluster functions for sequential execution on local host. #' #' @description #' All jobs executed under these cluster functions are executed #' sequentially, but in an independent, new R session. #' That is, \code{submitJob} does not return until the #' job has finished. The main use of this \code{ClusterFunctions} #' implementation is to test and debug programs on a local computer. #' #' Listing jobs returns an empty vector (as no jobs can be running when you call this) #' and \code{killJob} returns at once (for the same reason). #' #' @return [\code{\link{ClusterFunctions}}]. #' @family clusterFunctions #' @export makeClusterFunctionsLocal = function() { submitJob = function(conf, reg, job.name, rscript, log.file, job.dir, resources, arrayjobs) { system2(command = file.path(R.home("bin"), "Rscript"), args = rscript, stdout = log.file, stderr = log.file, wait = TRUE) makeSubmitJobResult(status = 0L, batch.job.id = "cfLocal") } killJob = function(conf, reg, batch.job.id) NULL listJobs = function(conf, reg) integer(0L) getArrayEnvirName = function() NA_character_ makeClusterFunctions(name = "Local", submitJob = submitJob, killJob = killJob, listJobs = listJobs, getArrayEnvirName = getArrayEnvirName) } BatchJobs/R/updateRegistry.R0000644000176200001440000000642612623105424015457 0ustar liggesusers#' ONLY FOR INTERNAL USAGE. #' @template arg_reg #' @return [any]. Updated \code{\link{Registry}} or \code{FALSE} if no updates were performed. #' @keywords internal #' @export updateRegistry = function(reg) { UseMethod("updateRegistry") } #' @method updateRegistry Registry #' @export updateRegistry.Registry = function(reg) { # Fix for missing package version (package versions < 1.0.527) if ("BatchJobs" %nin% names(reg$packages)) { reg$packages$BatchJobs = list(version = package_version("1.0.527")) } version.reg = reg$packages$BatchJobs$version version.pkg = packageVersion("BatchJobs") if (version.reg == version.pkg) { return(FALSE) } if (version.reg > version.pkg) { warningf("The registry has been used with BatchJobs version %s, installed is version %s. You should update BatchJobs on this machine.", version.reg, version.pkg) return(FALSE) } # update registry info("Updating Registry and DB to newer version.") if (version.reg < package_version("1.0.606")) { # create new resources dir resources.dir = getResourcesDir(reg$file.dir) checkDir(resources.dir, create = TRUE, check.empty = TRUE) query = sprintf("ALTER TABLE %s_job_status ADD COLUMN resources_timestamp INTEGER", reg$id) dbDoQuery(reg, query, flags = "rwc") # save dummy resources query = sprintf("UPDATE %s_job_status SET resources_timestamp = 0 WHERE submitted IS NOT NULL", reg$id) dbDoQuery(reg, query, flags = "rwc") saveResources(reg, resources = list(), timestamp = 0L) } if (version.reg < package_version("1.0.723")) { checkDir(getPendingDir(reg$file.dir), create = TRUE) } if (version.reg < package_version("1.0.1071")) { checkDir(getExportDir(reg$file.dir), create = TRUE) } if (version.reg < package_version("1.1")) { query = sprintf("ALTER TABLE %s_job_def ADD COLUMN jobname TEXT", reg$id) dbDoQuery(reg, query, flags = "rwc") reg$src.dirs = character(0L) } if (version.reg < package_version("1.2")) { reg$src.files = character(0L) } if (version.reg < package_version("1.4")) { query = sprintf("ALTER TABLE %s_job_status ADD COLUMN memory REAL", reg$id) dbDoQuery(reg, query, flags = "rwc") } if (version.reg < package_version("1.7")) { query = sprintf("ALTER TABLE %s_job_status ADD COLUMN error_time INTEGER", reg$id) dbDoQuery(reg, query, flags = "rwc") reg$read.only = FALSE } reg$packages$BatchJobs$version = version.pkg reg } adjustRegistryPaths = function(reg, file.dir, work.dir) { adjusted = FALSE # adjust file dir if necessary file.dir = sanitizePath(file.dir, make.absolute = TRUE) if (!isDirectory(file.dir)) stopf("file.dir does not exist or is not a directory: %s", file.dir) if (reg$file.dir != file.dir) { reg$file.dir = file.dir adjusted = TRUE } # adjust work dir if necessary if (missing(work.dir)) { if (!isDirectory(reg$work.dir)) warningf("The currently set work.dir '%s' does not exists. Use option 'work.dir' in loadRegistry to change it.", reg$work.dir) } else { work.dir = sanitizePath(work.dir, make.absolute = TRUE) if (!isDirectory(work.dir)) stopf("work.dir does not exist or is not a directory: %s", work.dir) reg$work.dir = work.dir adjusted = TRUE } if (adjusted) reg else FALSE } BatchJobs/R/batchMapResults.R0000644000176200001440000000576512571265761015567 0ustar liggesusers#' Maps a function over the results of a registry by using batchMap. #' #' @param reg [\code{\link{Registry}}]\cr #' Registry whose results should be mapped by \code{fun}. #' @param reg2 [\code{\link{Registry}}]\cr #' Empty registry that should store the job for the mapping. #' @param fun [\code{function(job, res, ...)}]\cr #' Function to map over results of \code{reg}. #' Further arguments come from ... of \code{batchMapResults} and \code{more.args}. #' @param ... [any]\cr #' Furher arguments to vectorize over (list or vector). #' Must all be the same length as number of results in \code{reg}. #' @param ids [\code{integer}]\cr #' Ids of jobs whose results should be mapped with \code{fun}. #' Default is all jobs. #' @param part [\code{character}] #' Only useful for multiple result files, then defines which result file part(s) should be loaded. #' \code{NA} means all parts are loaded, which is the default. #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @return Vector of type \code{integer} with job ids. #' @export #' @examples #' reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) #' # square some numbers #' f = function(x) x^2 #' batchMap(reg1, f, 1:10) #' #' # submit jobs and wait for the jobs to finish #' submitJobs(reg1) #' waitForJobs(reg1) #' #' # look at results #' reduceResults(reg1, fun = function(aggr,job,res) c(aggr, res)) #' #' reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) #' #' # define function to tranform results, we simply do the inverse of the squaring #' g = function(job, res) sqrt(res) #' batchMapResults(reg1, reg2, fun = g) #' #' # submit jobs and wait for the jobs to finish #' submitJobs(reg2) #' waitForJobs(reg2) #' #' # check results #' reduceResults(reg2, fun = function(aggr,job,res) c(aggr, res)) batchMapResults = function(reg, reg2, fun, ..., ids, part = NA_character_, more.args = list()) { # FIXME conserve jobnames checkRegistry(reg, writeable = FALSE) syncRegistry(reg) checkRegistry(reg2, writeable = TRUE) syncRegistry(reg2) assertFunction(fun, c("job", "res")) if (missing(ids)) { ids = dbGetJobIdsIfAllDone(reg) } else { ids = checkIds(reg, ids) if (length(dbFindDone(reg, ids, negate = TRUE, limit = 1L)) > 0L) stop("Not all jobs with corresponding ids finished (yet)!") } checkMoreArgs(more.args, reserved = c(".reg", ".fun", ".part")) if (dbGetJobCount(reg2) > 0L) stop("Registry 'reg2' is not empty!") if(reg$file.dir == reg2$file.dir) stop("Both registries cannot point to the same file dir. Files would get overwritten!") reg2$packages = insert(reg2$packages, reg$packages) saveRegistry(reg2) batchMap(reg2, batchMapResultsWrapper, ids, ..., more.args = c(more.args, list(.reg = reg, .fun = fun, .part = part))) } batchMapResultsWrapper = function(id, ..., .reg, .fun, .part) { .fun(job = getJob(.reg, id, check.id = FALSE), res = getResult(.reg, id, part = .part), ...) } BatchJobs/R/findJobs.R0000644000176200001440000000333212623105424014173 0ustar liggesusers#' Finds ids of jobs that match a query. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Subset of job ids to restrict the result to. #' Default is all jobs. #' @param pars [R expression]\cr #' All jobs whose parameters match the given expression are selected. #' This implies that you have named the parameters when you passed the vectors. #' If you forgot to do this you can use \code{.arg1}, \code{.arg2}, etc., to refer to the #' the unnamed ones. #' @param jobnames [\code{character}]\cr #' Restrict to jobs with stored names. Exact matching is used. #' @return [\code{integer}]. Ids for jobs which match the query. #' @export #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x, y) x * y #' batchExpandGrid(reg, f, x = 1:2, y = 1:3) #' findJobs(reg, pars = (y > 2)) findJobs = function(reg, ids, pars, jobnames) { checkRegistry(reg, strict = TRUE, writeable = FALSE) syncRegistry(reg) if (!missing(ids)) checkIds(reg, ids) if (missing(pars) && missing(jobnames)) return(getJobIds(reg)) if (!missing(jobnames)) { assertCharacter(jobnames, any.missing = FALSE) ids = dbMatchJobNames(reg, ids, jobnames) } if (!missing(pars)) { jobs = dbGetJobs(reg, ids) rename = function(pars) { ns = names(pars) if (is.null(ns)) { ns = rep.int("", length(pars)) } j = which(is.na(ns) | ns == "") ns[j] = paste0(".arg", seq_along(j)) setNames(pars, ns) } ind = vlapply(jobs, function(job, pars, ee) eval(pars, rename(job$pars), ee), pars = substitute(pars), ee = parent.frame()) ids = extractSubList(jobs[!is.na(ind) & ind], "id", element.value = integer(1L)) } ids } BatchJobs/R/getSSHWorkersInfo.R0000644000176200001440000000275512413244720015773 0ustar liggesusers#' Print and return R installation and other information for SSH workers. #' #' @description #' Workers are queried in parallel via \code{\link{callFunctionOnSSHWorkers}}. #' #' The function will display a warning if the first lib path on the worker #' is not writable as this indicates potential problems in the configuration #' and \code{\link{installPackagesOnSSHWorkers}} will not work. #' #' @param nodenames [\code{character}]\cr #' Nodenames of workers. #' @return [\code{list}]. Displayed information as a list named by nodenames. #' @export #' @seealso \code{\link{callFunctionOnSSHWorkers}} getSSHWorkersInfo = function(nodenames) { fun = function() { lib.paths = .libPaths() list( r.home = R.home(), session.info = sessionInfo(), lib.paths = lib.paths, is.lib.path.writeable = is.accessible(head(lib.paths, 1L)) ) } res = callFunctionOnSSHWorkers(nodenames, fun = fun, consecutive = FALSE, show.output = FALSE, use.names = TRUE, simplify = FALSE) for (nn in nodenames) { r = res[[nn]] messagef("Node: %s", nn) messagef(r$session.info$R.version$version.string) messagef("Platform: %s", r$session.info$platform) messagef("R Home: %s", r$r.home) messagef("First lib path: %s", head(r$lib.paths, 1L)) messagef("") } notok = names(Filter(function(r) !r$is.lib.path.writeable, res)) if (length(notok) > 0L) warningf("On the following nodes the first lib path is not writeable: %s", collapse(notok)) invisible(res) } BatchJobs/R/grepLogs.R0000644000176200001440000000525512623105424014225 0ustar liggesusers#' Grep log files for a pattern. #' #' @description #' Searches for occurence of \code{pattern} in log files. #' #' @template arg_reg #' @param ids [\code{integer}]\cr #' Ids of jobs to grep. #' Default is all terminated jobs (done + errors). #' @param pattern [\code{character(1)}]\cr #' Pattern to search for. See \code{\link{grep}}. #' Default is \code{"warn"}. #' @param ignore.case [\code{logical(1)}]\cr #' Ignore case. See \code{\link{grep}}. #' Default is \code{TRUE}. #' @param verbose [\code{logical(1)}]\cr #' Print matches. #' Default is \code{FALSE}. #' @param range [\code{integer(1)}]\cr #' If \code{verbose} is set to \code{TRUE}, print \code{range} #' leading and trailing lines for contextual information about the warning. #' Default is \code{2}. #' @return [\code{integer}]. Ids of jobs where pattern was found in the log file. #' @family debug #' @export grepLogs = function(reg, ids, pattern = "warn", ignore.case = TRUE, verbose = FALSE, range = 2L) { checkRegistry(reg, writeable = FALSE) syncRegistry(reg) if (missing(ids)) { ids = dbFindTerminated(reg) } else { nterminated = dbFindTerminated(reg, ids, negate = TRUE, limit = 1L) if (length(nterminated) > 0L) stopf("Not all jobs with provided ids have finished yet and therefore possess no log file, e.g. id=%i.", nterminated[1L]) } assertString(pattern) assertFlag(ignore.case) assertFlag(verbose) range = asCount(range) fids = dbGetFirstJobInChunkIds(reg, ids) fns = getLogFilePath(reg, ifelse(is.na(fids), ids, fids)) matched = logical(length(ids)) getLines = function(lines, match, range) { start = max(1L, match - range) stop = min(length(lines), match + range) collapse(lines[start:stop], "\n") } for(i in seq_along(fns)) { if (!file.exists(fns[i])) stopf("File '%s' does not exist.", fns[i]) # read lines from log and trim to output of job with id 'ids[i]' lines = readLines(fns[i]) start = grep(sprintf("^########## Executing jid=%i ##########$", ids[i]), lines) if (length(start) != 1L) stop("The output of the job with id=%i could not be found in file '%s' or was found more than once", ids[i], fns[i]) end = head(grep("^########## Executing jid = [0-9]+ ##########$", tail(lines, -start)), 1L) lines = lines[start:min(start+end, length(lines))] matches = grep(pattern, lines, ignore.case = ignore.case) matched[i] = (length(matches) > 0L) if (verbose && matched[i]) { messagef("\n##### Matches for job with id=%i (%s) #####", ids[i], basename(fns[i])) message(collapse(vcapply(matches, getLines, lines = lines, range = range), "\n---\n")) } } ids[matched] } BatchJobs/R/batchMap.R0000644000176200001440000000466613207234653014175 0ustar liggesusers#' @title Maps a function over lists or vectors, adding jobs to a registry. #' #' @description #' You can then submit these jobs to the batch system. #' #' @param reg [\code{\link{Registry}}]\cr #' Empty Registry that will store jobs for the mapping. #' @param fun [\code{function}]\cr #' Function to map over \code{...}. #' @param ... [any]\cr #' Arguments to vectorize over (list or vector). #' @param more.args [\code{list}]\cr #' A list of other arguments passed to \code{fun}. #' Default is empty list. #' @param use.names [\code{logical(1)}]\cr #' Store parameter names to enable named results in \code{\link{loadResults}} and some other functions. #' Default is \code{FALSE}. #' @return Vector of type \code{integer} with job ids. #' @examples #' reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) #' f = function(x) x^2 #' batchMap(reg, f, 1:10) #' print(reg) #' @export batchMap = function(reg, fun, ..., more.args = list(), use.names = FALSE) { checkRegistry(reg, strict = TRUE, writeable = TRUE) assertFunction(fun) ddd = list(...) if (length(ddd) == 0L) return(invisible(integer(0L))) n = unique(viapply(ddd, length)) if(length(n) != 1L) stop("All args in '...' must be of the same length!") if (n == 0L) return(invisible(integer(0L))) checkMoreArgs(more.args) assertFlag(use.names) if (dbGetJobCount(reg) > 0L) stop("Registry is not empty!") info("Adding %i jobs to DB.", n) # create seeds seed = reg$seed seeds = addIntModulo(seed, seq(0L, n-1L)) # serialize pars to char vector pars = mapply(function(...) { rawToChar(serialize(list(...), connection = NULL, ascii = TRUE)) }, ..., USE.NAMES = FALSE) fun.id = saveFunction(reg, fun, more.args) # generate jobnames col if (use.names) { jobname = getArgNames(ddd) if (is.null(jobname)) jobname = rep.int(NA_character_, n) } else { jobname = rep.int(NA_character_, n) } # add jobs to DB n = dbAddData(reg, "job_def", data = data.frame(fun_id = fun.id, pars = pars, jobname = jobname, stringsAsFactors = FALSE)) job.def.ids = dbGetLastAddedIds(reg, "job_def", "job_def_id", n) n = dbAddData(reg, "job_status", data = data.frame(job_def_id = job.def.ids, seed = seeds, stringsAsFactors = FALSE)) job.ids = dbGetLastAddedIds(reg, "job_status", "job_id", n) # we can only create the dir after we have obtained the ids from the DB createShardedDirs(reg, job.ids) invisible(job.ids) } BatchJobs/R/zzz.R0000644000176200001440000001032613207276646013311 0ustar liggesusers#' The BatchJobs package #' #' @section Additional information: #' #' \describe{ #' \item{Homepage:}{\url{https://github.com/tudo-r/BatchJobs}} #' \item{Wiki:}{\url{https://github.com/tudo-r/BatchJobs/wiki}} #' \item{FAQ:}{\url{https://github.com/tudo-r/BatchJobs/wiki/FAQ}} #' \item{Configuration:}{\url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}} #' } #' #' The package currently support the following further R options, which you can set #' either in your R profile file or a script via \code{\link{options}}: #' #' \describe{ #' \item{BatchJobs.verbose}{This boolean flag can be set to \code{FALSE} to reduce the #' console output of the package operations. Usually you want to see this output in interactive #' work, but when you use the package in e.g. knitr documents, #' it clutters the resulting document too much.} #' \item{BatchJobs.check.posix}{If this boolean flag is enabled, the package checks your #' registry file dir (and related user-defined directories) quite strictly to be POSIX compliant. #' Usually this is a good idea, you do not want to have strange chars in your file paths, #' as this might results in problems when these paths get passed to the scheduler or other #' command-line tools that the package interoperates with. #' But on some OS this check might be too strict and cause problems. #' Setting the flag to \code{FALSE} allows to disable the check entirely. #' The default is \code{FALSE} on Windows systems and \code{TRUE} else.} #' } #' #' @docType package #' @name BatchJobs NULL #' @import utils #' @import stats #' @import methods #' @import checkmate #' @import data.table #' @import DBI #' @import RSQLite #' @importFrom digest digest #' @importFrom brew brew #' @importFrom sendmailR sendmail #' @importFrom stringi stri_extract_first_regex #' @importFrom stringi stri_trim_both #' @importFrom stringi stri_split_fixed #' @importFrom stringi stri_split_regex #' @importFrom BBmisc %nin% #' @importFrom BBmisc chunk #' @importFrom BBmisc checkListElementClass #' @importFrom BBmisc clipString #' @importFrom BBmisc collapse #' @importFrom BBmisc convertToShortString #' @importFrom BBmisc convertListOfRowsToDataFrame #' @importFrom BBmisc dropNamed #' @importFrom BBmisc extractSubList #' @importFrom BBmisc filterNull #' @importFrom BBmisc insert #' @importFrom BBmisc isDirectory #' @importFrom BBmisc is.error #' @importFrom BBmisc isProperlyNamed #' @importFrom BBmisc isScalarNA #' @importFrom BBmisc isWindows #' @importFrom BBmisc lsort #' @importFrom BBmisc namedList #' @importFrom BBmisc names2 #' @importFrom BBmisc makeFileCache #' @importFrom BBmisc makeProgressBar #' @importFrom BBmisc makeSimpleFileLogger #' @importFrom BBmisc save2 load2 #' @importFrom BBmisc requirePackages #' @importFrom BBmisc setClasses setColNames setRowNames #' @importFrom BBmisc seq_col seq_row #' @importFrom BBmisc suppressAll #' @importFrom BBmisc system3 #' @importFrom BBmisc vcapply viapply vlapply vnapply #' @importFrom BBmisc warningf stopf messagef catf #' @importFrom BBmisc which.first NULL .BatchJobs.conf = new.env(parent = emptyenv()) .BatchJobs.conffiles = character(0L) .onAttach = function(libname, pkgname) { packageStartupMessage("The development of BatchJobs and BatchExperiments is discontinued.") packageStartupMessage("Consider switching to 'batchtools' for new features and improved stability") if (getOption("BatchJobs.verbose", default = TRUE)) { cf = .BatchJobs.conffiles packageStartupMessage(sprintf("Sourced %i configuration files: ", length(cf))) for (i in seq_along(cf)) packageStartupMessage(sprintf(" %i: %s", i, cf[i])) conf = getConfig() packageStartupMessage(printableConf(conf)) } } .onLoad = function(libname, pkgname) { options(BatchJobs.check.posix = getOption("BatchJobs.check.posix", default = !isWindows())) options(BatchJobs.clear.function.env = getOption("BatchJobs.clear.function.env", default = FALSE)) backports::import(pkgname) if (!isOnSlave()) { assignConfDefaults() if (getOption("BatchJobs.load.config", TRUE)) { pkg = if(missing(libname) || missing(pkgname)) find.package(package = "BatchJobs") else file.path(libname, pkgname) .BatchJobs.conffiles <<- readConfs(pkg) } } } BatchJobs/R/WorkerLinux.R0000644000176200001440000001137312623105424014732 0ustar liggesusers# ******************** Constructors ******************** # Construct a remote worker for a Linux machine via SSH. makeWorkerRemoteLinux = function(nodename, ssh.cmd, ssh.args, rhome, r.options, script, ncpus, max.jobs, max.load, nice) { makeWorker(ssh = TRUE, nodename, ssh.cmd, ssh.args, rhome, r.options, script, ncpus, max.jobs, max.load, nice, c("WorkerRemoteLinux", "WorkerLinux")) } # Construct a worker for local Linux machine to spawn parallel jobs. makeWorkerLocalLinux = function(r.options, script, ncpus, max.jobs, max.load, nice) { makeWorker(ssh = FALSE, nodename = "localhost", ssh.cmd = NULL, ssh.args = NULL, R.home(), r.options, script, ncpus, max.jobs, max.load, nice, c("WorkerLocalLinux", "WorkerLinux")) } # ******************** Interface implementation ******************** #' @export getWorkerNumberOfCPUs.WorkerLinux = function(worker) { as.integer(runWorkerCommand(worker, "number-of-cpus")) } #' @export getWorkerStatus.WorkerLinux = function(worker, file.dir) { res = runWorkerCommand(worker, "status", file.dir) setNames( as.list(as.numeric(stri_split_regex(res, "\\s+")[[1L]])), c("load", "n.rprocs", "n.rprocs.50", "n.jobs") ) } #' @export startWorkerJob.WorkerLinux = function(worker, rfile, outfile) { runWorkerCommand(worker, "start-job", c(worker$rhome, worker$nice, worker$r.options, rfile, outfile)) } #' @export killWorkerJob.WorkerLinux = function(worker, pid) { runWorkerCommand(worker, "kill-job", pid) } #' @export listWorkerJobs.WorkerLinux = function(worker, file.dir) { res = runWorkerCommand(worker, "list-jobs", file.dir) gsub("^\\s+|\\s+$", "", res) } # ******************** Run commands on OS ******************** # Runs a command, either by SSH or directly on localhost. # @param cmd [\code{character(1)}] # System command to run. # @param args [\code{character}] # System command arguments. # Default is \code{character(0)}. # @param stdin [\code{character(1)}] # See \code{\link{system3}}. # Default is \dQuote{}. # @param stop.on.exit.code [\code{character}] # See \code{\link{system3}}. # Default is \code{TRUE}. # @param ssh [\code{logical(1)}] # Use SSH? # Default is \code{FALSE}. # @param nodename [\code{character(1)}] # Nodename for SSH. # @return See \code{\link{system3}}. runOSCommandLinux = function(cmd, args = character(0L), stdin = "", stop.on.exit.code = TRUE, ssh = FALSE, ssh.cmd, ssh.args, nodename) { conf = getBatchJobsConf() if (ssh) { sys.cmd = ssh.cmd sys.args = c(ssh.args, nodename, sprintf("'%s'", collapse(c(cmd, args), sep = " "))) } else { sys.cmd = cmd sys.args = args } if (conf$debug) { catf("OS cmd: %s %s", sys.cmd, collapse(sys.args, " ")) res = try(system3(sys.cmd, sys.args, stdin = stdin, stdout = TRUE, stderr = TRUE, wait = TRUE, stop.on.exit.code = stop.on.exit.code)) catf("OS result:") print(res) } else { res = system3(sys.cmd, sys.args, stdin = stdin, stdout = TRUE, stderr = TRUE, wait = TRUE, stop.on.exit.code = stop.on.exit.code) } if(is.error(res)) stopf("Error in runLinuxOSCommand: %s (cmd: %s || args: %s)", as.character(res), sys.cmd, collapse(sys.args)) res } # Find helper script on a Linux machine in package dir. # @param rhome [\code{character(1)}] # RHOME dir. # @param r.options [\code{character}] # Options for R and Rscript, one option per element of the vector, # a la "--vanilla". # @param ssh [\code{logical(1)}] # Use SSH? # Default is \code{FALSE}. # @param nodename [\code{character(1)}] # Nodename for SSH. # @return [\code{character(1)}]. Path of script. findHelperScriptLinux = function(rhome, r.options, ssh = FALSE, ssh.cmd, ssh.args, nodename) { # i think we dont need to quote anything here, because system2 uses shQuote if (rhome == "") rscript = "Rscript" else rscript = file.path(rhome, "bin", "Rscript") minus.e = "-e \"message(normalizePath(system.file(\\\"bin/linux-helper\\\", package = \\\"BatchJobs\\\")))\"" args = c(r.options, minus.e) # only take the last line, if stuff is printed by Rscript before our message tail(runOSCommandLinux(rscript, args, ssh = ssh, ssh.cmd = ssh.cmd, ssh.args = ssh.args, nodename = nodename)$output, 1L) } # Perform a batch helper command on a Linux machine. # @param worker [\code{\link{WorkerLinux}}] # Worker. # @param command [\code{character(1)}] # Helper command. # @param args [\code{character}] # Arguments for helper command. # See documenation of linux-helper. runWorkerCommand = function(worker, command, args = character(0L)) { # in paths can be whitespaces and other bad stuff, quote it! args = sprintf("\"%s\"", args) script.args = c(command, args) runOSCommandLinux(worker$script, script.args, ssh = worker$ssh, ssh.cmd = worker$ssh.cmd, ssh.args = worker$ssh.args, nodename = worker$nodename)$output } BatchJobs/R/scheduleWorkerJobs.R0000644000176200001440000000252612357424573016262 0ustar liggesusers# is a worker busy, see rules below getWorkerSchedulerStatus = function(worker) { # we have already used up our maximal load on this node if (worker$status$n.jobs >= worker$max.jobs) return("J") # should not have too much load average if (worker$status$load[1L] > worker$max.load) return("L") # there are already ncpus expensive R jobs running on the node if (worker$status$n.rprocs.50 >= worker$ncpus) return("R") # should not have too many R sessions open if(worker$status$n.rprocs >= 3 * worker$ncpus) return("r") # else all clear, submit the job! return("A") } # update status of worker IN PLACE updateWorker = function(worker, file.dir, tdiff) { time = now() if (worker$available == "A" || time - worker$last.update >= tdiff) { worker$last.update = time worker$status = getWorkerStatus(worker, file.dir) worker$available = getWorkerSchedulerStatus(worker) } } # find worker via isBusyWorker and update workers while looking # workers with a low load are more likely to be selected when there are # multiple workers available findWorker = function(workers, file.dir, tdiff) { lapply(workers, updateWorker, file.dir = file.dir, tdiff = tdiff) rload = vnapply(workers, function(w) w$status$load / w$ncpus) Find(function(w) w$available=="A", sample(workers, prob = 1 / (rload + 0.1)), nomatch = NULL) } BatchJobs/R/applyJobFunction.R0000644000176200001440000000122112623105424015716 0ustar liggesusers#' applyJobFunction #' ONLY FOR INTERNAL USAGE. #' @template arg_reg #' @param job [\code{\link{Job}}]\cr #' Job. #' @param cache [\code{FileCache}]\cr #' Instance of \code{\link[BBmisc]{makeFileCache}}. #' @return [any]. Result of job. #' @keywords internal #' @export applyJobFunction = function(reg, job, cache) { UseMethod("applyJobFunction") } #' @method applyJobFunction Registry #' @export applyJobFunction.Registry = function(reg, job, cache) { fn = file.path(getFunDir(reg$file.dir), sprintf("%s.RData", job$fun.id)) stuff = cache(fn, parts = c("fun", "more.args"), simplify = FALSE) do.call(stuff$fun, c(job$pars, stuff$more.args)) } BatchJobs/R/addJob.R0000644000176200001440000000032712460747041013627 0ustar liggesuserssaveFunction = function(reg, fun, more.args) { fun = checkUserFunction(fun) fun.id = digest(list(fun, more.args)) save2(file = getFunFilePath(reg, fun.id), fun = fun, more.args = more.args) return(fun.id) } BatchJobs/MD50000644000176200001440000002620413207317714012432 0ustar liggesusersdde0ff3d60ecb9b449bc1b9210c999c4 *DESCRIPTION 9d6fc77f9cd80b60f62389c553a386c7 *LICENSE 16fba80e1f9a6cb744cec72844a36531 *NAMESPACE 4e41f20792d4f2fae8d0651f299739ef *NEWS dd63638b3add6d604d0ebb81789ca989 *R/Exports.R 48418bda70c625c777a7f7849f5c3fb3 *R/Job.R be18872f0ee37c037b499a65d5138078 *R/Packages.R e9fb228de1c90320c3b569ee25f8c66b *R/Registry.R e8a3a8d18859aece05208bbce370f61b *R/Resources.R c97ed6738af9426582d84184121218c3 *R/Worker.R a05a5ec448a448e7cf56b6cfa4627c16 *R/WorkerLinux.R 658f8c315735400ca5cf9fc0a2e33016 *R/addJob.R ac1772bcdee4d1a915f356b0a58118bd *R/applyJobFunction.R 0dc8510df9ac6d0d0b84091be67b8849 *R/batchApply.R bacf98f9d3e64f4d557357b59ddd66e5 *R/batchExpandGrid.R 01f38c0b4575e3a77e9a67cbb3c6bf56 *R/batchMap.R 5e5a54ecbc6b5635bf40edfd268151dc *R/batchMapQuick.R c8e121f26eec6a0a2b50a72817abba0b *R/batchMapResults.R 561b8cdb61b04d640bb6bf92438e065a *R/batchQuery.R d1aa124b0d7708797a74806a038a8e1e *R/batchReduce.R 111885d5bbace947129899c805321b15 *R/batchReduceResults.R c4fbff686659b0f7ef24b23a3faab700 *R/buffer.R cf814343a696dfcff087e5b3e3703e5b *R/callFunctionOnSSHWorkers.R fab25278bcc83a38f8db06ae6b817afb *R/checkIdValid.R 22e7dfc613f545c76e35b235e9b0e8b0 *R/checkIds.R c8487612d382dbfaec57e329112b1166 *R/clusterFunctions.R 2fcadda1f0235dc793e9d3f5d70964ed *R/clusterFunctionsHelpers.R f6f6934bcbc0cc1be3a49c8aaa42d959 *R/clusterFunctionsInteractive.R 29515d1b9477e1f15409b66a537f1027 *R/clusterFunctionsLSF.R 4e507dccaf6f9b6ae58ced8e92b42018 *R/clusterFunctionsLocal.R 7f44a660f97429f7dea27d6af7927823 *R/clusterFunctionsMulticore.R 89afd24fa287aca506f93df5f8ce7931 *R/clusterFunctionsOpenLava.R 1d4655c54104281cd52867571e23d670 *R/clusterFunctionsSGE.R 05d92243f7ccd492d693611c66ad8ff0 *R/clusterFunctionsSLURM.R fa64d8851f5a083160e9d358b2b5dad8 *R/clusterFunctionsSSH.R 021935873ebc8207322e426ebf532d82 *R/clusterFunctionsTorque.R 8a51776fe0263206dce2290bda268d77 *R/conf.R 075cbdf1d8f4ace221e52a120ffda279 *R/database.R 3d76275ec0683ed18c4295e65de23c92 *R/debugMulticore.R 6c5ce4e718c36b638d967d7fbf20f04c *R/debugSSH.R e7915b183e499baf48d26a66f7e4d0a9 *R/doJob.R 0ad8f262924e4aa8915eed4b7c5a59cc *R/filenames.R 50f9958156af62a1e9ba3a0695144a98 *R/filterResults.R 3247b8af8c28e7e1366c56b6288febd9 *R/findJobs.R 7bd8b499561f752370560d9749d8bd2b *R/findStatus.R bab10b4d6e0aa24a33f9c45c39dc083d *R/getErrorMessages.R ee15ecf22a31d2cac1606b1841a2cf9e *R/getJob.R 4871505379d00bbddf113340f48759a3 *R/getJobInfo.R 34d4d53e98bc8b511f10a8a065f03a48 *R/getJobLocation.R 26f893b9a2c2bd1aac0426c0d1bfe692 *R/getJobParamDf.R 449bae60c36b8cbe96f41c21436798bd *R/getLogFiles.R 02c342cac122cffdb158ef48be98d701 *R/getSSHWorkersInfo.R 260f1eb32da6e9ee296e00edd0042e4b *R/grepLogs.R 3864b5ac817f80ee12f6c679d4de9aa6 *R/helpers.R fc7b467082b004f08239e5c3796a760a *R/installPackagesOnSSHWorkers.R 38156503d672826c1cd25e5c00252f39 *R/killJobs.R e1c22132c77a9caac22b361771b724eb *R/loadRegistry.R 5d4d7f725f966998d97852f5e4edf7c0 *R/loadResult.R c0aa241eb7969a1aa2abf2b72a65025b *R/loadResults.R b66f69f2b58d78105bede355419935c4 *R/reduceResults.R 191a98316c5f8b8f8d7ff2d0afae112e *R/resetJobs.R a0cbe2f88a748469eba52c0ebb72f498 *R/sanitizePath.R 54f6af0f47082c1d0b076d67372a2cb5 *R/scheduleWorkerJobs.R a3030fb66c111c3fbb18ad66b2b17a99 *R/sendMail.R c33c7216ef5e970ff1269e452fa8a525 *R/setJobFunction.R 13d2dadbce92b51458e82b0983833eb9 *R/setJobNames.R bb0f395cf2caa8328d2364a0347537ec *R/showClusterStatus.R 39c410ceada87d0f135ac07b3baff5ba *R/showLog.R 982e57a1b302a2889e0f195ae6a4e182 *R/showStatus.R 38e472da280ab30ca1c212e2dc0fa27d *R/sourceRegistryFiles.R 01445c94d81dcfc859fb67e579baabde *R/submitJobs.R f4a65ac31b1f4d4ae21d963e72b4bc41 *R/sweepRegistry.R bb1dcfd5ca85ac0a55009ccc63c12a20 *R/syncRegistry.R b99be9f7a8a23b945f0e454568188b7e *R/testJob.R 04d1e9da4f68eadf8abc55729da8aa6f *R/updateRegistry.R cc3a452fe7c86b9680a6d0e761d8467f *R/waitForJobs.R 66c7d3a39bd7326764b4d66f2fcf5e18 *R/writeFiles.R c5ed1afdfa4aec7a939a832b17ad1f72 *R/zzz.R 1f72c38068bf885bbcb699519e99e8a8 *inst/CITATION d9ddf818a9837e0159ce6826293d0902 *inst/bin/linux-helper b7e4f63a659e5fe827946829a6167b2c *inst/etc/BatchJobs_global_config.R 0db192cb57c417c94ed8f326aaf4d3bd *man/BatchJobs.Rd 41cf77fb1c8719adefc6722f4de960f0 *man/addRegistryPackages.Rd 4b00cb3fd8b8fd07ca4d989f635b13b9 *man/addRegistrySourceDirs.Rd 7550073639a561ee5830b6367c7d79e0 *man/addRegistrySourceFiles.Rd 4d30edcfb8e6a018be2b71c14e7ebb31 *man/applyJobFunction.Rd 3c69d6b7ff3807098b8ec10efc2d13fe *man/batchExpandGrid.Rd f69c65723e9b975ee83e2970b888b3b6 *man/batchExport.Rd a37aa65e3d4821c2c2d7245b3e3a9bf5 *man/batchMap.Rd 48301a7f9be13e80dfc5bb6b53614ca8 *man/batchMapQuick.Rd 5c85f4ec8a47c3db43cfa31b2a7b75c8 *man/batchMapResults.Rd 2d90f8f2d1ad1a35716b4ee13af7b577 *man/batchQuery.Rd 2b4feb413412a655f6d3d8c4a0debced *man/batchReduce.Rd daf488f64a460501ea578e79abd47a4c *man/batchReduceResults.Rd 306d333cd3364b27f366418a70945b1a *man/batchUnexport.Rd d8d327280266c701954ae0ad06ece7cc *man/callFunctionOnSSHWorkers.Rd ee542fb5bbcc485062ff9f2171567c54 *man/cfBrewTemplate.Rd ea585c705ad54f23b571c974f840d99d *man/cfHandleUnknownSubmitError.Rd 658bef9f6ca20071d5702ea52ca57ef7 *man/cfKillBatchJob.Rd 6e3d93323ac774878ae1c943b8a9d958 *man/cfReadBrewTemplate.Rd af2ad34ae22213864bf531eb1b838d80 *man/checkIds.Rd 02164103f08a4c990507f5a0d0921bc3 *man/configuration.Rd b4aee6a69d63fc48778ff345888e8a97 *man/copyRequiredJobFiles.Rd 6d31830ceff7a355bc9581788c4a0293 *man/dbCreateJobDefTable.Rd db7f6d900d63288d40006653c68ca584 *man/dbGetJobs.Rd 56b1bdbfbfadfa666455228a664361e4 *man/debugMulticore.Rd 6ce7df83a2af6704df32b6f0ec9d9a3e *man/debugSSH.Rd d4763f5939c15e1a023c0388a3e352cd *man/filterResults.Rd d60367a2a8732b510bbe5d43f4ddaceb *man/findJobs.Rd 2eb7a338e03e2b0731db73f4b4d8a8d9 *man/findState.Rd a1fca57063ee5c0a46f1afdb7c40a09d *man/getConfig.Rd a94f250f3851c2c9ef7e6e2be1b5d68d *man/getErrorMessages.Rd ce12af8af1f7fc1db96a072b4106254f *man/getJob.Rd 526fcb539562fdfd0bbe41d97348f74e *man/getJobIds.Rd 67b92e818ad0ed8cb3e6fa409d595d19 *man/getJobInfo.Rd 348785f1f4933551a5dba8b4a7bfc617 *man/getJobLocation.Rd f6ccc4233acafae6e9188c36a65947fe *man/getJobNr.Rd dcb4c7c745d044da0fb48605a23e5adb *man/getJobParamDf.Rd e9f2184fd66aafbeeb43936cf6dbe043 *man/getJobResources.Rd 97231765c53d360791b2335e5804a115 *man/getJobs.Rd f0995a511f844b0035815eae789c3074 *man/getLogFiles.Rd 1bb7483ddf3f0d2a8bd26f482884842b *man/getResources.Rd 234add9ec0e4ea13680b69634a5f199c *man/getSSHWorkersInfo.Rd 9dffa24fbdc2555f3deeafb279542870 *man/grepLogs.Rd e2d10d4aeb7fe4c271173d908f89b502 *man/installPackagesOnSSHWorkers.Rd 5a293beefaadbb16b1df529616f6f41e *man/killJobs.Rd d71c6b14f396615ad6cf28837ac81df6 *man/loadConfig.Rd c85ba2255e29ab512cef69b601f7d0af *man/loadExports.Rd 8e49ae7db275e21970630af5cb693285 *man/loadRegistry.Rd 90c8a09724a55d429f8ef2bac062db43 *man/loadResult.Rd 70ecb08b09f4b188937193312ffc7e89 *man/loadResults.Rd f453d109f5578ca6ee3977a50685eb17 *man/makeClusterFunctions.Rd aa996826dcc548306fe0ab69769d6999 *man/makeClusterFunctionsInteractive.Rd 991723f6d7bcf29b1cb7c4baf547da2c *man/makeClusterFunctionsLSF.Rd fc70f184b162c962b44f5d4a3b4c94b1 *man/makeClusterFunctionsLocal.Rd 239132b66042d2cd3be05d33efd37659 *man/makeClusterFunctionsMulticore.Rd 1176964ebeec764a18191bc231238c78 *man/makeClusterFunctionsOpenLava.Rd fa6d69e206ba1f529789c571bec16b09 *man/makeClusterFunctionsSGE.Rd b4151c9460e674efd1f27b38e53ce2f2 *man/makeClusterFunctionsSLURM.Rd f2db5aa400c7a7d58068182f40b96763 *man/makeClusterFunctionsSSH.Rd 09becccf43c9a6af87a1a1340b244220 *man/makeClusterFunctionsTorque.Rd 720ccf7f5b2085c591b73e87433dc3bf *man/makeJob.Rd 3b7f9db3f81bc28a47947bdc222a3db5 *man/makeRegistry.Rd 66572d64327710555db1d28e98eab40a *man/makeSSHWorker.Rd 69aa66da9d9dfa00e9c54ca33e5dbcef *man/makeSubmitJobResult.Rd 66276e125d520ca4212219be616b4a3e *man/reduceResults.Rd 981d97edd019042da431d58978c80335 *man/removeRegistry.Rd 47fa8eefbe6ccb1ed6c2df76e72cf0c8 *man/removeRegistryPackages.Rd 8e2feee1a3eb26b3ff1115bb8837bff8 *man/removeRegistrySourceDirs.Rd b0589913cc27318c23f1f0c34eb1dc05 *man/removeRegistrySourceFiles.Rd dcebb05792879a01752374a20c6fa666 *man/resetJobs.Rd 0d720b11bf7cf712cf2d99a838eba203 *man/sanitizePath.Rd 6efc249b37a3513536e327b34a9e9493 *man/setConfig.Rd 4fee7290f56f8b803c1a36d2d06ab2e1 *man/setJobFunction.Rd 0db939346bc38f777be939d131dc8d61 *man/setJobNames.Rd b87976a3e3a98e6bc6fc4bb0d2f52013 *man/setRegistryPackages.Rd 928415de05ea6ae1f21eebaecb2e88fc *man/showClusterStatus.Rd 35c75b291b06959d54fec49d652af980 *man/showLog.Rd 5ebda59809e399bc7f698d4b222d6d88 *man/showStatus.Rd 86d74ceda6264da6917359d7d70bf5de *man/sourceRegistryFiles.Rd fad7d3bb208d0b4ea80a2d26b5b8ca70 *man/submitJobs.Rd 807a937912e2a4f062008c2b9f563b2c *man/sweepRegistry.Rd 19600ac50066e47f349ddcc2b8473eea *man/syncRegistry.Rd 0b94e75875e0efa2bbd24bc7b15faeb1 *man/testJob.Rd a1f063773baab228646e75a26cffee31 *man/updateRegistry.Rd 1a54b305142ed2e18bc190d9bf4c2d6b *man/waitForJobs.Rd b325cc865a02178fc95dc61eaacd6da2 *tests/run-all.R f688e2f203eb7b24e26878de2e1f2c85 *tests/testthat/helpers.R 5c078b0719f217189be23c5c112926d5 *tests/testthat/test_Registry.R e2a51c9197bfd500a777d41921028658 *tests/testthat/test_batchExpandGrid.R 01d5879c5226a2afbcf1aa6b17d61456 *tests/testthat/test_batchMap.R c187957f7b4052bf6fac9c3177bdab93 *tests/testthat/test_batchMapQuick.R 21d86cbbc521c5961063e8f072649fa3 *tests/testthat/test_batchMapResults.R 0454d68de8d66a7c2c30e7497c7aae23 *tests/testthat/test_batchReduce.R cd1a71f7750f18268e2b31ef35010d43 *tests/testthat/test_batchReduceResults.R b313cd78a45958c572d2e92f9dcdaf29 *tests/testthat/test_database.R 86de302489b01d02437161f8ee504fe4 *tests/testthat/test_doJob.R f0b83053e618ec807f8367dcf7360b1c *tests/testthat/test_export.R e56ae4875570d3c14a647fe61a76c701 *tests/testthat/test_filterResults.R ebe1497f20792bee8ff9d4bb9c938a37 *tests/testthat/test_findJobs.R e2410eaba2fc0eb3f2a013ad23f6b423 *tests/testthat/test_findStatus.R 39e6e89d03c362f58348f60d1acd85a4 *tests/testthat/test_getJob.R e7b25395c5b2d6c204908447c53d9868 *tests/testthat/test_getJobInfo.R fe36cc94d24eefd02113595f1d53e966 *tests/testthat/test_getJobParamDf.R 4a01f997f9bee2d581c2daf17501c0a9 *tests/testthat/test_getLogFiles.R c5116748ae0445761773a0daa28bae12 *tests/testthat/test_grepLogs.R d980fd7a056ce2c075f2a6a641cac218 *tests/testthat/test_loadResults.R 1a102d5b4a729ee80e406fa4f1aa0db5 *tests/testthat/test_makeRegistry.R f0d74d9f0e61dc56861a3d8713a2ff8c *tests/testthat/test_packages.R 6cd4d52824429ccff0bd32d932fe4570 *tests/testthat/test_reduceResults.R e41a108af2685c8f12c4ea889d2850ce *tests/testthat/test_resetJobs.R 95526b2defbf0e833f5ee1e3403dbc2b *tests/testthat/test_resources.R d4abfc3fa39eaeff66843a6e442c9360 *tests/testthat/test_sanitizePath.R 3498c69bc5a65ec0603a0704d1bbae04 *tests/testthat/test_setJobFunction.R b56e0a7adca05051de44e52954576542 *tests/testthat/test_showStatus.R dbc86bd58a082e3fd0adba10d6e12594 *tests/testthat/test_sourceRegistryFiles.R f1a8703f9f8a55a5d4c538a9138f5385 *tests/testthat/test_sqlquotes.R be0baf58829e85b7aa63a044cfd786e1 *tests/testthat/test_submitJobs.R 6c54b743a389973f4d8f73246b8b816b *tests/testthat/test_sweepRegistry.R 73bca9b56fbc16b0c48691e81ce5e21c *tests/testthat/test_testJob.R 1a7c80ff18054815b6eef0225f8ec350 *tests/testthat/test_waitForJobs.R cb842420345de10a618937f551c9fa01 *tests/testthat/test_zzz_cleanup.R BatchJobs/DESCRIPTION0000644000176200001440000000210513207317714013622 0ustar liggesusersPackage: BatchJobs Title: Batch Computing with R Description: Provides Map, Reduce and Filter variants to generate jobs on batch computing systems like PBS/Torque, LSF, SLURM and Sun Grid Engine. Multicore and SSH systems are also supported. For further details see the project web page. Author: Bernd Bischl , Michel Lang , Henrik Bengtsson Maintainer: Bernd Bischl URL: https://github.com/tudo-r/BatchJobs BugReports: https://github.com/tudo-r/BatchJobs/issues MailingList: batchjobs@googlegroups.com License: BSD_2_clause + file LICENSE Depends: R (>= 3.0.0), BBmisc (>= 1.9), methods Imports: backports (>= 1.1.1), brew, checkmate (>= 1.8.0), data.table (>= 1.9.6), DBI, digest, parallel, RSQLite (>= 1.0.9011), sendmailR, stats, stringi (>= 0.4-1), utils Suggests: MASS, testthat LazyData: yes ByteCompile: yes Version: 1.7 RoxygenNote: 6.0.1 NeedsCompilation: no Packaged: 2017-11-28 15:42:44 UTC; bischl Repository: CRAN Date/Publication: 2017-11-28 17:41:32 UTC BatchJobs/man/0000755000176200001440000000000013207234653012671 5ustar liggesusersBatchJobs/man/makeClusterFunctionsLSF.Rd0000644000176200001440000000347013207234653017701 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsLSF.R \name{makeClusterFunctionsLSF} \alias{makeClusterFunctionsLSF} \title{Create cluster functions for LSF systems.} \usage{ makeClusterFunctionsLSF(template.file, list.jobs.cmd = c("bjobs", "-u $USER", "-w")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{bsub} command. Jobs are killed using the \code{bkill} command and the list of running jobs is retrieved using \code{bjobs -u $USER -w}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfLSF}. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/loadResults.Rd0000644000176200001440000000265713207234653015473 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/loadResults.R \name{loadResults} \alias{loadResults} \title{Loads result files for id vector.} \usage{ loadResults(reg, ids, part = NA_character_, simplify = FALSE, use.names = "ids", missing.ok = FALSE, impute.val = NULL) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all done jobs.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{simplify}{[\code{logical(1)}]\cr Should the result be simplified to a vector, matrix or higher dimensional array if possible? Default is \code{TRUE}.} \item{use.names}{[\code{character(1)}]\cr Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) or return a unnamed result (\dQuote{none}). Default is \code{ids}.} \item{missing.ok}{[\code{logical(1)}]\cr If \code{FALSE} an error is thrown if the results are not found. Otherwise missing results are imputed to \code{NULL}. Default is \code{FALSE}.} \item{impute.val}{[any]\cr The value to return when no result is available. Defaults to \code{NULL} (the previous behavior)} } \value{ [\code{list}]. Results of jobs as list, possibly named by ids. } \description{ Loads result files for id vector. } \seealso{ \code{\link{reduceResults}} } BatchJobs/man/addRegistrySourceDirs.Rd0000644000176200001440000000174613207234653017454 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{addRegistrySourceDirs} \alias{addRegistrySourceDirs} \title{Add source dirs to registry.} \usage{ addRegistrySourceDirs(reg, src.dirs, src.now = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{src.dirs}{[\code{character}]\cr Paths to add to registry. See \code{\link{makeRegistry}}.} \item{src.now}{[\code{logical(1)}] Source files now on master? Default is \code{TRUE}.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/updateRegistry.Rd0000644000176200001440000000064513207234653016200 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/updateRegistry.R \name{updateRegistry} \alias{updateRegistry} \title{ONLY FOR INTERNAL USAGE.} \usage{ updateRegistry(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ [any]. Updated \code{\link{Registry}} or \code{FALSE} if no updates were performed. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/dbCreateJobDefTable.Rd0000644000176200001440000000054313207234653016715 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/database.R \name{dbCreateJobDefTable} \alias{dbCreateJobDefTable} \title{ONLY FOR INTERNAL USAGE.} \usage{ dbCreateJobDefTable(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ Nothing. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/getJobs.Rd0000644000176200001440000000076713207234653014567 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getJob.R \name{getJobs} \alias{getJobs} \title{Get jobs from registry by id.} \usage{ getJobs(reg, ids, check.ids = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{check.ids}{[\code{logical(1)}]\cr Check the job ids? Default is \code{TRUE}.} } \value{ [list of \code{\link{Job}}]. } \description{ Get jobs from registry by id. } BatchJobs/man/setJobNames.Rd0000644000176200001440000000114513207234653015373 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/setJobNames.R \name{setJobNames} \alias{setJobNames} \title{Set job names.} \usage{ setJobNames(reg, ids, jobnames) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{jobnames}{[\code{character}]\cr Character vector with length equal to \code{length(ids)}. \code{NA} removes the names stored in the registry. A single \code{NA} is replicated to match the length of ids provided.} } \value{ Named vector of job ids. } \description{ Set job names. } BatchJobs/man/applyJobFunction.Rd0000644000176200001440000000103413207234653016444 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/applyJobFunction.R \name{applyJobFunction} \alias{applyJobFunction} \title{applyJobFunction ONLY FOR INTERNAL USAGE.} \usage{ applyJobFunction(reg, job, cache) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{job}{[\code{\link{Job}}]\cr Job.} \item{cache}{[\code{FileCache}]\cr Instance of \code{\link[BBmisc]{makeFileCache}}.} } \value{ [any]. Result of job. } \description{ applyJobFunction ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/installPackagesOnSSHWorkers.Rd0000644000176200001440000000234313207234653020517 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/installPackagesOnSSHWorkers.R \name{installPackagesOnSSHWorkers} \alias{installPackagesOnSSHWorkers} \title{Install packages on SSH workers.} \usage{ installPackagesOnSSHWorkers(nodenames, pkgs, repos = getOption("repos"), consecutive = TRUE, show.output = consecutive, ...) } \arguments{ \item{nodenames}{[\code{character}]\cr Nodenames of workers.} \item{pkgs}{[\code{character}]\cr See \code{\link{install.packages}}.} \item{repos}{[\code{character}]\cr See \code{\link{install.packages}}. If the user must be queried this is of course done on the master.} \item{consecutive}{[\code{logical(1)}]\cr See \code{\link{callFunctionOnSSHWorkers}}. Default is \code{TRUE}.} \item{show.output}{[\code{logical(1)}]\cr See \code{\link{callFunctionOnSSHWorkers}}. Default is \code{consecutive}.} \item{...}{[any]\cr Passed to \code{\link{install.packages}}.} } \value{ Nothing. } \description{ Installation is done via \code{\link{callFunctionOnSSHWorkers}} and \code{\link{install.packages}}. Note that as usual the function tries to install the packages into the first path of \code{.libPaths()} of each each worker. } \seealso{ \code{\link{callFunctionOnSSHWorkers}} } BatchJobs/man/makeClusterFunctionsMulticore.Rd0000644000176200001440000000527013207234653021220 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsMulticore.R \name{makeClusterFunctionsMulticore} \alias{makeClusterFunctionsMulticore} \title{Use multiple cores on local Linux machine to spawn parallel jobs.} \usage{ makeClusterFunctionsMulticore(ncpus = max(getOption("mc.cores", parallel::detectCores()) - 1L, 1L), max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) } \arguments{ \item{ncpus}{[\code{integer(1)}]\cr Number of VPUs of worker. Default is to use all cores but one, where total number of cores "available" is given by option \code{\link[base:options]{mc.cores}} and if that is not set it is inferred by \code{\link[parallel]{detectCores}}.} \item{max.jobs}{[\code{integer(1)}]\cr Maximal number of jobs that can run concurrently for the current registry. Default is \code{ncpus}.} \item{max.load}{[\code{numeric(1)}]\cr Load average (of the last 5 min) at which the worker is considered occupied, so that no job can be submitted. Default is inferred by \code{\link[parallel]{detectCores}}, cf. argument \code{ncpus}.} \item{nice}{[\code{integer(1)}]\cr Process priority to run R with set via nice. Integers between -20 and 19 are allowed. If missing, processes are not nice'd and the system default applies (usually 0).} \item{r.options}{[\code{character}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} \item{script}{[\code{character(1)}]\cr Path to helper bash script which interacts with the worker. You really should not have to touch this, as this would imply that we have screwed up and published an incompatible version for your system. This option is only provided as a last resort for very experienced hackers. Note that the path has to be absolute. This is what is done in the package: \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} Default means to take it from package directory.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Jobs are spawned by starting multiple R sessions on the commandline (similar like on true batch systems). Packages \code{parallel} or \code{multicore} are not used in any way. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/killJobs.Rd0000644000176200001440000000353713207234653014741 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/killJobs.R \name{killJobs} \alias{killJobs} \title{Kill some jobs on the batch system.} \usage{ killJobs(reg, ids, progressbar = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs to kill. Default is none.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} } \value{ [\code{integer}]. Ids of killed jobs. } \description{ Kill jobs which have already been submitted to the batch system. If a job is killed its internal state is reset as if it had not been submitted at all. The function informs if (a) the job you want to kill has not been submitted, (b) the job has already terminated, (c) for some reason no batch job id is available. In all 3 cases above, nothing is changed for the state of this job and no call to the internal kill cluster function is generated. In case of an error when killing, the function tries - after a short sleep - to kill the remaining batch jobs again. If this fails again for some jobs, the function gives up. Only jobs that could be killed are reset in the DB. } \examples{ \dontrun{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) Sys.sleep(x) batchMap(reg, f, 1:10 + 5) submitJobs(reg) waitForJobs(reg) # kill all jobs currently _running_ killJobs(reg, findRunning(reg)) # kill all jobs queued or running killJobs(reg, findNotTerminated(reg)) } } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/makeRegistry.Rd0000644000176200001440000000670613207234653015637 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Registry.R \name{makeRegistry} \alias{makeRegistry} \alias{Registry} \title{Construct a registry object.} \usage{ makeRegistry(id, file.dir, sharding = TRUE, work.dir, multiple.result.files = FALSE, seed, packages = character(0L), src.dirs = character(0L), src.files = character(0L), skip = TRUE) } \arguments{ \item{id}{[\code{character(1)}]\cr Name of registry. Displayed e.g. in mails or in cluster queue.} \item{file.dir}{[\code{character(1)}]\cr Path where files regarding the registry / jobs should be saved. Default is \dQuote{-files} in current working directory if \code{id} is set.} \item{sharding}{[\code{logical(1)}]\cr Enable sharding to distribute result files into different subdirectories? Important if you have many experiments. Default is \code{TRUE}.} \item{work.dir}{[\code{character(1)}]\cr Working directory for R process when experiment is executed. Default is the current working directory when registry is created.} \item{multiple.result.files}{[\code{logical(1)}]\cr Should a result file be generated for every list element of the returned list of the job function? Note that the function provided to \code{\link{batchMap}} or \code{\link{batchReduce}} must return a named list if this is set to \code{TRUE}. The result file will be named \dQuote{-result-.RData} instead of \dQuote{-result.RData}. Default is \code{FALSE}.} \item{seed}{[\code{integer(1)}]\cr Start seed for experiments. The first experiment in the registry will use this seed, for the subsequent ones the seed is incremented by 1. Default is a random number from 1 to \code{.Machine$integer.max/2}.} \item{packages}{[\code{character}]\cr Packages that will always be loaded on each node. Default is \code{character(0)}.} \item{src.dirs}{[\code{character}]\cr Directories containing R scripts to be sourced on registry load (both on slave and master). Files not matching the pattern \dQuote{\\.[Rr]$} are ignored. Useful if you have many helper functions that are needed during the execution of your jobs. These files should only contain function definitions and no executable code. Default is \code{character(0)}.} \item{src.files}{[\code{character}]\cr R scripts files to be sourced on registry load (both on slave and master). Useful if you have many helper functions that are needed during the execution of your jobs. These files should only contain function and constant definitions and no long running, executable code. These paths are considered to be relative to your \code{work.dir}. As a last remedy in problematic cases you can use absolute paths, by passing paths that start with \dQuote{/}, see the comment about \code{file.dir} and \code{work.dir} above, where we allow the same thing. Note that this is a less portable approach and therefore usually a less good idea. Default is \code{character(0)}.} \item{skip}{[\code{logical(1)}]\cr Skip creation of a new registry if a registry is found in \code{file.dir}. Defaults to \code{TRUE}.} } \value{ [\code{\link{Registry}}] } \description{ Note that if you don't want links in your paths (\code{file.dir}, \code{work.dir}) to get resolved and have complete control over the way the path is used internally, pass an absolute path which begins with \dQuote{/}. } \details{ Every object is a list that contains the passed arguments of the constructor. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) print(reg) } BatchJobs/man/getErrorMessages.Rd0000644000176200001440000000146513207234653016447 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getErrorMessages.R \name{getErrorMessages} \alias{getErrorMessages} \title{Get error messages of jobs.} \usage{ getErrorMessages(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs with errors.} } \value{ [\code{character}]. Error messages for jobs as character vector\cr \code{NA} if job has terminated successfully. } \description{ Get error messages of jobs. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/configuration.Rd0000644000176200001440000000066113207234653016032 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/conf.R \name{configuration} \alias{configuration} \alias{.BatchJobs.R} \title{BatchJobs configuration.} \description{ In order to understand how the package should be configured please read \url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}. } \seealso{ Other conf: \code{\link{getConfig}}, \code{\link{loadConfig}}, \code{\link{setConfig}} } BatchJobs/man/makeJob.Rd0000644000176200001440000000152213207234653014530 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Job.R \name{makeJob} \alias{makeJob} \alias{Job} \title{Creates a job description.} \usage{ makeJob(id = NA_integer_, fun, fun.id = digest(fun), pars, name, seed) } \arguments{ \item{id}{[\code{integer(1)}]\cr Job id, determined by DB autoincrement. Default is \code{NA}.} \item{fun}{[\code{function}]\cr Job function to apply on parameters.} \item{fun.id}{[\code{character(1)}]\cr Id used to store function on disk. Default is \code{digest(fun)}.} \item{pars}{[\code{list}]\cr Parameter list for job function.} \item{name}{[\code{character(1)}]\cr Alias name for this job.} \item{seed}{[\code{integer(1)}]\cr Random seed for job.} } \description{ Usually you will not do this manually. Every object is a list that contains the passed arguments of the constructor. } BatchJobs/man/makeClusterFunctionsSGE.Rd0000644000176200001440000000346113207234653017673 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsSGE.R \name{makeClusterFunctionsSGE} \alias{makeClusterFunctionsSGE} \title{Create cluster functions for Sun Grid Engine systems.} \usage{ makeClusterFunctionsSGE(template.file, list.jobs.cmd = c("qstat", "-u $USER")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{qsub} command. Jobs are killed using the \code{qdel} command and the list of running jobs is retrieved using \code{qselect}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSGE}. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/batchExport.Rd0000644000176200001440000000236613207234653015452 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Exports.R \name{batchExport} \alias{batchExport} \title{Export R object to be available on the slaves.} \usage{ batchExport(reg, ..., li = list(), overwrite = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{...}{[any]\cr Objects to export. You must provide a valid name.} \item{li}{[\code{list}]\cr More objects to export provided as a named list.} \item{overwrite}{[\code{logical(1)}]\cr If set to \code{FALSE} (default), exported objects are protected from being overwritten by multiple calls of this function. Setting this to \code{TRUE} disables this check.} } \value{ [\code{character}]. Invisibly returns a character vector of exported objects. } \description{ Saves objects as \code{RData} files in the \dQuote{exports} subdirectory of your \code{file.dir} to be later loaded on the slaves. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/makeClusterFunctionsSLURM.Rd0000644000176200001440000000351413207234653020156 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsSLURM.R \name{makeClusterFunctionsSLURM} \alias{makeClusterFunctionsSLURM} \title{Create cluster functions for SLURM-based systems.} \usage{ makeClusterFunctionsSLURM(template.file, list.jobs.cmd = c("squeue", "-h", "-o \%i", "-u $USER")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{sbatch} command. Jobs are killed using the \code{scancel} command and the list of running jobs is retrieved using \code{squeue}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfSLURM}. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/batchMapResults.Rd0000644000176200001440000000406313207234653016264 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchMapResults.R \name{batchMapResults} \alias{batchMapResults} \title{Maps a function over the results of a registry by using batchMap.} \usage{ batchMapResults(reg, reg2, fun, ..., ids, part = NA_character_, more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry whose results should be mapped by \code{fun}.} \item{reg2}{[\code{\link{Registry}}]\cr Empty registry that should store the job for the mapping.} \item{fun}{[\code{function(job, res, ...)}]\cr Function to map over results of \code{reg}. Further arguments come from ... of \code{batchMapResults} and \code{more.args}.} \item{...}{[any]\cr Furher arguments to vectorize over (list or vector). Must all be the same length as number of results in \code{reg}.} \item{ids}{[\code{integer}]\cr Ids of jobs whose results should be mapped with \code{fun}. Default is all jobs.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ Vector of type \code{integer} with job ids. } \description{ Maps a function over the results of a registry by using batchMap. } \examples{ reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) # square some numbers f = function(x) x^2 batchMap(reg1, f, 1:10) # submit jobs and wait for the jobs to finish submitJobs(reg1) waitForJobs(reg1) # look at results reduceResults(reg1, fun = function(aggr,job,res) c(aggr, res)) reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) # define function to tranform results, we simply do the inverse of the squaring g = function(job, res) sqrt(res) batchMapResults(reg1, reg2, fun = g) # submit jobs and wait for the jobs to finish submitJobs(reg2) waitForJobs(reg2) # check results reduceResults(reg2, fun = function(aggr,job,res) c(aggr, res)) } BatchJobs/man/cfHandleUnknownSubmitError.Rd0000644000176200001440000000177213207234653020451 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfHandleUnknownSubmitError} \alias{cfHandleUnknownSubmitError} \title{Cluster functions helper: Handle an unknown error during job submission.} \usage{ cfHandleUnknownSubmitError(cmd, exit.code, output) } \arguments{ \item{cmd}{[\code{character(1)}]\cr OS command used to submit the job, e.g. qsub.} \item{exit.code}{[\code{integer(1)}]\cr Exit code of the OS command, should not be 0.} \item{output}{[\code{character}]\cr Output of the OS command, hopefully an informative error message. If these are mutiple lines in a vector, they are automatically pasted together.} } \value{ [\code{\link{SubmitJobResult}}]. } \description{ This function is only intended for use in your own cluster functions implementation. Simply constructs a \code{\link{SubmitJobResult}} object with status code 101, NA as batch job id and an informative error message containing the output of the OS command in \code{output}. } BatchJobs/man/loadResult.Rd0000644000176200001440000000171013207234653015275 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/loadResult.R \name{loadResult} \alias{loadResult} \title{Loads a specific result file.} \usage{ loadResult(reg, id, part = NA_character_, missing.ok = FALSE, impute.val = NULL) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of job.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{missing.ok}{[\code{logical(1)}]\cr If \code{FALSE} an error is thrown if no result file is found. Otherwise \code{NULL} is returned. Default is \code{FALSE}.} \item{impute.val}{[any]\cr The value to return when no result is available. Defaults to \code{NULL} (the previous behavior)} } \value{ [any]. Result of job. } \description{ Loads a specific result file. } \seealso{ \code{\link{reduceResults}} } BatchJobs/man/batchMap.Rd0000644000176200001440000000203613207234653014700 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchMap.R \name{batchMap} \alias{batchMap} \title{Maps a function over lists or vectors, adding jobs to a registry.} \usage{ batchMap(reg, fun, ..., more.args = list(), use.names = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Empty Registry that will store jobs for the mapping.} \item{fun}{[\code{function}]\cr Function to map over \code{...}.} \item{...}{[any]\cr Arguments to vectorize over (list or vector).} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} \item{use.names}{[\code{logical(1)}]\cr Store parameter names to enable named results in \code{\link{loadResults}} and some other functions. Default is \code{FALSE}.} } \value{ Vector of type \code{integer} with job ids. } \description{ You can then submit these jobs to the batch system. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) print(reg) } BatchJobs/man/showClusterStatus.Rd0000644000176200001440000000131413207234653016705 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/showClusterStatus.R \name{showClusterStatus} \alias{showClusterStatus} \title{Show information about available computational resources on cluster.} \usage{ showClusterStatus(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry. Must not be passed and this is the default.} } \value{ [\code{data.frame}]. } \description{ Currently only supported for multicore and SSH mode. Displays: Name of node, current load, number of running R processes, number of R processes with more than 50% load, number of BatchJobs jobs running. The latter counts either jobs belonging to \code{reg} or all BatchJobs jobs if reg was not passed. } BatchJobs/man/sanitizePath.Rd0000644000176200001440000000127513207234653015630 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sanitizePath.R \name{sanitizePath} \alias{sanitizePath} \title{Sanitize a path} \usage{ sanitizePath(path, make.absolute = TRUE, normalize.absolute = FALSE) } \arguments{ \item{path}{[\code{character}]\cr Vector of paths to sanitize.} \item{make.absolute}{[\code{logical}]\cr If \code{TRUE} convert to an absolute path.} \item{normalize.absolute}{[\code{logical}]\cr Also call \code{\link[base]{normalizePath}} on absolute paths? This will immediately resolve symlinks.} } \value{ \code{character} with sanitized paths. } \description{ Replaces backward slashes with forward slashes and optionally normalizes the path. } BatchJobs/man/setConfig.Rd0000644000176200001440000000130313207234653015076 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/conf.R \name{setConfig} \alias{setConfig} \title{Set and overwrite configuration settings} \usage{ setConfig(conf = list(), ...) } \arguments{ \item{conf}{[\code{Config} or \code{list}]\cr List of configuration parameters as returned by \code{\link{loadConfig}} or \code{\link{getConfig}}.} \item{...}{[\code{ANY}]\cr Named configuration parameters. Overwrites parameters in \code{conf}, if provided.} } \value{ Invisibly returns a list of configuration settings. } \description{ Set and overwrite configuration settings } \seealso{ Other conf: \code{\link{configuration}}, \code{\link{getConfig}}, \code{\link{loadConfig}} } BatchJobs/man/waitForJobs.Rd0000644000176200001440000000332213207234653015411 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/waitForJobs.R \name{waitForJobs} \alias{waitForJobs} \title{Wait for termination of jobs on the batch system.} \usage{ waitForJobs(reg, ids, sleep = 10, timeout = 604800, stop.on.error = FALSE, progressbar = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Vector of job ids. Default is all submitted jobs not yet terminated.} \item{sleep}{[\code{numeric(1)}]\cr Seconds to sleep between status updates. Default is \code{10}.} \item{timeout}{[\code{numeric(1)}]\cr After waiting \code{timeout} seconds, show a message and return \code{FALSE}. This argument may be required on some systems where, e.g., expired jobs or jobs on hold are problematic to detect. If you don't want a timeout, set this to \code{Inf}. Default is \code{604800} (one week).} \item{stop.on.error}{[\code{logical(1)}]\cr Immediately return if a job terminates with an error? Default is \code{FALSE}.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} } \value{ [\code{logical(1)}]. Returns \code{TRUE} if all jobs terminated successfully and \code{FALSE} if either an error occurred or the timeout is reached. } \description{ Waits for termination of jobs while displaying a progress bar containing summarizing informations of the jobs. The following abbreviations are used in the progress bar: \dQuote{S} for number of jobs on system, \dQuote{D} for number of jobs successfully terminated, \dQuote{E} for number ofjobs terminated with an R exception and \dQuote{R} for number of jobs currently running on the system. } BatchJobs/man/addRegistrySourceFiles.Rd0000644000176200001440000000175413207234653017614 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{addRegistrySourceFiles} \alias{addRegistrySourceFiles} \title{Add source files to registry.} \usage{ addRegistrySourceFiles(reg, src.files, src.now = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{src.files}{[\code{character}]\cr Paths to add to registry. See \code{\link{makeRegistry}}.} \item{src.now}{[\code{logical(1)}] Source files now on master? Default is \code{TRUE}.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.files} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/copyRequiredJobFiles.Rd0000644000176200001440000000074413207234653017256 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/testJob.R \name{copyRequiredJobFiles} \alias{copyRequiredJobFiles} \title{ONLY FOR INTERNAL USAGE.} \usage{ copyRequiredJobFiles(reg1, reg2, id) } \arguments{ \item{reg1}{[\code{\link{Registry}}]\cr Source registry.} \item{reg2}{[\code{\link{Registry}}]\cr Detination registry.} \item{id}{[\code{character(1)}]\cr Job id.} } \value{ Nothing. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/makeSubmitJobResult.Rd0000644000176200001440000000226613207234653017121 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctions.R \name{makeSubmitJobResult} \alias{makeSubmitJobResult} \alias{SubmitJobResult} \title{Create a SubmitJobResult object.} \usage{ makeSubmitJobResult(status, batch.job.id, msg, ...) } \arguments{ \item{status}{[\code{integer(1)}]\cr Launch status of job. 0 means success, codes bewteen 1 and 100 are temporary errors and any error greater than 100 is a permanent failure.} \item{batch.job.id}{[\code{character(1)}]\cr Unique id of this job on batch system. Note that this is not the usual job id used in BatchJobs! Must be globally unique so that the job can be terminated using just this information.} \item{msg}{[\code{character(1)}]\cr Optional error message in case \code{status} is not equal to 0. Default is \dQuote{OK}, \dQuote{TEMPERR}, \dQuote{ERROR}, depending on \code{status}.} \item{...}{[\code{any}]\cr Currently unused.} } \value{ [\code{\link{SubmitJobResult}}]. A list, containing \code{status}, \code{batch.job.id} and \code{msg}. } \description{ Use this function in your implementation of \code{\link{makeClusterFunctions}} to create a return value for the \code{submitJob} function. } BatchJobs/man/showStatus.Rd0000644000176200001440000000251513207234653015347 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/showStatus.R \name{showStatus} \alias{showStatus} \alias{getStatus} \title{Retrieve or show status information about jobs.} \usage{ showStatus(reg, ids, run.and.exp = TRUE, errors = 10L) getStatus(reg, ids, run.and.exp = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of selected jobs. Default is all jobs.} \item{run.and.exp}{[\code{logical(1)}]\cr Show running and expired jobs? Requires to list the job on the batch system. If not possible, because that cluster function is not avilable, this option is ignored anyway. Default is \code{TRUE}.} \item{errors}{[\code{integer(1)}]\cr How many of the error messages should be displayed if any errors occured in the jobs? Default is 10.} } \value{ [\code{list}]. List of absolute job numbers. \code{showStatus} returns them invisibly. } \description{ E.g.: How many there are, how many are done, any errors, etc. \code{showStatus} displays on the console, \code{getStatus} returns an informative result without console output. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) submitJobs(reg) waitForJobs(reg) # should show 10 submitted jobs, which are all done. showStatus(reg) } BatchJobs/man/batchMapQuick.Rd0000644000176200001440000000403013207234653015671 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchMapQuick.R \name{batchMapQuick} \alias{batchMapQuick} \title{Combination of makeRegistry, batchMap and submitJobs.} \usage{ batchMapQuick(fun, ..., more.args = list(), file.dir = NULL, packages = character(0L), chunk.size, n.chunks, chunks.as.arrayjobs = FALSE, inds, resources = list()) } \arguments{ \item{fun}{[\code{function}]\cr Function to map over \code{...}.} \item{...}{[any]\cr Arguments to vectorize over (list or vector).} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} \item{file.dir}{[\code{character}]\cr See \code{\link{makeRegistry}}. Default is \code{NULL}, which means that it is created in the current directory under the name \dQuote{bmq_[random alphanumerics]}.} \item{packages}{[\code{character}]\cr See \code{\link{makeRegistry}}.} \item{chunk.size}{[\code{integer(1)}]\cr Preferred number of jobs in each chunk. Can not be used in combination with \code{n.chunks}. Note that the ids will get shuffled to balance out possible run time differences. Default is not to use chunking.} \item{n.chunks}{[\code{integer(1)}]\cr Preferred number chunks. Can not be used in combination with \code{chunk.size}. Note that the ids will get shuffled to balance out possible run time differences. Default is not to use chunking.} \item{chunks.as.arrayjobs}{[\code{logical(1)}]\cr Submit chunks as array jobs? Default is \code{FALSE}.} \item{inds}{[\code{integer}]\cr Indices of ids / chunks to submit. Default is all. If ids get chunked, this subsets the list of shuffled ids.} \item{resources}{[\code{list}]\cr Required resources for all batch jobs. Default is empty list.} } \value{ [\code{\link{Registry}}] } \description{ Combination of \code{\link{makeRegistry}}, \code{\link{batchMap}} and \code{\link{submitJobs}} for quick computations on the cluster. Should only be used by skilled users who know what they are doing. Creates the file.dir, maps function, potentially chunks jobs and submits them. } BatchJobs/man/loadRegistry.Rd0000644000176200001440000000220513207234653015627 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/loadRegistry.R \name{loadRegistry} \alias{loadRegistry} \title{Load a previously saved registry.} \usage{ loadRegistry(file.dir, work.dir, adjust.paths = FALSE) } \arguments{ \item{file.dir}{[\code{character(1)}]\cr Location of the file.dir to load the registry from.} \item{work.dir}{[\code{character(1)}]\cr Location of the work. Unchanged if missing.} \item{adjust.paths}{[\code{logical(1)}]\cr If set to \code{FALSE} (default), the paths for the \code{file.dir} and \code{work.dir} will not be updated. Set to \code{TRUE} if you moved the directoy to another system \emph{after} all computations finished.} } \value{ [\code{\link{Registry}}]. } \description{ Loads a previously created registry from the file system. The \code{file.dir} is automatically updated upon load if \code{adjust.paths} is set to \code{TRUE}, so be careful if you use the registry on multiple machines simultaneously, e.g. via sshfs or a samba share. There is a heuristic included which tries to detect if the location of the registry has changed and returns a read-only registry if necessary. } BatchJobs/man/batchQuery.Rd0000644000176200001440000000150613207234653015271 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchQuery.R \name{batchQuery} \alias{batchQuery} \title{Manually query the BatchJobs database} \usage{ batchQuery(reg, query, flags = "ro") } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{query}{[\code{character(1)}]\cr SQL query to send to the database.} \item{flags}{[\code{character(1)}]\cr One of \dQuote{ro}, \dQuote{rw} or \dQuote{rwc} which is translated to \code{SQLITE_RO}, \code{SQLITE_RW} or \code{SQLITE_RWC}, respectively. See \link[RSQLite]{SQLITE_RO} for more info.} } \value{ [\code{data.frame}] Result of the query. } \description{ Manually query the BatchJobs database } \examples{ reg = makeRegistry("test", file.dir = tempfile()) batchMap(reg, identity, i = 1:10) batchQuery(reg, "SELECT * FROM test_job_status") } BatchJobs/man/reduceResults.Rd0000644000176200001440000001212713207234653016014 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/reduceResults.R \name{reduceResults} \alias{reduceResults} \alias{reduceResultsList} \alias{reduceResultsVector} \alias{reduceResultsMatrix} \alias{reduceResultsDataFrame} \alias{reduceResultsDataTable} \title{Reduce results from result directory.} \usage{ reduceResults(reg, ids, part = NA_character_, fun, init, impute.val, progressbar = TRUE, ...) reduceResultsList(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, progressbar = TRUE) reduceResultsVector(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val) reduceResultsMatrix(reg, ids, part = NA_character_, fun, ..., rows = TRUE, use.names = "ids", impute.val) reduceResultsDataFrame(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val, strings.as.factors = default.stringsAsFactors()) reduceResultsDataTable(reg, ids, part = NA_character_, fun, ..., use.names = "ids", impute.val) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of selected jobs. Default is all jobs for which results are available.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{fun}{[\code{function}]\cr For \code{reduceResults}, a function \code{function(aggr, job, res, ...)} to reduce things, for all others, a function \code{function(job, res, ...)} to select stuff. Here, \code{job} is the current job descriptor (see \code{\link{Job}}), \code{result} is the current result object and \code{aggr} are the so far aggregated results. When using \code{reduceResults}, your function should add the stuff you want to have from \code{job} and \code{result} to \code{aggr} and return that. When using the other reductions, you should select the stuff you want to have from \code{job} and \code{result} and return something that can be coerced to an element of the selected return data structure (reasonable conversion is tried internally). Default behavior for this argument is to return \code{res}, except for \code{reduceResults} where no default is available.} \item{init}{[\code{ANY}]\cr Initial element, as used in \code{\link{Reduce}}. Default is first result.} \item{impute.val}{[any]\cr For \code{reduceResults}: If not missing, the value of \code{impute.val} is passed to function \code{fun} as argument \code{res} for jobs with missing results.\cr For the specialized reduction functions \code{reduceResults[Type]}: If not missing, \code{impute.val} is used as a replacement for the return value of \code{fun} on missing results.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} \item{...}{[any]\cr Additional arguments to \code{fun}.} \item{use.names}{[\code{character(1)}]\cr Name the results with job ids (\dQuote{ids}), stored job names (\dQuote{names}) or return a unnamed result (\dQuote{none}). Default is \code{ids}.} \item{rows}{[\code{logical(1)}]\cr Should the selected vectors be used as rows (or columns) in the result matrix? Default is \code{TRUE}.} \item{strings.as.factors}{[\code{logical(1)}] Should all character columns in result be converted to factors? Default is \code{default.stringsAsFactors()}.} } \value{ Aggregated results, return type depends on function. If \code{ids} is empty: \code{reduceResults} returns \code{init} (if available) or \code{NULL}, \code{reduceResultsVector} returns \code{c()}, \code{reduceResultsList} returns \code{list()}, \code{reduceResultsMatrix} returns \code{matrix(0,0,0)}, \code{reduceResultsDataFrame} returns \code{data.frame()}. } \description{ The following functions provide ways to reduce result files into either specific R objects (like vectors, lists, matrices or data.frames) or to arbitrarily aggregate them, which is a more general operation. } \examples{ # generate results: reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:5) submitJobs(reg) waitForJobs(reg) # reduce results to a vector reduceResultsVector(reg) # reduce results to sum reduceResults(reg, fun = function(aggr, job, res) aggr+res) reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) list(a = x, b = as.character(2*x), c = x^2) batchMap(reg, f, 1:5) submitJobs(reg) waitForJobs(reg) # reduce results to a vector reduceResultsVector(reg, fun = function(job, res) res$a) reduceResultsVector(reg, fun = function(job, res) res$b) # reduce results to a list reduceResultsList(reg) # reduce results to a matrix reduceResultsMatrix(reg, fun = function(job, res) res[c(1,3)]) reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = TRUE) reduceResultsMatrix(reg, fun = function(job, res) c(foo = res$a, bar = res$c), rows = FALSE) # reduce results to a data.frame print(str(reduceResultsDataFrame(reg))) # reduce results to a sum reduceResults(reg, fun = function(aggr, job, res) aggr+res$a, init = 0) } BatchJobs/man/batchReduceResults.Rd0000644000176200001440000000405013207234653016752 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchReduceResults.R \name{batchReduceResults} \alias{batchReduceResults} \title{Reduces results via a binary function and adds jobs for this to a registry.} \usage{ batchReduceResults(reg, reg2, fun, ids, part = NA_character_, init, block.size, more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry whose results should be reduced by \code{fun}.} \item{reg2}{[\code{\link{Registry}}]\cr Empty registry that should store the job for the mapping.} \item{fun}{[\code{function(aggr, job, res, ...)}]\cr Function to reduce results with.} \item{ids}{[\code{integer}]\cr Ids of jobs whose results should be reduced with \code{fun}. Default is all jobs.} \item{part}{[\code{character}] Only useful for multiple result files, then defines which result file part(s) should be loaded. \code{NA} means all parts are loaded, which is the default.} \item{init}{[any]\cr Initial object for reducing.} \item{block.size}{[\code{integer(1)}]\cr Number of results reduced in one job.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ Vector of type \code{integer} with job ids. } \description{ Each jobs reduces a certain number of results on one slave. You can then submit these jobs to the batch system. Later, you can do a final reduction with \code{\link{reduceResults}} on the master. } \examples{ # generating example results: reg1 = makeRegistry(id = "BatchJobsExample1", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg1, f, 1:20) submitJobs(reg1) waitForJobs(reg1) # define function to reduce on slave, we want to sum the squares myreduce = function(aggr, job, res) aggr + res # sum 5 results on each slave process, i.e. 4 jobs reg2 = makeRegistry(id = "BatchJobsExample2", file.dir = tempfile(), seed = 123) batchReduceResults(reg1, reg2, fun = myreduce, init = 0, block.size = 5) submitJobs(reg2) waitForJobs(reg2) # now reduce one final time on master reduceResults(reg2, fun = myreduce) } BatchJobs/man/filterResults.Rd0000644000176200001440000000201313207234653016023 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/filterResults.R \name{filterResults} \alias{filterResults} \title{Find all results where a specific condition is true.} \usage{ filterResults(reg, ids, fun, ...) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs whose results you want to test for the condition. Default is all jobs for which results are available.} \item{fun}{[\code{fun(job, res)}]\cr Predicate function that returns \code{TRUE} or \code{FALSE}.} \item{...}{[any]\cr Additional arguments to \code{fun}.} } \value{ [\code{integer}]. Ids of jobs where \code{fun(job, result)} returns \code{TRUE}. } \description{ Find all results where a specific condition is true. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) submitJobs(reg) waitForJobs(reg) # which square numbers are even: filterResults(reg, fun = function(job, res) res \%\% 2 == 0) } BatchJobs/man/loadConfig.Rd0000644000176200001440000000113213207234653015222 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/conf.R \name{loadConfig} \alias{loadConfig} \title{Load a specific configuration file.} \usage{ loadConfig(conffile = ".BatchJobs.R") } \arguments{ \item{conffile}{[\code{character(1)}]\cr Location of the configuration file to load. Default is \dQuote{.BatchJobs.conf} in the current working directory.} } \value{ Invisibly returns a list of configuration settings. } \description{ Load a specific configuration file. } \seealso{ Other conf: \code{\link{configuration}}, \code{\link{getConfig}}, \code{\link{setConfig}} } BatchJobs/man/getJobInfo.Rd0000644000176200001440000000404513207234653015211 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getJobInfo.R \name{getJobInfo} \alias{getJobInfo} \title{Get computational information of jobs.} \usage{ getJobInfo(reg, ids, pars = FALSE, prefix.pars = FALSE, select, unit = "seconds") } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{pars}{[\code{logical(1)}]\cr Include job parameters in the output? Default is \code{FALSE}.} \item{prefix.pars}{[\code{logical(1)}]\cr Should a prefix be added to job parameter names (column names) to avoid name clashes? Default is \code{FALSE}.} \item{select}{[\code{character}]\cr Select only a subset of columns. Usually this is not required and you can subset yourself, but in some rare cases it may be advantageous to not query all information. Note that the column \dQuote{id} (job id) is always selected. If not provided, all columns are queried and returned.} \item{unit}{[\code{character(1)}]\cr Unit to convert execution and queing times to. Possible values: \dQuote{seconds}, \dQuote{minutes}, \dQuote{hours}, \dQuote{days} and \dQuote{weeks}. Default is \dQuote{seconds}.} } \value{ [\code{data.frame}]. } \description{ Returns time stamps (submitted, started, done, error), time running, approximate memory usage (in Mb), error messages (shortened, see \code{\link{showLog}} for detailed error messages), time in queue, hostname of the host the job was executed, assigned batch ID, the R PID and the seed of the job. To estimate memory usage the sum of the last column of \code{\link[base]{gc}} is used. Column \dQuote{time.running} displays the time until either the job was done, or an error occured; it will by \code{NA} in case of time outs or hard R crashes. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/setRegistryPackages.Rd0000644000176200001440000000153713207234653017151 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Packages.R \name{setRegistryPackages} \alias{setRegistryPackages} \title{Set packages for a registry.} \usage{ setRegistryPackages(reg, packages) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{packages}{[\code{character}]\cr Character vector of package names to load.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{packages} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}} } BatchJobs/man/grepLogs.Rd0000644000176200001440000000252613207234653014747 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/grepLogs.R \name{grepLogs} \alias{grepLogs} \title{Grep log files for a pattern.} \usage{ grepLogs(reg, ids, pattern = "warn", ignore.case = TRUE, verbose = FALSE, range = 2L) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs to grep. Default is all terminated jobs (done + errors).} \item{pattern}{[\code{character(1)}]\cr Pattern to search for. See \code{\link{grep}}. Default is \code{"warn"}.} \item{ignore.case}{[\code{logical(1)}]\cr Ignore case. See \code{\link{grep}}. Default is \code{TRUE}.} \item{verbose}{[\code{logical(1)}]\cr Print matches. Default is \code{FALSE}.} \item{range}{[\code{integer(1)}]\cr If \code{verbose} is set to \code{TRUE}, print \code{range} leading and trailing lines for contextual information about the warning. Default is \code{2}.} } \value{ [\code{integer}]. Ids of jobs where pattern was found in the log file. } \description{ Searches for occurence of \code{pattern} in log files. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/makeClusterFunctions.Rd0000644000176200001440000000640313207234653017333 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctions.R \name{makeClusterFunctions} \alias{makeClusterFunctions} \alias{ClusterFunctions} \title{Create a ClusterFuntions object.} \usage{ makeClusterFunctions(name, submitJob, killJob, listJobs, getArrayEnvirName, class = NULL, ...) } \arguments{ \item{name}{[\code{character(1)}]\cr Name of cluster functions.} \item{submitJob}{[\code{function(conf, reg, job.name, rscript, log.file, job.dir, resources, ...)}]\cr Function to submit a new job. Must return a \code{\link{SubmitJobResult}} object.\cr The arguments are:\cr conf [\code{environment}]: The user configuration.\cr reg [\code{\link{Registry}}]: The registry.\cr job.name [\code{character(1)}]: Name of job, used if the job is displayed on the batch system. This is just for display and not an id!\cr rscript [\code{character(1)}]: File path to R script that is used to execute the job.\cr log.file [\code{character(1)}]: File path where log file (.Rout) has to be placed.\cr job.dir [\code{character(1)}]: Directory where all files relating to this job are placed.\cr resources [\code{list}]: Freely definable list of required resources for this job, e.g. walltime or memory.} \item{killJob}{[\code{function(conf, reg, batch.job.id)}]\cr Function to kill a job on the batch system. Make sure that you definately kill the job! Return value is currently ignored.\cr The arguments are:\cr conf [\code{environment}]: The user configuration.\cr reg [\code{\link{Registry}}]: The registry.\cr batch.job.id [\code{character(1)}]: Batch job id, as produced by \code{submitJob}.\cr Set \code{killJob} to \code{NULL} if killing jobs cannot be supported.} \item{listJobs}{[\code{function(conf, reg)}]\cr List all jobs on the batch system for the current user / registry. This includes queued, running, held, idle, etc. jobs. Must return an integer vector of batch job ids, same format as they are produced by \code{submitJob}. It does not matter if you return a few job ids too many (e.g. all for the current user instead of all for the current registry), but you have to include all relevant ones. The arguments are:\cr conf [\code{environment}]: The user configuration.\cr reg [\code{\link{Registry}}]: The registry.\cr Set \code{listJobs} to \code{NULL} if listing jobs cannot be supported.} \item{getArrayEnvirName}{[\code{function()}]\cr Returns the name of the environment variable specifying the array ID. Should return \code{NA} if not supported.} \item{class}{[\code{character(1)}]\cr Optional class name for cluster functions object. Useful to provide a nice print method which might show additional information about the workers. Default is \code{NULL}.} \item{...}{[\code{any}]\cr Currently ignored.} } \description{ Use this funtion when you implement a backend for a batch system. You must define the functions specified in the arguments. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}} } BatchJobs/man/submitJobs.Rd0000644000176200001440000000706513207234653015311 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/submitJobs.R \name{submitJobs} \alias{submitJobs} \title{Submit jobs or chunks of jobs to batch system via cluster function.} \usage{ submitJobs(reg, ids, resources = list(), wait, max.retries = 10L, chunks.as.arrayjobs = FALSE, job.delay = FALSE, progressbar = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Vector for job id or list of vectors of chunked job ids. Only corresponding jobs are submitted. Chunked jobs will get executed sequentially as a single job for the scheduler. Default is all jobs which were not yet submitted to the batch system.} \item{resources}{[\code{list}]\cr Required resources for all batch jobs. The elements of this list (e.g. something like \dQuote{walltime} or \dQuote{nodes} are defined by your template job file. Defaults can be specified in your config file. Default is empty list.} \item{wait}{[\code{function(retries)}]\cr Function that defines how many seconds should be waited in case of a temporary error. Default is exponential back-off with \code{10*2^retries}.} \item{max.retries}{[\code{integer(1)}]\cr Number of times to submit one job again in case of a temporary error (like filled queues). Each time \code{wait} is called to wait a certain number of seconds. Default is 10 times.} \item{chunks.as.arrayjobs}{[\code{logical(1)}]\cr If ids are passed as a list of chunked job ids, execute jobs in a chunk as array jobs. Note that your scheduler and your template must be adjusted to use this option. Default is \code{FALSE}.} \item{job.delay}{[\code{function(n, i)} or \code{logical(1)}]\cr Function that defines how many seconds a job should be delayed before it starts. This is an expert option and only necessary to change when you want submit extremely many jobs. We then delay the jobs a bit to write the submit messages as early as possible to avoid writer starvation. \code{n} is the number of jobs and \code{i} the number of the ith job. The default function used with \code{job.delay} set to \code{TRUE} is no delay for 100 jobs or less and otherwise \code{runif(1, 0.1*n, 0.2*n)}. If set to \code{FALSE} (the default) delaying jobs is disabled.} \item{progressbar}{[\code{logical(1)}]\cr Set to \code{FALSE} to disable the progress bar. To disable all progress bars, see \code{\link[BBmisc]{makeProgressBar}}.} } \value{ [\code{integer}]. Vector of submitted job ids. } \description{ If the internal submit cluster function completes successfully, the \code{retries} counter is set back to 0 and the next job or chunk is submitted. If the internal submit cluster function returns a fatal error, the submit process is completely stopped and an exception is thrown. If the internal submit cluster function returns a temporary error, the submit process waits for a certain time, which is determined by calling the user-defined \code{wait}-function with the current \code{retries} counter, the counter is increased by 1 and the same job is submitted again. If \code{max.retries} is reached the function simply terminates. Potential temporary submit warnings and errors are logged inside your file directory in the file \dQuote{submit.log}. To keep track you can use \code{tail -f [file.dir]/submit.log} in another terminal. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) x^2 batchMap(reg, f, 1:10) submitJobs(reg) waitForJobs(reg) # Submit the 10 jobs again, now randomized into 2 chunks: chunked = chunk(getJobIds(reg), n.chunks = 2, shuffle = TRUE) submitJobs(reg, chunked) } BatchJobs/man/makeClusterFunctionsInteractive.Rd0000644000176200001440000000267413207234653021537 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsInteractive.R \name{makeClusterFunctionsInteractive} \alias{makeClusterFunctionsInteractive} \title{Create cluster functions for sequential execution in same session.} \usage{ makeClusterFunctionsInteractive(write.logs = TRUE) } \arguments{ \item{write.logs}{[\code{logical(1)}]\cr Sink the output to log files. Turning logging off can increase the speed of calculations but makes it next to impossible to debug. Default is \code{TRUE}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ All jobs executed under these cluster functions are executed sequentially, in the same interactive R process that you currently are. That is, \code{submitJob} does not return until the job has finished. The main use of this \code{ClusterFunctions} implementation is to test and debug programs on a local computer. Listing jobs returns an empty vector (as no jobs can be running when you call this) and \code{killJob} returns at once (for the same reason). } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/findState.Rd0000644000176200001440000000604213207234653015103 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/findStatus.R \name{findDone} \alias{findDone} \alias{findNotDone} \alias{findMissingResults} \alias{findErrors} \alias{findNotErrors} \alias{findTerminated} \alias{findNotTerminated} \alias{findSubmitted} \alias{findNotSubmitted} \alias{findOnSystem} \alias{findNotOnSystem} \alias{findRunning} \alias{findNotRunning} \alias{findStarted} \alias{findNotStarted} \alias{findExpired} \alias{findDisappeared} \title{Find jobs depending on computional state.} \usage{ findDone(reg, ids, limit = NULL) findNotDone(reg, ids, limit = NULL) findMissingResults(reg, ids, limit = NULL) findErrors(reg, ids, limit = NULL) findNotErrors(reg, ids, limit = NULL) findTerminated(reg, ids, limit = NULL) findNotTerminated(reg, ids, limit = NULL) findSubmitted(reg, ids, limit = NULL) findNotSubmitted(reg, ids, limit = NULL) findOnSystem(reg, ids, limit = NULL) findNotOnSystem(reg, ids, limit = NULL) findRunning(reg, ids, limit = NULL) findNotRunning(reg, ids, limit = NULL) findStarted(reg, ids, limit = NULL) findNotStarted(reg, ids, limit = NULL) findExpired(reg, ids, limit = NULL) findDisappeared(reg, ids, limit = NULL) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Subset of job ids to restrict the result to. Default is all jobs.} \item{limit}{[\code{integer(1)}]\cr Limit the number of returned ids. Default is all ids.} } \value{ [\code{integer}]. Ids of jobs. } \description{ \code{findDone}: Find jobs which succesfully terminated. \code{findNotDone}: Find jobs for which results are still missing. \code{findMissingResults}: Deprecated. Alias for findNotDone. \code{findErrors}: Find jobs where errors occured. \code{findNotErrors}: Find jobs where no errors occured. \code{findTerminated}: Find jobs which have terminated (done / error). \code{findNotTerminated}: Find jobs which have not terminated (not done / no error). \code{findSubmitted}: Find jobs which have been submitted. \code{findNotSubmitted}: Find jobs which have not been submitted. \code{findOnSystem}: Find jobs which are present on the batch system at the moment. \code{findNotOnSystem}: Find jobs which are not present on the batch system at the moment. \code{findRunning}: Find jobs which are running. \code{findNotRunning}: Find jobs which are not running. \code{findStarted}: Find jobs which have been started on the batch system. \code{findStarted}: Find jobs which have not been started on the batch system. \code{findNotRunning}: Find jobs which are not running. \code{findExpired}: Find jobs where walltime was probably hit. Right now the heuristic is as follows: Find all jobs that have started, did not abort with an error, did not complete with a result and are not submitted or running anymore. Note that this heuristic does not include jobs the scheduler looses before starting. \code{findDisappeared}: Find jobs which disappeared from the system. Right now the heuristic is as follows: Find all jobs that are submitted but not started nor on the system anymore. } BatchJobs/man/cfKillBatchJob.Rd0000644000176200001440000000171413207234653015764 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfKillBatchJob} \alias{cfKillBatchJob} \title{Cluster functions helper: Kill a batch job via OS command} \usage{ cfKillBatchJob(cmd, batch.job.id, max.tries = 3L) } \arguments{ \item{cmd}{[\code{character(1)}]\cr OS command, e.g. \dQuote{qdel}.} \item{batch.job.id}{[\code{character(1)}]\cr Id of the batch job on the batch system.} \item{max.tries}{[\code{integer(1)}]\cr Number of total times to try execute the OS command in cases of failures. Default is \code{3}.} } \value{ Nothing. } \description{ This function is only intended for use in your own cluster functions implementation. Calls the OS command to kill a job via \code{system} like this: \dQuote{cmd batch.job.id}. If the command returns an exit code > 0, the command is repeated after a 1 second sleep \code{max.tries-1} times. If the command failed in all tries, an exception is generated. } BatchJobs/man/findJobs.Rd0000644000176200001440000000205013207234653014713 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/findJobs.R \name{findJobs} \alias{findJobs} \title{Finds ids of jobs that match a query.} \usage{ findJobs(reg, ids, pars, jobnames) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Subset of job ids to restrict the result to. Default is all jobs.} \item{pars}{[R expression]\cr All jobs whose parameters match the given expression are selected. This implies that you have named the parameters when you passed the vectors. If you forgot to do this you can use \code{.arg1}, \code{.arg2}, etc., to refer to the the unnamed ones.} \item{jobnames}{[\code{character}]\cr Restrict to jobs with stored names. Exact matching is used.} } \value{ [\code{integer}]. Ids for jobs which match the query. } \description{ Finds ids of jobs that match a query. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x, y) x * y batchExpandGrid(reg, f, x = 1:2, y = 1:3) findJobs(reg, pars = (y > 2)) } BatchJobs/man/testJob.Rd0000644000176200001440000000305513207234653014575 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/testJob.R \name{testJob} \alias{testJob} \title{Tests a job by running it with Rscript in a new process.} \usage{ testJob(reg, id, resources = list(), external = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of job to test. Default is first job id of registry.} \item{resources}{[\code{list}]\cr Usually not needed, unless you call the function \code{\link{getResources}} in your job. See \code{\link{submitJobs}}. Default is empty list.} \item{external}{[\code{logical(1)}]\cr Run test in an independent external R session instead of current. The former allows to uncover missing variable definitions (which may accidentially be defined in the current global environment) and the latter is useful to get traceable execeptions. Default is \code{TRUE}.} } \value{ [any]. Result of job. If the job did not complete because of an error, NULL is returned. } \description{ Useful for debugging. Note that neither the registry, database or file directory are changed. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x) if (x==1) stop("oops") else x batchMap(reg, f, 1:2) testJob(reg, 2) } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}} } BatchJobs/man/syncRegistry.Rd0000644000176200001440000000122513207234653015665 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/syncRegistry.R \name{syncRegistry} \alias{syncRegistry} \title{Syncronize staged queries into the registry.} \usage{ syncRegistry(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ Invisibly returns \code{TRUE} on success. } \description{ If the option \dQuote{staged.queries} is enabled, all communication from the nodes to the master is done via files in the subdirectory \dQuote{pending} of the \code{file.dir}. This function checks for such files and merges the information into the database. Usually you do not have to call this function yourself. } BatchJobs/man/makeClusterFunctionsOpenLava.Rd0000644000176200001440000000352113207234653020757 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsOpenLava.R \name{makeClusterFunctionsOpenLava} \alias{makeClusterFunctionsOpenLava} \title{Create cluster functions for OpenLava systems.} \usage{ makeClusterFunctionsOpenLava(template.file, list.jobs.cmd = c("bjobs", "-u $USER", "-w")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{bsub} command. Jobs are killed using the \code{bkill} command and the list of running jobs is retrieved using \code{bjobs -u $USER -w}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfOpenLava}. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/showLog.Rd0000644000176200001440000000302413207234653014601 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/showLog.R \name{showLog} \alias{showLog} \title{Display the contents of a log file.} \usage{ showLog(reg, id, pager = getOption("pager")) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of selected job. Default is first id in registry.} \item{pager}{[\code{any}]\cr Pager to use to display the log. Defaults to \code{getOption("pager")}. This option is passed to \code{file.show} and is highly OS dependant and GUI dependant. If either R's pager or the environment variable \dQuote{PAGER} is set to \dQuote{less} or \dQuote{vim}, the correct part of the log file will be shown. Otherwise you find information about the correct part in the beginning of the displayed file.} } \value{ [\code{character(1)}]. Invisibly returns path to log file. } \description{ Display the contents of a log file, useful in case of errors. Note this rare special case: When you use chunking, submit some jobs, some jobs fail, then you resubmit these jobs again in different chunks, the log files will contain the log of the old, failed job as well. \code{showLog} tries to jump to the correct part of the new log file with a supported pager. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{testJob}} } BatchJobs/man/getJobLocation.Rd0000644000176200001440000000076013207234653016066 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getJobLocation.R \name{getJobLocation} \alias{getJobLocation} \title{Get the physical location of job files on the hard disk.} \usage{ getJobLocation(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} } \value{ [\code{character}] Vector of directories. } \description{ Get the physical location of job files on the hard disk. } BatchJobs/man/debugSSH.Rd0000644000176200001440000000351613207234653014631 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/debugSSH.R \name{debugSSH} \alias{debugSSH} \title{Helper function to debug SSH mode.} \usage{ debugSSH(nodename, ssh.cmd = "ssh", ssh.args = character(0L), rhome = "", r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), dir = getwd()) } \arguments{ \item{nodename}{[\code{character(1)}]\cr Node on which worker should be constructed for the test.} \item{ssh.cmd}{[\code{character(1)})]\cr CLI command to ssh into remote node. Default is \dQuote{ssh}.} \item{ssh.args}{[\code{character}]\cr CLI args for \code{ssh.cmd}. Default is none.} \item{rhome}{[\code{character(1)}]\cr Path to R installation on the worker. \dQuote{} means R installation on the PATH is used, of course this implies that it must be on the PATH (also for non-interactive shells)! Default is \dQuote{}.} \item{r.options}{[\code{list}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} \item{dir}{[\code{character(1)}]\cr Path where internally used test registries can be created. Note that this must be shared for the worker. Default is current working directory.} } \value{ Nothing. } \description{ Useful in case of configuration problems. Tries different operations of increasing difficulty and provides debug output on the console. Note that this function does not access nor use information specified for your cluster functions in your configuration. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/removeRegistrySourceDirs.Rd0000644000176200001440000000155213207234653020214 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{removeRegistrySourceDirs} \alias{removeRegistrySourceDirs} \title{Remove packages from registry.} \usage{ removeRegistrySourceDirs(reg, src.dirs) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{src.dirs}{[\code{character}]\cr Paths to remove from registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.dirs} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/getJobIds.Rd0000644000176200001440000000050013207234653015025 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Job.R \name{getJobIds} \alias{getJobIds} \title{Get ids of jobs in registry.} \usage{ getJobIds(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ [\code{character}]. } \description{ Get ids of jobs in registry. } BatchJobs/man/getResources.Rd0000644000176200001440000000063113207234653015632 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Resources.R \name{getResources} \alias{getResources} \title{Function to get job resources in job function.} \usage{ getResources() } \value{ [\code{list}]. } \description{ Return the list passed to \code{\link{submitJobs}}, e.g. nodes, walltime, etc. } \details{ Can only be called in job function during job execution on slave. } BatchJobs/man/batchReduce.Rd0000644000176200001440000000251013207234653015367 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchReduce.R \name{batchReduce} \alias{batchReduce} \title{Reduces via a binary function over a list adding jobs to a registry.} \usage{ batchReduce(reg, fun, xs, init, block.size, more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Empty Registry.} \item{fun}{[\code{function(aggr, x, ...)}]\cr Function to reduce \code{xs} with.} \item{xs}{[\code{vector}]\cr Vector to reduce.} \item{init}{[any]\cr Initial object for reducing.} \item{block.size}{[\code{integer(1)}]\cr Number of elements of \code{xs} reduced in one job.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ Vector of type \code{integer} with job ids. } \description{ Each jobs reduces a certain number of elements on one slave. You can then submit these jobs to the batch system. } \examples{ # define function to reduce on slave, we want to sum a vector f = function(aggr, x) aggr + x reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) # sum 20 numbers on each slave process, i.e. 5 jobs batchReduce(reg, fun = f, 1:100, init = 0, block.size = 5) submitJobs(reg) waitForJobs(reg) # now reduce one final time on master reduceResults(reg, fun = function(aggr,job,res) f(aggr, res)) } BatchJobs/man/makeClusterFunctionsTorque.Rd0000644000176200001440000000351313207234653020532 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsTorque.R \name{makeClusterFunctionsTorque} \alias{makeClusterFunctionsTorque} \title{Create cluster functions for torque-based systems.} \usage{ makeClusterFunctionsTorque(template.file, list.jobs.cmd = c("qselect", "-u $USER", "-s EHQRTW")) } \arguments{ \item{template.file}{[\code{character(1)}]\cr Path to a brew template file that is used for the PBS job file.} \item{list.jobs.cmd}{[\code{character}]\cr Change default system command / options to list jobs. The first entry is the command, the following the options. See \code{\link[BBmisc]{system3}}.} } \value{ [\code{\link{ClusterFunctions}}]. } \description{ Job files are created based on the brew template \code{template.file}. This file is processed with brew and then submitted to the queue using the \code{qsub} command. Jobs are killed using the \code{qdel} command and the list of running jobs is retrieved using \code{qselect}. The user must have the appropriate privileges to submit, delete and list jobs on the cluster (this is usually the case). The template file can access all arguments passed to the \code{submitJob} function, see here \code{\link{ClusterFunctions}}. It is the template file's job to choose a queue for the job and handle the desired resource allocations. Examples can be found on \url{https://github.com/tudo-r/BatchJobs/tree/master/examples/cfTorque}. } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/addRegistryPackages.Rd0000644000176200001440000000151613207234653017103 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Packages.R \name{addRegistryPackages} \alias{addRegistryPackages} \title{Add packages to registry.} \usage{ addRegistryPackages(reg, packages) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{packages}{[\code{character}]\cr Packages to add to registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{packages} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/getLogFiles.Rd0000644000176200001440000000134013207234653015362 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getLogFiles.R \name{getLogFiles} \alias{getLogFiles} \title{Get log file paths for jobs.} \usage{ getLogFiles(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} } \value{ [\code{character}]. Vector of file paths to log files. } \description{ Get log file paths for jobs. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/debugMulticore.Rd0000644000176200001440000000160413207234653016133 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/debugMulticore.R \name{debugMulticore} \alias{debugMulticore} \title{Helper function to debug multicore mode.} \usage{ debugMulticore(r.options) } \arguments{ \item{r.options}{[\code{list}] Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} } \value{ Nothing. } \description{ Useful in case of severe errors. Tries different operations of increasing difficulty and provides debug output on the console } \seealso{ Other debug: \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/removeRegistryPackages.Rd0000644000176200001440000000153613207234653017652 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Packages.R \name{removeRegistryPackages} \alias{removeRegistryPackages} \title{Remove packages from registry.} \usage{ removeRegistryPackages(reg, packages) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{packages}{[\code{character}]\cr Packages to remove from registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{packages} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/getJobParamDf.Rd0000644000176200001440000000077313207234653015634 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getJobParamDf.R \name{getJobParamDf} \alias{getJobParamDf} \title{Retrieve Job Parameters.} \usage{ getJobParamDf(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} } \value{ [\code{data.frame}]. Rows are named with job ids. } \description{ Returns parameters for all jobs as the rows of a data.frame. } \examples{ # see batchExpandGrid } BatchJobs/man/getJobResources.Rd0000644000176200001440000000127613207234653016273 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Resources.R \name{getJobResources} \alias{getJobResources} \title{Function to get the resources that were submitted for some jobs.} \usage{ getJobResources(reg, ids, as.list = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all submitted jobs.} \item{as.list}{[\code{integer(1)}]\cr If \code{FALSE} a data.frame will be returned. Default is \code{TRUE}.} } \value{ [\code{list} | \code{data.frame}]. List (or data.frame) of resource lists as passed to \code{\link{submitJobs}}. } \description{ Throws an error if call it for unsubmitted jobs. } BatchJobs/man/batchUnexport.Rd0000644000176200001440000000163613207234653016014 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Exports.R \name{batchUnexport} \alias{batchUnexport} \title{Unload exported R objects.} \usage{ batchUnexport(reg, what) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{what}{[\code{character}]\cr Names of objects to remove.} } \value{ [\code{character}]. Invisibly returns a character vector of unexported objects. } \description{ Removes \code{RData} files from the \dQuote{exports} subdirectory of your \code{file.dir} and thereby prevents loading on the slave. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/getSSHWorkersInfo.Rd0000644000176200001440000000140313207234653016504 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getSSHWorkersInfo.R \name{getSSHWorkersInfo} \alias{getSSHWorkersInfo} \title{Print and return R installation and other information for SSH workers.} \usage{ getSSHWorkersInfo(nodenames) } \arguments{ \item{nodenames}{[\code{character}]\cr Nodenames of workers.} } \value{ [\code{list}]. Displayed information as a list named by nodenames. } \description{ Workers are queried in parallel via \code{\link{callFunctionOnSSHWorkers}}. The function will display a warning if the first lib path on the worker is not writable as this indicates potential problems in the configuration and \code{\link{installPackagesOnSSHWorkers}} will not work. } \seealso{ \code{\link{callFunctionOnSSHWorkers}} } BatchJobs/man/sweepRegistry.Rd0000644000176200001440000000316213207234653016036 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sweepRegistry.R \name{sweepRegistry} \alias{sweepRegistry} \title{Sweep obsolete files from the file system.} \usage{ sweepRegistry(reg, sweep = c("scripts", "conf")) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{sweep}{[\code{character}]\cr Possible choices: Temporary R scripts of jobs, really not needed for anything else then execution (\dQuote{scripts}), log file of jobs, think about whether you later want to inspect them (\dQuote{logs}), BatchJobs configuration files which are temporarily stored on submit, really not needed for anything else then execution (\dQuote{conf}), resource lists of \code{\link{submitJobs}} which are temporarily stored on submit, think about whether you later want to inspect them (\dQuote{resources}), Default is \code{c("scripts", "conf")}.} } \value{ [\code{logical}]. Invisibly returns \code{TRUE} on success and \code{FALSE} if some files could not be removed. } \description{ Removes R scripts, log files, resource informations and temporarily stored configuration files from the registry's file directory. Assuming all your jobs completed successfully, none of these are needed for further work. This operation potentially releases quite a lot of disk space, depending on the number of your jobs. BUT A HUGE WORD OF WARNING: IF you later notice something strange and need to determine the reason for it, you are at a huge disadvantage. Only do this at your own risk and when you are sure that you have successfully completed a project and only want to archive your produced experiments and results. } BatchJobs/man/batchExpandGrid.Rd0000644000176200001440000000257013207234653016213 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/batchExpandGrid.R \name{batchExpandGrid} \alias{batchExpandGrid} \title{Map function over all combinations.} \usage{ batchExpandGrid(reg, fun, ..., more.args = list()) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Empty Registry that will store jobs for the mapping.} \item{fun}{[\code{function}]\cr Function to map over the combinations.} \item{...}{[any]\cr Vectors that are used to compute all combinations. If the arguments are named, these names are used to bind to arguments of \code{fun}.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} } \value{ [\code{data.frame}]. Expanded grid of combinations produced by \code{\link{expand.grid}}. } \description{ Maps an n-ary-function over a list of all combinations which are given by some vectors. Internally \code{\link{expand.grid}} is used to compute the combinations, then \code{\link{batchMap}} is called. } \examples{ reg = makeRegistry(id = "BatchJobsExample", file.dir = tempfile(), seed = 123) f = function(x, y, z) x * y + z # lets store the param grid grid = batchExpandGrid(reg, f, x = 1:2, y = 1:3, more.args = list(z = 10)) submitJobs(reg) waitForJobs(reg) y = reduceResultsVector(reg) # later, we can always access the param grid like this grid = getJobParamDf(reg) cbind(grid, y = y) } BatchJobs/man/BatchJobs.Rd0000644000176200001440000000321112623105424015006 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/zzz.R \docType{package} \name{BatchJobs} \alias{BatchJobs} \alias{BatchJobs-package} \title{The BatchJobs package} \description{ The BatchJobs package } \section{Additional information}{ \describe{ \item{Homepage:}{\url{https://github.com/tudo-r/BatchJobs}} \item{Wiki:}{\url{https://github.com/tudo-r/BatchJobs/wiki}} \item{FAQ:}{\url{https://github.com/tudo-r/BatchJobs/wiki/FAQ}} \item{Configuration:}{\url{https://github.com/tudo-r/BatchJobs/wiki/Configuration}} } The package currently support the following further R options, which you can set either in your R profile file or a script via \code{\link{options}}: \describe{ \item{BatchJobs.verbose}{This boolean flag can be set to \code{FALSE} to reduce the console output of the package operations. Usually you want to see this output in interactive work, but when you use the package in e.g. knitr documents, it clutters the resulting document too much.} \item{BatchJobs.check.posix}{If this boolean flag is enabled, the package checks your registry file dir (and related user-defined directories) quite strictly to be POSIX compliant. Usually this is a good idea, you do not want to have strange chars in your file paths, as this might results in problems when these paths get passed to the scheduler or other command-line tools that the package interoperates with. But on some OS this check might be too strict and cause problems. Setting the flag to \code{FALSE} allows to disable the check entirely. The default is \code{FALSE} on Windows systems and \code{TRUE} else.} } } BatchJobs/man/getJobNr.Rd0000644000176200001440000000050413207234653014671 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Job.R \name{getJobNr} \alias{getJobNr} \title{Get number of jobs in registry.} \usage{ getJobNr(reg) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} } \value{ [\code{integer(1)}]. } \description{ Get number of jobs in registry. } BatchJobs/man/makeClusterFunctionsSSH.Rd0000644000176200001440000000361013207234653017706 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsSSH.R \name{makeClusterFunctionsSSH} \alias{makeClusterFunctionsSSH} \title{Create an SSH cluster to execute jobs.} \usage{ makeClusterFunctionsSSH(..., workers) } \arguments{ \item{...}{[\code{\link{SSHWorker}}]\cr Worker objects, all created with \code{\link{makeSSHWorker}}.} \item{workers}{[list of \code{\link{SSHWorker}}]\cr Alternative way to pass workers.} } \value{ [\code{ClusterFunctions}]. } \description{ Worker nodes must share the same file system and be accessible by ssh without manually entering passwords (e.g. by ssh-agent or passwordless pubkey). Note that you can also use this function to parallelize on multiple cores on your local machine. But you still have to run an ssh server and provide passwordless access to localhost. } \examples{ \dontrun{ # Assume you have three nodes larry, curley and moe. All have 6 # cpu cores. On curley and moe R is installed under # "/opt/R/R-current" and on larry R is installed under # "/usr/local/R/". larry should not be used extensively because # somebody else wants to compute there as well. # Then a call to 'makeClusterFunctionsSSH' # might look like this: cluster.functions = makeClusterFunctionsSSH( makeSSHWorker(nodename = "larry", rhome = "/usr/local/R", max.jobs = 2), makeSSHWorker(nodename = "curley", rhome = "/opt/R/R-current"), makeSSHWorker(nodename = "moe", rhome = "/opt/R/R-current")) } } \seealso{ \code{\link{makeSSHWorker}} Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsLocal}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/man/checkIds.Rd0000644000176200001440000000133313207234653014675 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/checkIds.R \name{checkIds} \alias{checkIds} \title{Check job ids.} \usage{ checkIds(reg, ids, check.present = TRUE, len = NULL) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Vector of job ids.} \item{check.present}{[\code{logical(1)}]\cr Check if the ids are present in the database? Default is \code{TRUE}.} \item{len}{[\code{integer(1)}]\cr Expected length of id vector. Passed to \code{\link[checkmate]{asInteger}}.} } \value{ Invisibly returns the vector of ids, converted to integer. } \description{ Simply checks if probided vector of job ids is valid and throws an error if something is odd. } BatchJobs/man/getJob.Rd0000644000176200001440000000072113207234653014372 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/getJob.R \name{getJob} \alias{getJob} \title{Get job from registry by id.} \usage{ getJob(reg, id, check.id = TRUE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{id}{[\code{integer(1)}]\cr Id of job.} \item{check.id}{[\code{logical(1)}]\cr Check the job id? Default is \code{TRUE}.} } \value{ [\code{\link{Job}}]. } \description{ Get job from registry by id. } BatchJobs/man/resetJobs.Rd0000644000176200001440000000304413207234653015121 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/resetJobs.R \name{resetJobs} \alias{resetJobs} \title{Reset computational state of jobs.} \usage{ resetJobs(reg, ids, force = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs to kill. Default is none.} \item{force}{[\code{logical(1)}]\cr Reset jobs without checking whether they are currently running. READ THE DETAILS SECTION! Default is \code{FALSE}.} } \value{ Vector of reseted job ids. } \description{ Reset state of jobs in the database. Useful under two circumstances: Either to re-submit them because of changes in e.g. external data or to resolve rare issues when jobs are killed in an unfortunate state and therefore blocking your registry. The function internally lists all jobs on the batch system and if those include some of the jobs you want to reset, it informs you to kill them first by raising an exception. If you really know what you are doing, you may set \code{force} to \code{TRUE} to omit this sanity check. Note that this is a dangerous operation to perform which may harm the database integrity. In this case you HAVE to make externally sure that none of the jobs you want to reset are still running. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{setJobFunction}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/cfBrewTemplate.Rd0000644000176200001440000000221713207234653016066 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfBrewTemplate} \alias{cfBrewTemplate} \title{Cluster functions helper: Brew your template into a job description file.} \usage{ cfBrewTemplate(conf, template, rscript, extension) } \arguments{ \item{conf}{[\code{environment}]\cr BatchJobs configuration.} \item{template}{[\code{character(1)}]\cr Job desfription template as a char vecrtor, possibly read in via \code{\link{cfReadBrewTemplate}}.} \item{rscript}{[\code{character(1)}]\cr File path to you your corresponding R script for the job.} \item{extension}{[\code{character(1)}]\cr Extension for the job description file, e.g. \dQuote{pbs}.} } \value{ [\code{character(1)}]. File path of result. } \description{ This function is only intended for use in your own cluster functions implementation. Calls brew silently on your template, any error will lead to an exception. If debug mode is turned on in the configuration, the file is stored at the same place as the corresponding R script in the \dQuote{jobs}-subdir of your files directory, otherwise in the temp dir via \code{\link{tempfile}}. } BatchJobs/man/sourceRegistryFiles.Rd0000644000176200001440000000102213207234653017167 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{sourceRegistryFiles} \alias{sourceRegistryFiles} \title{Source registry files} \usage{ sourceRegistryFiles(reg, envir = .GlobalEnv) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{envir}{[\code{environment}]\cr Environment to source the files into. Default is the global environment.} } \value{ Nothing. } \description{ Sources all files found in \code{src.dirs} and specified via \code{src.files}. } BatchJobs/man/loadExports.Rd0000644000176200001440000000172413207234653015470 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Exports.R \name{loadExports} \alias{loadExports} \title{Load exported R data objects.} \usage{ loadExports(reg, what = NULL) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{what}{[\code{character}]\cr Names of objects to load. Defaults to all objects exported.} } \value{ [\code{character}]. Invisibly returns a character vector of loaded objects. } \description{ Loads exported \code{RData} object files in the \dQuote{exports} subdirectory of your \code{file.dir} and assigns the objects to the global environment. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{removeRegistrySourceFiles}}, \code{\link{setRegistryPackages}} } BatchJobs/man/setJobFunction.Rd0000644000176200001440000000265013207234653016117 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/setJobFunction.R \name{setJobFunction} \alias{setJobFunction} \title{Sets the job function for already existing jobs.} \usage{ setJobFunction(reg, ids, fun, more.args = list(), reset = TRUE, force = FALSE) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of jobs. Default is all jobs.} \item{fun}{[\code{function}]\cr Replacement function.} \item{more.args}{[\code{list}]\cr A list of other arguments passed to \code{fun}. Default is empty list.} \item{reset}{[\code{logical(1)}]\cr Reset job status via \code{\link{resetJobs}}. Default is \code{TRUE}.} \item{force}{[\code{logical(1)}]\cr See \code{\link{resetJobs}}. Default is \code{FALSE}.} } \value{ Nothing. } \description{ Use this function only as last measure when there is a bug in a part of your job function and you have already computed a large number of (unaffected) results. This function allows you to fix the error and to associate the jobs with the corrected function. Note that by default the computational state of the affected jobs is also reset. } \seealso{ Other debug: \code{\link{debugMulticore}}, \code{\link{debugSSH}}, \code{\link{getErrorMessages}}, \code{\link{getJobInfo}}, \code{\link{getLogFiles}}, \code{\link{grepLogs}}, \code{\link{killJobs}}, \code{\link{resetJobs}}, \code{\link{showLog}}, \code{\link{testJob}} } BatchJobs/man/callFunctionOnSSHWorkers.Rd0000644000176200001440000000431613207234653020035 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/callFunctionOnSSHWorkers.R \name{callFunctionOnSSHWorkers} \alias{callFunctionOnSSHWorkers} \title{Call an arbitrary function on specified SSH workers.} \usage{ callFunctionOnSSHWorkers(nodenames, fun, ..., consecutive = FALSE, show.output = consecutive, simplify = TRUE, use.names = TRUE, dir = getwd()) } \arguments{ \item{nodenames}{[\code{character}]\cr Nodenames of workers to call function on. Only workers which were specified in your \code{\link{makeClusterFunctionsSSH}} configuration can be used.} \item{fun}{[\code{function}]\cr Function to call on workers.} \item{...}{[any]\cr Arguments for \code{fun}.} \item{consecutive}{[\code{logical(1)}]\cr Do calls consecutively and always wait until each worker is done. Default is \code{FALSE}.} \item{show.output}{[\code{logical(1)}]\cr Show output of workers on master during call. Can be useful to see what is happening. Can only be used in consecutive mode. Default is \code{consecutive}.} \item{simplify}{[\code{logical(1)}]\cr Should the result be simplified? See \code{\link{sapply}}. Default is \code{TRUE}.} \item{use.names}{[\code{logical(1)}]\cr Name results by \code{nodenames}. Default is \code{TRUE}.} \item{dir}{[\code{character(1)}]\cr Directory under which a temporary registry will be created in a subdirectory for communication. This has to be somewhere on the shared filesystem. The created subdirectory will be cleaned up on exit. Default is current working directory.} } \value{ Results of function calls, either a list or simplified. } \description{ Calls can be made in parallel or consecutively, the function waits until all calls have finished and returns call results. In consecutive mode the output on the workers can also be shown on the master during computation. Please read and understand the comments for argument \code{dir}. Note that this function should only be used for short administrative tasks or information gathering on the workers, the true work horse for real computation is \code{\link{submitJobs}}. In \code{\link{makeSSHWorker}} various options for load management are possible. Note that these will be ignored for the current call to execute it immediatly. } BatchJobs/man/dbGetJobs.Rd0000644000176200001440000000065513207234653015031 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/database.R \name{dbGetJobs} \alias{dbGetJobs} \title{ONLY FOR INTERNAL USAGE.} \usage{ dbGetJobs(reg, ids) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ids}{[\code{integer}]\cr Ids of selected jobs.} } \value{ [list of \code{\link{Job}}]. Retrieved jobs from DB. } \description{ ONLY FOR INTERNAL USAGE. } \keyword{internal} BatchJobs/man/getConfig.Rd0000644000176200001440000000071313207234653015066 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/conf.R \name{getConfig} \alias{getConfig} \title{Returns a list of BatchJobs configuration settings} \usage{ getConfig() } \value{ \code{list} of current configuration variables with classs \dQuote{Config}. } \description{ Returns a list of BatchJobs configuration settings } \seealso{ Other conf: \code{\link{configuration}}, \code{\link{loadConfig}}, \code{\link{setConfig}} } BatchJobs/man/makeSSHWorker.Rd0000644000176200001440000000460313207234653015650 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsSSH.R \name{makeSSHWorker} \alias{makeSSHWorker} \alias{SSHWorker} \title{Create SSH worker for SSH cluster functions.} \usage{ makeSSHWorker(nodename, ssh.cmd = "ssh", ssh.args = character(0L), rhome = "", ncpus, max.jobs, max.load, nice, r.options = c("--no-save", "--no-restore", "--no-init-file", "--no-site-file"), script) } \arguments{ \item{nodename}{[\code{character(1)}]\cr Host name of node.} \item{ssh.cmd}{[\code{character(1)})]\cr CLI command to ssh into remote node. Default is \dQuote{ssh}.} \item{ssh.args}{[\code{character}]\cr CLI args for \code{ssh.cmd}. Default is none.} \item{rhome}{[\code{character(1)}]\cr Path to R installation on worker. \dQuote{} means R installation on the PATH is used, of course this implies that it must be on the PATH (also for non-interactive shells)! Default is \dQuote{}.} \item{ncpus}{[\code{integers(1)}]\cr Number of VPUs of worker. Default means to query the worker via \dQuote{/proc/cpuinfo}.} \item{max.jobs}{[\code{integer(1)}]\cr Maximal number of jobs that can run concurrently for the current registry. Default is \code{ncpus}.} \item{max.load}{[\code{numeric(1)}]\cr Load average (of the last 5 min) at which the worker is considered occupied, so that no job can be submitted. Default is \code{ncpus-1}.} \item{nice}{[\code{integer(1)}]\cr Process priority to run R with set via nice. Integers between -20 and 19 are allowed. If missing, processes are not nice'd and the system default applies (usually 0). Default is no niceing.} \item{r.options}{[\code{character}]\cr Options for R and Rscript, one option per element of the vector, a la \dQuote{--vanilla}. Default is \code{c("--no-save", "--no-restore", "--no-init-file", "--no-site-file")}.} \item{script}{[\code{character(1)}]\cr Path to helper bash script which interacts with the worker. You really should not have to touch this, as this would imply that we have screwed up and published an incompatible version for your system. This option is only provided as a last resort for very experienced hackers. Note that the path has to be absolute. This is what is done in the package: \url{https://github.com/tudo-r/BatchJobs/blob/master/inst/bin/linux-helper} Default means to take it from package directory.} } \value{ [\code{\link{SSHWorker}}]. } \description{ Create SSH worker for SSH cluster functions. } BatchJobs/man/removeRegistrySourceFiles.Rd0000644000176200001440000000156313207234653020357 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/sourceRegistryFiles.R \name{removeRegistrySourceFiles} \alias{removeRegistrySourceFiles} \title{Remove source files from registry.} \usage{ removeRegistrySourceFiles(reg, src.files) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{src.files}{[\code{character}]\cr Paths to remove from registry.} } \value{ [\code{\link{Registry}}]. Changed registry. } \description{ Mutator function for \code{src.files} in \code{\link{makeRegistry}}. } \seealso{ Other exports: \code{\link{addRegistryPackages}}, \code{\link{addRegistrySourceDirs}}, \code{\link{addRegistrySourceFiles}}, \code{\link{batchExport}}, \code{\link{batchUnexport}}, \code{\link{loadExports}}, \code{\link{removeRegistryPackages}}, \code{\link{removeRegistrySourceDirs}}, \code{\link{setRegistryPackages}} } BatchJobs/man/cfReadBrewTemplate.Rd0000644000176200001440000000121513207234653016657 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsHelpers.R \name{cfReadBrewTemplate} \alias{cfReadBrewTemplate} \title{Cluster functions helper: Read in your brew template file.} \usage{ cfReadBrewTemplate(template.file) } \arguments{ \item{template.file}{[\code{character(1)}]\cr File path.} } \value{ [\code{character}]. } \description{ This function is only intended for use in your own cluster functions implementation. Simply reads your template and returns it as a character vector. If you do this in the constructor of your cluster functions once, you can avoid this repeated file access later on. } BatchJobs/man/removeRegistry.Rd0000644000176200001440000000152413207234653016210 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/Registry.R \name{removeRegistry} \alias{removeRegistry} \title{Remove a registry object.} \usage{ removeRegistry(reg, ask = c("yes", "no")) } \arguments{ \item{reg}{[\code{\link{Registry}}]\cr Registry.} \item{ask}{[\code{character(1)}]\cr If \code{"yes"} the user is prompted to confirm the action. If trying to prompt the user this way in a non-interactive session, then an informative error is generated. If \code{"no"}, the registry will be removed without further confirmation.} } \value{ [\code{logical[1]}] } \description{ If there are no live/running jobs, the registry will be closed and all of its files will be removed from the file system. If there are live/running jobs, an informative error is generated. The default is to prompt the user for confirmation. } BatchJobs/man/makeClusterFunctionsLocal.Rd0000644000176200001440000000226013207234653020303 0ustar liggesusers% Generated by roxygen2: do not edit by hand % Please edit documentation in R/clusterFunctionsLocal.R \name{makeClusterFunctionsLocal} \alias{makeClusterFunctionsLocal} \title{Create cluster functions for sequential execution on local host.} \usage{ makeClusterFunctionsLocal() } \value{ [\code{\link{ClusterFunctions}}]. } \description{ All jobs executed under these cluster functions are executed sequentially, but in an independent, new R session. That is, \code{submitJob} does not return until the job has finished. The main use of this \code{ClusterFunctions} implementation is to test and debug programs on a local computer. Listing jobs returns an empty vector (as no jobs can be running when you call this) and \code{killJob} returns at once (for the same reason). } \seealso{ Other clusterFunctions: \code{\link{makeClusterFunctionsInteractive}}, \code{\link{makeClusterFunctionsLSF}}, \code{\link{makeClusterFunctionsMulticore}}, \code{\link{makeClusterFunctionsOpenLava}}, \code{\link{makeClusterFunctionsSGE}}, \code{\link{makeClusterFunctionsSLURM}}, \code{\link{makeClusterFunctionsSSH}}, \code{\link{makeClusterFunctionsTorque}}, \code{\link{makeClusterFunctions}} } BatchJobs/LICENSE0000644000176200001440000000007412357424573013133 0ustar liggesusersYEAR: 2013-2014 COPYRIGHT HOLDER: Bernd Bischl, Michel Lang