diff --git a/.gitignore b/.gitignore index e38ef5e15f..b10f4b78fb 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ doc/ # Ignore "main.lst" files main.lst +/main_*.gms # Ignore .RData and log files only in the root folder /*.RData diff --git a/CHANGELOG.md b/CHANGELOG.md index 2818616c75..af220936cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### input data/calibration ### changed -- +- **scripts** Harmonize command line arguments for output.R: prefix `--` is now required, change `--outputdir` to `--outputdirs` + [[#2413](https://github.com/remindmodel/remind/pull/2413)] ### added - diff --git a/output.R b/output.R index fea64a7b29..1838239336 100755 --- a/output.R +++ b/output.R @@ -21,60 +21,8 @@ options(error = quote({ q() })) -helpText <- " -#' Rscript output.R [options] -#' -#' [options] can be the following flags: -#' -#' --help, -h show this help text and exit -#' --test, -t tests output.R without actually starting any run -#' --update update packages in renv first, incompatible with --renv= -#' -#' [options] can also specify the following variables. If they are not specified -#' but needed, the scripts will ask the user. -#' -#' --comp= comp=single means output for single runs -#' (e.g. reporting, ...) -#' comp=comparison means scripts to compare runs -#' (e.g. compareScenarios2, ...) -#' comp=export means scripts to export runs -#' (e.g. xlsx_IIASA, ...) -#' -#' --filename_prefix= string to be added to filenames by some output scripts -#' (compareScenarios, xlsx_IIASA) -#' -#' --output= output=compareScenarios2 directly selects the specific -#' script (without .R extension) -#' -#' --outputdir= Directly specify the output directories, bypassing run -#' selection, as a comma-separated list -#' (e.g. outputdir=./output/SSP2-Base-rem-1,./output/NDC-rem-1) -#' -#' --aliases= Specify aliases for the runs given in outputdir, -#' as a comma-separated list -#' (e.g. aliases=default,modified) -#' -#' --remind_dir= path to remind or output directories where runs can be -#' found. Defaults to ./output but can also be used to -#' specify multiple folders, comma-separated, such as -#' remind_dir=.,../otherremind -#' -#' --renv= load the renv located at , incompatible with -#' --update -#' -#' --slurmConfig= use slurmConfig=priority, short or standby to specify -#' slurm selection. You may also pass multiple SLURM -#' arguments such as slurmConfig='--qos=priority --mem=8000' -" - +# preliminary option parsing to avoid loading any libraries that cause trouble with renv argv <- get0("argv", ifnotfound = commandArgs(trailingOnly = TRUE)) -timestamp <- format(Sys.time(), "%Y-%m-%d_%H.%M.%S") - -if (any(c("-h", "--help") %in% argv)) { - message(gsub("#' ?", '', helpText)) - q() -} - # run updates before loading any packages if ("--update" %in% argv) { stopifnot(`--update must not be used together with --renv=...` = !any(startsWith(argv, "--renv="))) @@ -85,7 +33,7 @@ if ("--update" %in% argv) { renv::load(renvProject) } -# load landuse library +library(optparse) library(lucode2) library(gms) require(stringr, quietly = TRUE) @@ -93,16 +41,44 @@ require(stringr, quietly = TRUE) # Import all functions from the scripts/start folder invisible(sapply(list.files("scripts/start", pattern = "\\.R$", full.names = TRUE), source)) -flags <- NULL -### Define arguments that can be read from command line -if (!exists("source_include")) { - # if this script is not being sourced by another script but called from the command line via Rscript read the command - # line arguments and let the user choose the slurm options - flags <- readArgs("outputdir", "output", "aliases", "comp", "remind_dir", "slurmConfig", "filename_prefix", - .flags = c(t = "--test", h = "--help")) +# parse options from command line and return them as a named list +parseOptions <- function() { + options <- list( + make_option(c("-t", "--test"), action="store_true", default=FALSE, + help="test output.R without actually starting any run"), + make_option("--update", action="store_true", default=FALSE, + help="update packages in renv first, incompatible with --renv"), + make_option("--comp", type="character", default=NULL, + help="specify output type: 'single' for single runs (e.g. reporting), 'comparison' for run comparisons (e.g. compareScenarios2), or 'export' to export runs (e.g. xlsx_IIASA)"), + make_option("--filename_prefix", type="character", default=NULL, + help="string to be added to filenames by some output scripts (compareScenarios, xlsx_IIASA)"), + make_option("--output", type="character", default=NULL, + help="directly select a specific script (without .R extension)"), + make_option("--outputdirs", type="character", default=NULL, + help="directly specify output directories as comma-separated list (e.g. ./output/SSP2-Base-rem-1,./output/NDC-rem-1)"), + make_option("--aliases", type="character", default=NULL, + help="Specify aliases for the runs given in outputdirs as a comma-separated list (e.g. aliases=default,modified)"), + make_option("--remind_dir", type="character", default=NULL, + help="path to remind or output directories where runs can be found. Defaults to ./output. Can specify multiple comma-separated folders (e.g. .,../otherremind)"), + make_option("--renv", type="character", default=NULL, + help="load the renv located at , incompatible with --update"), + make_option("--slurmConfig", type="character", default=NULL, + help="specify SLURM selection: use 'priority', 'short', or 'standby', or pass multiple SLURM arguments (e.g. '--qos=priority --mem=8000')") + ) + parser <- OptionParser(usage="Rscript output.R [options prefixed by --, e.g. --comp=single]", option_list=options, + description="[options] can be the following flags and variables. If variables are not specified but needed, the scripts will ask the user.") + # these flags appear in the various output scripts and are necessary here + # if you add a command line argument to a script, add it here as well + additionalScriptOptions = list("profileNames", "runs", "folder", "project", + "outputFilename", "model", "mapping", "summationFile", "logFile", "removeFromScen", + "addToScen", "iiasatemplate", "timesteps", "validationConfig", "interactive") + for (option in additionalScriptOptions) { + parser <- add_option(parser, paste0("--", option), help="This option is used in an output script, see your script for information.") + } + return(parse_args(parser)) } -choose_slurmConfig_output <- function(output) { +chooseSlurmConfigOutput <- function(output) { slurm_options <- c("--qos=priority", "--qos=short", "--qos=standby", "--qos=priority --mem=8000", "--qos=short --mem=8000", "--qos=standby --mem=8000", "--qos=priority --mem=32000") @@ -129,12 +105,12 @@ choose_slurmConfig_output <- function(output) { return(if (any(identifier)) slurm_options[as.numeric(which(identifier))] else slurm_options[1]) } -choose_filename_prefix <- function(modules, title = "") { +chooseFilenamePrefix <- function(modules, title = "") { cat(paste0("\n\n ", title, "Please choose a prefix for filenames of ", paste(modules, collapse=", "), ".\n")) cat(" For example compareScenarios2 uses it for the filenames: compScen-yourprefix-2022-….pdf.\n Use only A-Za-z0-9_-, or leave empty:\n\n") filename_prefix <- gms::getLine() if(grepl("[^A-Za-z0-9_-]", filename_prefix)) { - filename_prefix <- choose_filename_prefix(modules, title = paste("No, this contained special characters, try again.\n",title)) + filename_prefix <- chooseFilenamePrefix(modules, title = paste("No, this contained special characters, try again.\n",title)) } return(filename_prefix) } @@ -161,19 +137,15 @@ promptForAliases <- function(outputdirs, scenarios) { return(scenarios) } - -if (exists("source_include")) { - comp <- "single" -} else if (! exists("comp")) { +chooseCompMode <- function() { modes <- c("single" = "Output for single run", "comparison" = "Comparison across runs", "export" = "Export", "exit" = "Exit") comp <- names(modes)[which(chooseFromList(unname(modes), type = "output mode", multiple = FALSE, returnBoolean = TRUE, userinfo = "Leave empty for 'single'."))] if (length(comp) == 0) comp <- names(modes)[[1]] if (comp == "exit") q() + return(comp) } -if (isFALSE(comp)) comp <- "single" # legacy from times only two comp modes existed -if (isTRUE(comp)) comp <- "comparison" -if (! exists("output")) { +chooseOutputScript <- function(comp) { # search for R scripts in scripts/output subfolders modules <- gsub("\\.R$", "", grep("\\.R$", list.files(paste0("./scripts/output/", comp)), value = TRUE)) # if more than one option exists, let user choose @@ -183,20 +155,20 @@ if (! exists("output")) { if (length(output) == 0) output <- defaultoutput # move "reporting" to first position, if it exists output <- c(if ("reporting" %in% output) "reporting", output[! output %in% "reporting"]) + return(output) } -# Select output directories if not defined by readArgs -if (! exists("outputdir")) { +chooseOutputDirs <- function(output, remind_dir) { modulesNeedingMif <- c("compareScenarios2", "xlsx_IIASA", "policyCosts", "Ariadne_output", "plot_compare_iterations", "varListHtml", "fixOnRef", "MAGICC7_AR6", "validateScenarios", "checkClimatePercentiles", "selectPlots", "checkProjectSummations") needingMif <- any(modulesNeedingMif %in% output) && ! "reporting" %in% output[[1]] - if (exists("remind_dir")) { - dir_folder <- c(file.path(remind_dir, "output"), remind_dir) - } else { + if (is.null(remind_dir)) { defaultcfg <- readDefaultConfig(".") dir_folder <- unique(c("output", dirname(defaultcfg$results_folder))) + } else { + dir_folder <- c(file.path(remind_dir, "output"), remind_dir) } dirs <- dirname(Sys.glob(file.path(dir_folder, "*", "fulldata.gdx"))) if (needingMif) dirs <- intersect(dirs, unique(dirname(Sys.glob(file.path(dir_folder, "*", "REMIND_generic_*.mif"))))) @@ -219,47 +191,24 @@ if (! exists("outputdir")) { returnBoolean = TRUE, multiple = FALSE) outputdirs <- c(rbind(outputdirs, dirs[policyrun[-1]])) # generate 3,1,4,1,5,1 out of 3,4,5 and policyrun 1 } -} else { - outputdirs <- outputdir + return(outputdirs) } -# aliases are names for scenarios (=outputdirs) that are prompted when the scenarios have duplicate names -# currently only compareScenarios2 uses aliases, feel free to implement it for your comparison script -modules_supporting_aliases <- c("compareScenarios2") -if (!exists("aliases") && any(modules_supporting_aliases %in% output)) { +chooseAliases <- function(output, outputdirs) { + modules_supporting_aliases <- c("compareScenarios2") + if (!any(modules_supporting_aliases %in% output)) { + return() + } scenarios <- unname(lucode2::getScenNames(outputdirs)) aliases <- make.unique(scenarios) # For better scripting backwards compatibility, don't prompt for aliases if output dirs were selected with command line parameters - if (!exists("outputdir")) { + if (!exists("source_include")) { aliases = promptForAliases(outputdirs, aliases) } + return(aliases) } -if (exists("aliases")) { - stopifnot("Number of aliases and outputdirs must be equal" = length(aliases) == length(outputdirs)) -} - -if (comp %in% c("comparison", "export")) { - # ask for filename_prefix, if one of the modules that use it is selected - modules_using_filename_prefix <- c("compareScenarios2", "xlsx_IIASA", "varListHtml", "selectPlots") - if (!exists("filename_prefix")) { - if (any(modules_using_filename_prefix %in% output)) { - filename_prefix <- choose_filename_prefix(modules = intersect(modules_using_filename_prefix, output)) - } else { - filename_prefix <- "" - } - } - - # choose the slurm options. If you use command line arguments, use slurmConfig=priority or standby - modules_using_slurmConfig <- c("compareScenarios2", "validateScenarios") - if (!exists("slurmConfig") && any(modules_using_slurmConfig %in% output)) { - slurmConfig <- choose_slurmConfig_output(output = output) - } - if (exists("slurmConfig")) { - if (slurmConfig %in% c("priority", "short", "standby")) { - slurmConfig <- paste0("--qos=", slurmConfig) - } - } +runComparisonOrExport <- function(comp, output, outputdirs, aliases, filename_prefix, slurmConfig, test) { # Set value source_include so that loaded scripts know, that they are # included as source (instead of a load from command line) source_include <- TRUE @@ -268,7 +217,7 @@ if (comp %in% c("comparison", "export")) { for (rout in output) { name <- paste(rout, ".R", sep = "") if (file.exists(paste0("scripts/output/", comp, "/", name))) { - if ("--test" %in% flags) { + if (test) { message("Test mode, not executing ", paste0("scripts/output/", comp, "/", name)) } else { message("\n\n## Executing ", name) @@ -284,33 +233,11 @@ if (comp %in% c("comparison", "export")) { message("\nCould not find ", name) } } -} else { # comp = single - # define slurm class or direct execution - outputInteractive <- c("plotIterations", "integratedDamageCosts") - if (! exists("source_include")) { - if (any(output %in% outputInteractive)) { - slurmConfig <- "direct" - } - # if this script is not being sourced by another script but called from the command line via Rscript let the user - # choose the slurm options - if (!exists("slurmConfig")) { - slurmConfig <- choose_slurmConfig_output(output = output) - if (slurmConfig != "direct") slurmConfig <- combine_slurmConfig("--nodes=1 --tasks-per-node=1 --time=120", slurmConfig) - } - if (slurmConfig %in% c("priority", "short", "standby")) { - slurmConfig <- paste0("--nodes=1 --tasks-per-node=1 --qos=", slurmConfig) - } - if (isTRUE(slurmConfig %in% "direct")) { - flags <- c(flags, "--interactive") # to tell scripts they can run in interactive mode - } - } else { - # if being sourced by another script execute the output scripts directly without sending them to the cluster - slurmConfig <- "direct" - } +} +runSingle <- function(output, outputdirs, slurmConfig, test) { # comp = single # Execute outputscripts for all chosen folders for (outputdir in outputdirs) { - if (exists("cfg")) { title <- cfg$title gms <- cfg$gms @@ -322,21 +249,14 @@ if (comp %in% c("comparison", "export")) { if (!exists("source_include")) { magpie_folder <- getwd() message("Load data from ", file.path(outputdir, "config.Rdata")) - if (file.exists(file.path(outputdir, "config.Rdata"))) { - load(file.path(outputdir, "config.Rdata")) - title <- cfg$title - gms <- cfg$gms - revision <- cfg$inputRevision - } else { - config <- grep("\\.cfg$", list.files(outputdir), value = TRUE) - l <- readLines(file.path(outputdir, config)) - title <- strsplit(grep("(cfg\\$|)title +<-", l, value = TRUE), "\"")[[1]][2] - gms <- list() - gms$scenarios <- strsplit(grep("(cfg\\$|)gms\\$scenarios +<-", l, value = TRUE), "\"")[[1]][2] - revision <- as.numeric(unlist(strsplit(grep("(cfg\\$|)inputRevision +<-", l, value = TRUE), "<-[ \t]*"))[2]) - } + # Old .cfg files will not be read anymore + stopifnot(file.exists(file.path(outputdir, "config.Rdata"))) + load(file.path(outputdir, "config.Rdata")) + title <- cfg$title + gms <- cfg$gms + revision <- cfg$inputRevision } - + # Set value source_include so that loaded scripts know, that they are # included as source (instead of a load from command line) source_include <- TRUE @@ -353,38 +273,37 @@ if (comp %in% c("comparison", "export")) { message("\nStarting output generation for ", outputdir, "\n") name <- paste0(output, ".R") scriptsfound <- file.exists(paste0("scripts/output/single/", name)) - if ("--test" %in% flags) { + if (any(! scriptsfound)) { + warning("Skipping output scripts not found in scripts/output/single: ", name[! scriptsfound]) + } + if (test) { message("Test mode, not executing scripts/output/single/", paste(name, collapse = ", ")) - } else { - if (slurmConfig == "direct") { - # execute output script directly (without sending it to slurm) - for (n in name[scriptsfound]) { - message("Executing ", n) - tmp.env <- new.env() - tmp.error <- try(sys.source(paste0("scripts/output/single/", n), envir = tmp.env)) - # rm(list=ls(tmp.env),envir=tmp.env) - rm(tmp.env) - gc() - if (!is.null(tmp.error)) { - warning("Script ", n, " was stopped by an error and not executed properly!") - } + next + } + if (slurmConfig == "direct") { + # execute output script directly (without sending it to slurm) + for (n in name[scriptsfound]) { + message("Executing ", n) + tmp.env <- new.env() + tmp.error <- try(sys.source(paste0("scripts/output/single/", n), envir = tmp.env)) + # rm(list=ls(tmp.env),envir=tmp.env) + rm(tmp.env) + gc() + if (!is.null(tmp.error)) { + warning("Script ", n, " was stopped by an error and not executed properly!") } - } else { - # send the output script to slurm - logfile <- file.path(outputdir, paste0("log_output_", timestamp, ".txt")) - Rscripts <- paste0("Rscript scripts/output/single/", name, " outputdir=", outputdir, collapse = "; ") - slurmcmd <- paste0("sbatch ", slurmConfig, " --job-name=", logfile, " --output=", logfile, - " --mail-type=END,FAIL --comment=output.R --wrap='", Rscripts, "'") - message("Sending to slurm: ", paste(name, collapse = ", "), ". Find log in ", logfile) - system(slurmcmd) } - # finished - message("\nFinished ", ifelse(slurmConfig == "direct", "", "starting job for "), "output generation for ", outputdir, "!\n") - } - if (any(! scriptsfound)) { - warning("Skipping those output script selected that could not be found in scripts/output/single: ", - name[! scriptsfound]) + } else { + # send the output script to slurm + timestamp <- format(Sys.time(), "%Y-%m-%d_%H.%M.%S") + logfile <- file.path(outputdir, paste0("log_output_", timestamp, ".txt")) + Rscripts <- paste0("Rscript scripts/output/single/", name, " --outputdir=", outputdir, collapse = "; ") + slurmcmd <- paste0("sbatch ", slurmConfig, " --job-name=", logfile, " --output=", logfile, + " --mail-type=END,FAIL --comment=output.R --wrap='", Rscripts, "'") + message("Sending to slurm: ", paste(name, collapse = ", "), ". Find log in ", logfile) + system(slurmcmd) } + message("\nFinished ", ifelse(slurmConfig == "direct", "", "starting job for "), "output generation for ", outputdir, "!\n") } rm(source_include) @@ -393,3 +312,76 @@ if (comp %in% c("comparison", "export")) { } } } + +#' main function of the script +#' prompts the user for all required information to start an output script +#' +#' @param args parsed command line arguments as a named list, for documentation of the arguments, +#' see the command line arguments of this script +output <- function(args) { + remind_dir <- if (is.null(args[["remind_dir"]])) NULL else unlist(strsplit(args[["remind_dir"]], ",")) + comp <- if (is.null(args[["comp"]])) chooseCompMode() else args[["comp"]] + output <- if (is.null(args[["output"]])) chooseOutputScript(comp) else unlist(strsplit(args[["output"]], ",")) + outputdirs <- if (is.null(args[["outputdirs"]])) chooseOutputDirs(output, args[["remind_dir"]]) else unlist(strsplit(args[["outputdirs"]], ",")) + + if (comp %in% c("comparison", "export")) { + # aliases are names for scenarios (=outputdirs) that are prompted when the scenarios have duplicate names + # currently only compareScenarios2 uses aliases, feel free to implement it for your comparison script + if (is.null(args[["aliases"]])) { + aliases <- chooseAliases(output, outputdirs) + } else { + aliases <- unlist(strsplit(args[["aliases"]], ",")) + stopifnot("Number of aliases and outputdirs must be equal" = length(aliases) == length(outputdirs)) + } + if (is.null(args[["filename_prefix"]])) { + # ask for filename_prefix, if one of the modules that use it is selected + modules_using_filename_prefix <- c("compareScenarios2", "xlsx_IIASA", "varListHtml", "selectPlots") + if (any(modules_using_filename_prefix %in% output)) { + filename_prefix <- chooseFilenamePrefix(modules = intersect(modules_using_filename_prefix, output)) + } else { + filename_prefix <- "" + } + } else { + filename_prefix = args[["filename_prefix"]] + } + + # choose the slurm options. If you use command line arguments, use slurmConfig=priority or standby + modules_using_slurmConfig <- c("compareScenarios2", "validateScenarios") + if (any(modules_using_slurmConfig %in% output)) { + if (is.null(args[["slurmConfig"]])) { + slurmConfig <- chooseSlurmConfigOutput(output = output) + } else if (args[["slurmConfig"]] %in% c("priority", "short", "standby")) { + slurmConfig <- paste0("--qos=", args[["slurmConfig"]]) + } else { + slurmConfig = args[["slurmConfig"]] + } + } + runComparisonOrExport(comp, output, outputdirs, aliases, filename_prefix, slurmConfig, args[["test"]]) + } else { + # define slurm class or direct execution + outputInteractive <- c("plotIterations", "integratedDamageCosts") + if (exists("source_include") || any(output %in% outputInteractive)) { + # if being sourced by another script execute the output scripts directly without sending them to the cluster + slurmConfig <- "direct" + } else { + # if it is called from the command line via Rscript let the user choose the slurm options + if (is.null(args[["slurmConfig"]])) { + slurmConfig <- chooseSlurmConfigOutput(output = output) + if (slurmConfig != "direct") slurmConfig <- combine_slurmConfig("--nodes=1 --tasks-per-node=1 --time=120", slurmConfig) + } else if (args[["slurmConfig"]] %in% c("priority", "short", "standby")) { + slurmConfig <- paste0("--nodes=1 --tasks-per-node=1 --qos=", args[["slurmConfig"]]) + } else if (isTRUE(args[["slurmConfig"]] %in% "direct")) { + interactive = TRUE + } else { + slurmConfig = args[["slurmConfig"]] + } + } + runSingle(output, outputdirs, slurmConfig, args[["test"]]) + } +} + +if (exists("source_include")) { + output(passedArgs) +} else { + output(parseOptions()) +} diff --git a/scripts/cs2/run_compareScenarios2.R b/scripts/cs2/run_compareScenarios2.R index 6235371e48..b1e2a8859f 100644 --- a/scripts/cs2/run_compareScenarios2.R +++ b/scripts/cs2/run_compareScenarios2.R @@ -8,11 +8,11 @@ library(piamPlotComparison) if (!exists("source_include")) { - lucode2::readArgs("outputDirs", "outFileName", "profileName", "aliases") + lucode2::readArgs("outputdirs", "outFileName", "profileName", "aliases") } run_compareScenarios2 <- function( - outputDirs, + outputdirs, outFileName, profileName, aliases=NULL @@ -34,15 +34,15 @@ run_compareScenarios2 <- function( system(paste0("mkdir ", outFileName)) outDir <- normalizePath(outFileName, mustWork = TRUE) - outputDirs <- unique(normalizePath(outputDirs, mustWork = TRUE)) + outputdirs <- unique(normalizePath(outputdirs, mustWork = TRUE)) if (profileName == "EDGE-Transport") { - mifPath <- normalizePath(file.path(outputDirs, "EDGE-T", "Transport.mif"), mustWork = TRUE) + mifPath <- normalizePath(file.path(outputdirs, "EDGE-T", "Transport.mif"), mustWork = TRUE) } else { - mifPath <- remind2::getMifScenPath(outputDirs, mustWork = TRUE) + mifPath <- remind2::getMifScenPath(outputdirs, mustWork = TRUE) } - histPath <- remind2::getMifHistPath(outputDirs[1], mustWork = TRUE) - scenConfigPath <- remind2::getCfgScenPath(outputDirs, mustWork = TRUE) + histPath <- remind2::getMifHistPath(outputdirs[1], mustWork = TRUE) + scenConfigPath <- remind2::getCfgScenPath(outputdirs, mustWork = TRUE) # predefined arguments args <- list( @@ -94,4 +94,4 @@ run_compareScenarios2 <- function( message("Done!\n") } -run_compareScenarios2(outputDirs, outFileName, profileName, aliases) +run_compareScenarios2(outputdirs, outFileName, profileName, aliases) diff --git a/scripts/output/comparison/compareScenarios2.R b/scripts/output/comparison/compareScenarios2.R index feacc3f2c4..746eb2ac06 100644 --- a/scripts/output/comparison/compareScenarios2.R +++ b/scripts/output/comparison/compareScenarios2.R @@ -38,7 +38,7 @@ determineDefaultProfiles <- function(outputDir) { # Start compareScenarios2 either on the cluster or locally. startComp <- function( - outputDirs, + outputdirs, nameCore, profileName, aliases=NULL @@ -64,10 +64,10 @@ startComp <- function( " --mail-type=END,FAIL --time=200", if (!grepl("--mem", slurmConfig)) " --mem=8000", " --wrap=\"Rscript ", script, - " outputDirs=", paste(outputDirs, collapse = ","), - " profileName=", profileName, - " outFileName=", outFileName, - " aliases=", paste(aliases, collapse = ","), + " --outputdirs=", paste(outputdirs, collapse = ","), + " --profileName=", profileName, + " --outFileName=", outFileName, + " --aliases=", paste(aliases, collapse = ","), "\"") cat(clcom, "\n") system(clcom) @@ -115,7 +115,7 @@ nameCore <- paste0(filename_prefix, ifelse(filename_prefix == "", "", "-"), time # Start a job for each profile. for (profileName in profileNames) { startComp( - outputDirs = outputdirs, + outputdirs = outputdirs, nameCore = nameCore, profileName = profileName, aliases = aliases) diff --git a/scripts/output/comparison/policyCosts.R b/scripts/output/comparison/policyCosts.R index f82861b77e..fce38c2f5a 100644 --- a/scripts/output/comparison/policyCosts.R +++ b/scripts/output/comparison/policyCosts.R @@ -202,7 +202,7 @@ if (!exists("source_include")) { # Check that "outputdirs" has an even number of entries. if (length(outputdirs) %% 2 != 0) { message(crayon::red("\nOutputdirs has an uneven number of entries...")) - message("To start again, run: ", crayon::blue("Rscript output.R comp=comparison output=policyCosts"), "\n\n") + message("To start again, run: ", crayon::blue("Rscript output.R --comp=comparison --output=policyCosts"), "\n\n") q() } @@ -244,7 +244,7 @@ if (exists("source_include")) { message(crayon::green("Great!")) } else { message(crayon::red("\nGood-bye!")) - message("To start again, run: ", crayon::blue("Rscript output.R comp=comparison output=policyCosts"), "\n\n") + message("To start again, run: ", crayon::blue("Rscript output.R --comp=comparison --output=policyCosts"), "\n\n") q() } # Get special requests from user diff --git a/scripts/output/comparison/runtimeMAgPIENash.R b/scripts/output/comparison/runtimeMAgPIENash.R index c6d83cb969..e14710190e 100644 --- a/scripts/output/comparison/runtimeMAgPIENash.R +++ b/scripts/output/comparison/runtimeMAgPIENash.R @@ -172,4 +172,4 @@ p <- ggplot(data,aes(x=start, y=run, color=phase)) + ylab("") + xlab("") -ggsave("output/runtime.png", p, width = 16, height = 5) \ No newline at end of file +ggsave("output/runtime.png", p, width = 16, height = 5) diff --git a/scripts/output/export/xlsx_IIASA.R b/scripts/output/export/xlsx_IIASA.R index a15f67465d..936fff463a 100644 --- a/scripts/output/export/xlsx_IIASA.R +++ b/scripts/output/export/xlsx_IIASA.R @@ -30,7 +30,7 @@ mapping <- NULL # file obtained from piamInterfac iiasatemplate <- NULL # provided for each project, can be yaml or xlsx with a column 'Variable' checkSummation <- TRUE # if TRUE, tries to use the one from mapping. Or specify here -# note: you can also pass all these options to output.R, so 'Rscript output.R logFile=mylogfile.txt' works. +# note: you can also pass all these options to output.R, so 'Rscript output.R --logFile=mylogfile.txt' works. lucode2::readArgs("project") diff --git a/scripts/output/single/MAGICC7_AR6.R b/scripts/output/single/MAGICC7_AR6.R index d97d187d52..7425712349 100644 --- a/scripts/output/single/MAGICC7_AR6.R +++ b/scripts/output/single/MAGICC7_AR6.R @@ -31,7 +31,7 @@ library(remindClimateAssessment) if (!exists("source_include")) { # Define arguments that can be read from command line outputdir <- "." - lucode2::readArgs("outputdir", "gdxName", "gdx_ref_name", "gdx_refpolicycost_name") + lucode2::readArgs("outputdir") } runTimes <- c() diff --git a/scripts/output/single/fixOnRef.R b/scripts/output/single/fixOnRef.R index cd0dda24a1..8ebeb69abf 100644 --- a/scripts/output/single/fixOnRef.R +++ b/scripts/output/single/fixOnRef.R @@ -113,7 +113,7 @@ fixOnMif <- function(outputdir) { if (! isTRUE(fixeddata) && isTRUE(envi$cfg$fixOnRefAuto %in% c(TRUE, "TRUE"))) { d <- fixeddata update <- "data from reference run because cfg$fixOnRefAuto=TRUE." - } else if (! isTRUE(fixeddata) && exists("flags") && isTRUE("--interactive" %in% flags)) { + } else if (! isTRUE(fixeddata) && exists("interactive")) { message("\nDo you want to fix that by overwriting ", title, " mif with reference run ", refname, " for t < ", startyear, "?\nType: y/N") if (tolower(gms::getLine()) %in% c("y", "yes")) { diff --git a/scripts/output/single/reportLogit.R b/scripts/output/single/reportLogit.R index 147311ff64..48e3c6edbb 100644 --- a/scripts/output/single/reportLogit.R +++ b/scripts/output/single/reportLogit.R @@ -15,19 +15,16 @@ library(reshape2) if(!exists("source_include")) { #Define arguments that can be read from command line - output_folder <- "dummy" - readArgs("output_folder") - -} else { - output_folder <- outputdir + outputdir <- "dummy" + readArgs("outputdir") } #---------------READ-IN DATA ---------------------------- filename = "Logit_buildings.csv" if (file.exists(filename)) { df = read.csv(filename, stringsAsFactors = F) -}else if (file.exists(file.path(output_folder,filename))) { - df = read.csv(file.path(output_folder,filename), stringsAsFactors = F) +}else if (file.exists(file.path(outputdir,filename))) { + df = read.csv(file.path(outputdir,filename), stringsAsFactors = F) } else { stop("No capital_unit.csv file found - please perform postprocessing first!") diff --git a/scripts/start/run.R b/scripts/start/run.R index 147314bc8b..b5da362d45 100644 --- a/scripts/start/run.R +++ b/scripts/start/run.R @@ -252,18 +252,24 @@ run <- function() { source_include <- TRUE # Postprocessing / Output Generation - output <- cfg$output - outputdir <- cfg$results_folder # make sure the renv used for the run is also used for generating output if (!is.null(renv::project())) { - if (normalizePath(renv::project()) != normalizePath(outputdir)) { - warning("loaded renv=", normalizePath(renv::project()), " and outputdir=", normalizePath(outputdir), " must be equal.") + if (normalizePath(renv::project()) != normalizePath(cfg$results_folder)) { + warning("loaded renv=", normalizePath(renv::project()), " and outputdir=", normalizePath(cfg$results_folder), " must be equal.") } message("Using ", normalizePath(renv::project()), " as renv project") argv <- c(get0("argv"), paste0("--renv=", renv::project())) } + # these arguments are read by output.R to replace the command line interface + # this is not more explicit yet due to renv doubts + passedArgs <- list( + outputdirs = cfg$results_folder, + output = cfg$output, + comp = "single", + test = FALSE + ) sys.source("output.R",envir=new.env()) # get runtime for output timeOutputEnd <- Sys.time() diff --git a/scripts/vs/run_validateScenarios.R b/scripts/vs/run_validateScenarios.R index ca07d277ce..8cffeb3b2b 100644 --- a/scripts/vs/run_validateScenarios.R +++ b/scripts/vs/run_validateScenarios.R @@ -7,12 +7,12 @@ library(piamValidation) -lucode2::readArgs("outputDirs", "validationConfig") +lucode2::readArgs("outputdirs", "validationConfig") # working directory is assumed to be the remind directory -outputDirs <- normalizePath(outputDirs, mustWork = TRUE) -mifPath <- remind2::getMifScenPath(outputDirs, mustWork = TRUE) -histPath <- remind2::getMifHistPath(outputDirs[1], mustWork = TRUE) +outputdirs <- normalizePath(outputdirs, mustWork = TRUE) +mifPath <- remind2::getMifScenPath(outputdirs, mustWork = TRUE) +histPath <- remind2::getMifHistPath(outputdirs[1], mustWork = TRUE) # option 1: HTML validation Report piamValidation::validationReport(c(mifPath, histPath), validationConfig) diff --git a/tests/testthat/test_06-output.R b/tests/testthat/test_06-output.R index af68343af1..4a84ca41e4 100644 --- a/tests/testthat/test_06-output.R +++ b/tests/testthat/test_06-output.R @@ -8,7 +8,7 @@ skipIfFast() skipIfPreviousFailed() test_that("output.R -> single -> reporting works", { - output <- localSystem2("Rscript", c("output.R", "comp=single", "output=reporting", "outputdir=output/testOneRegi", + output <- localSystem2("Rscript", c("output.R", "--comp=single", "--output=reporting", "--outputdirs=output/testOneRegi", "slurmConfig='--qos=priority --mem=8000 --wait --time=120'")) printIfFailed(output) expectSuccessStatus(output) @@ -20,8 +20,8 @@ test_that("output.R -> single -> reporting works", { test_that("output.R -> export -> xlsx_IIASA works", { exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*")) unlink(exportfiles) - output <- localSystem2("Rscript", c("output.R", "project=TESTTHAT", "filename_prefix=TESTTHAT", - "comp=export", "output=xlsx_IIASA", "outputdir=output/testOneRegi")) + output <- localSystem2("Rscript", c("output.R", "--project=TESTTHAT", "--filename_prefix=TESTTHAT", + "--comp=export", "--output=xlsx_IIASA", "--outputdirs=output/testOneRegi")) printIfFailed(output) exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*")) expect_true(sum(grepl("REMIND_TESTTHAT.*xlsx$", exportfiles)) == 1) diff --git a/tutorials/05_AnalysingModelOutputs.md b/tutorials/05_AnalysingModelOutputs.md index 4b2064c9f8..29b30a6c55 100644 --- a/tutorials/05_AnalysingModelOutputs.md +++ b/tutorials/05_AnalysingModelOutputs.md @@ -173,9 +173,9 @@ Another useful script it `validateScenarios`, which performs an automated check You can also specify the parameters in the command line, for example starting a `compareScenario2` run without any prefix as: ``` bash -Rscript output.R comp=comparison filename_prefix= output=compareScenarios2 slurmConfig=priority +Rscript output.R --comp=comparison --filename_prefix= --output=compareScenarios2 --slurmConfig=priority ``` -If you want to compare runs from different REMIND folders, add `remind_dir=.,../otherremind` as a parameter. +If you want to compare runs from different REMIND folders, add `--remind_dir=.,../otherremind` as a parameter. How to create new plots is described in the tutorial [8_Advanced_AnalysingModelOutputs.Rmd](./08_Advanced_AnalysingModelOutputs.Rmd). Another useful and compatible resource for generating plots (e.g. box plots) from REMIND results is UTokyo's *mipplot* R package: https://github.com/UTokyo-mip/mipplot. diff --git a/tutorials/13_Submit_to_IIASA_database.md b/tutorials/13_Submit_to_IIASA_database.md index d6b7c6e52a..57c211c187 100644 --- a/tutorials/13_Submit_to_IIASA_database.md +++ b/tutorials/13_Submit_to_IIASA_database.md @@ -37,12 +37,12 @@ The script requires the inputs as above, expect that it lets you select the `mif You can specify the information above in two ways: Either edit [`xlsx_IIASA.R`](../scripts/output/export/xlsx_IIASA.R) and add a project in a similar way to `NGFS` or `ENGAGE`. You can then start the scripts with: ``` -Rscript output.R comp=export output=xlsx_IIASA project=NGFS +Rscript output.R --comp=export --output=xlsx_IIASA --project=NGFS ``` You do not need to specify `comp` and `output` in the command line, you can just wait to be asked for it. An alternative is to specify everything individually as command-line arguments: ``` -Rscript output.R comp=export output=xlsx_IIASA model="REMIND 3.2" mapping=AR6,AR6_NGFS addToScen=whatever removeFromScen=C_ filename_prefix=test +Rscript output.R --comp=export --output=xlsx_IIASA --model="REMIND 3.2" --mapping=AR6,AR6_NGFS --addToScen=whatever --removeFromScen=C_ --filename_prefix=test ``` All the information printed to you during the run will also be present in the logfile whose path will be told you at the end. diff --git a/tutorials/14_Calculate_policy_costs.md b/tutorials/14_Calculate_policy_costs.md index 7ca9ffd785..3404a0f309 100644 --- a/tutorials/14_Calculate_policy_costs.md +++ b/tutorials/14_Calculate_policy_costs.md @@ -46,7 +46,7 @@ scenarioAndReference=( "${scenarioAndReference[@]/#/output/C_}" ) scenarioAndReference=( "${scenarioAndReference[@]/%/-rem-$remnr}" ) scenarioAndReference=( $( IFS="," ; echo "${scenarioAndReference[*]}") ) -Rscript output.R comp=T output=policyCosts outputdir=$scenarioAndReference +Rscript output.R --comp=T --output=policyCosts --outputdirs=$scenarioAndReference ``` Short version if you have a single base run for all your scenarios and want to automatically start a compareScenario2 and an IIASA export: @@ -64,6 +64,6 @@ Rscript scripts/output/comparison/policyCosts.R outputdirs=$outputstringpc speci outputarraycs=( "${runs[@]/%/-rem-${remnr}}" ) outputarraycs=( "${outputarraycs[@]/#/output/C_}" ) outputstringcs="$(IFS=,; echo "${outputarraycs[*]}")" -Rscript output.R comp=export output=xlsx_IIASA outputdir=$outputstringcs project=NGFS_v4 filename_prefix=NGFS_v4 -Rscript output.R comp=comparison output=compareScenarios2 outputdir=$outputstringcs filename_prefix=NGFS_v4 slurmConfig=priority profileNames=REMIND-MAgPIE +Rscript output.R --comp=export --output=xlsx_IIASA --outputdirs=$outputstringcs --project=NGFS_v4 --filename_prefix=NGFS_v4 +Rscript output.R --comp=comparison --output=compareScenarios2 --outputdirs=$outputstringcs --filename_prefix=NGFS_v4 --slurmConfig=priority --profileNames=REMIND-MAgPIE ```