Refactor output.R

.gitignore

@@ -42,6 +42,7 @@ doc/
 
 # Ignore "main.lst" files
 main.lst
+/main_*.gms
 
 # Ignore .RData and log files only in the root folder
 /*.RData

CHANGELOG.md

@@ -10,7 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### input data/calibration
 
 ### changed
--
+- **scripts** Harmonize command line arguments for output.R: prefix `--` is now required, change `--outputdir` to `--outputdirs`
+    [[#2413](https://github.com/remindmodel/remind/pull/2413)]
 
 ### added
 -

scripts/cs2/run_compareScenarios2.R

@@ -8,11 +8,11 @@
 library(piamPlotComparison)
 
 if (!exists("source_include")) {
-  lucode2::readArgs("outputDirs", "outFileName", "profileName", "aliases")
+  lucode2::readArgs("outputdirs", "outFileName", "profileName", "aliases")
 }
 
 run_compareScenarios2 <- function(
-  outputDirs,
+  outputdirs,
   outFileName,
   profileName,
   aliases=NULL
@@ -34,15 +34,15 @@ run_compareScenarios2 <- function(
   system(paste0("mkdir ", outFileName))
   outDir <- normalizePath(outFileName, mustWork = TRUE)
 
-  outputDirs <- unique(normalizePath(outputDirs, mustWork = TRUE))
+  outputdirs <- unique(normalizePath(outputdirs, mustWork = TRUE))
 
   if (profileName == "EDGE-Transport") {
-    mifPath <- normalizePath(file.path(outputDirs, "EDGE-T", "Transport.mif"), mustWork = TRUE)
+    mifPath <- normalizePath(file.path(outputdirs, "EDGE-T", "Transport.mif"), mustWork = TRUE)
   } else {
-    mifPath <- remind2::getMifScenPath(outputDirs, mustWork = TRUE)
+    mifPath <- remind2::getMifScenPath(outputdirs, mustWork = TRUE)
   }
-  histPath <- remind2::getMifHistPath(outputDirs[1], mustWork = TRUE)
-  scenConfigPath <- remind2::getCfgScenPath(outputDirs, mustWork = TRUE)
+  histPath <- remind2::getMifHistPath(outputdirs[1], mustWork = TRUE)
+  scenConfigPath <- remind2::getCfgScenPath(outputdirs, mustWork = TRUE)
 
   # predefined arguments
   args <- list(
@@ -94,4 +94,4 @@ run_compareScenarios2 <- function(
   message("Done!\n")
 }
 
-run_compareScenarios2(outputDirs, outFileName, profileName, aliases)
+run_compareScenarios2(outputdirs, outFileName, profileName, aliases)

scripts/output/comparison/compareScenarios2.R

@@ -38,7 +38,7 @@ determineDefaultProfiles <- function(outputDir) {
 
 # Start compareScenarios2 either on the cluster or locally.
 startComp <- function(
-  outputDirs,
+  outputdirs,
   nameCore,
   profileName,
   aliases=NULL
@@ -64,10 +64,10 @@ startComp <- function(
       " --mail-type=END,FAIL --time=200",
       if (!grepl("--mem", slurmConfig)) " --mem=8000",
       " --wrap=\"Rscript ", script,
-      " outputDirs=", paste(outputDirs, collapse = ","),
-      " profileName=", profileName,
-      " outFileName=", outFileName,
-      " aliases=", paste(aliases, collapse = ","),
+      " --outputdirs=", paste(outputdirs, collapse = ","),
+      " --profileName=", profileName,
+      " --outFileName=", outFileName,
+      " --aliases=", paste(aliases, collapse = ","),
       "\"")
     cat(clcom, "\n")
     system(clcom)
@@ -115,7 +115,7 @@ nameCore <- paste0(filename_prefix, ifelse(filename_prefix == "", "", "-"), time
 # Start a job for each profile.
 for (profileName in profileNames) {
   startComp(
-    outputDirs = outputdirs,
+    outputdirs = outputdirs,
     nameCore = nameCore,
     profileName = profileName,
     aliases = aliases)

scripts/output/comparison/policyCosts.R

@@ -202,7 +202,7 @@ if (!exists("source_include")) {
 # Check that "outputdirs" has an even number of entries.
 if (length(outputdirs) %% 2 != 0) {
   message(crayon::red("\nOutputdirs has an uneven number of entries..."))
-  message("To start again, run: ", crayon::blue("Rscript output.R comp=comparison output=policyCosts"), "\n\n")
+  message("To start again, run: ", crayon::blue("Rscript output.R --comp=comparison --output=policyCosts"), "\n\n")
   q()
 }
 
@@ -244,7 +244,7 @@ if (exists("source_include")) {
     message(crayon::green("Great!"))
   } else {
     message(crayon::red("\nGood-bye!"))
-    message("To start again, run: ", crayon::blue("Rscript output.R comp=comparison output=policyCosts"), "\n\n")
+    message("To start again, run: ", crayon::blue("Rscript output.R --comp=comparison --output=policyCosts"), "\n\n")
     q()
   }
   # Get special requests from user

scripts/output/comparison/runtimeMAgPIENash.R

@@ -172,4 +172,4 @@ p <- ggplot(data,aes(x=start, y=run, color=phase)) +
   ylab("") +
   xlab("")
 
-ggsave("output/runtime.png", p, width = 16, height = 5)
+ggsave("output/runtime.png", p, width = 16, height = 5)

scripts/output/export/xlsx_IIASA.R

@@ -30,7 +30,7 @@ mapping <- NULL                                # file obtained from piamInterfac
 iiasatemplate <- NULL                          # provided for each project, can be yaml or xlsx with a column 'Variable'
 checkSummation <- TRUE                         # if TRUE, tries to use the one from mapping. Or specify here
 
-# note: you can also pass all these options to output.R, so 'Rscript output.R logFile=mylogfile.txt' works.
+# note: you can also pass all these options to output.R, so 'Rscript output.R --logFile=mylogfile.txt' works.
 lucode2::readArgs("project")
 
 

scripts/output/single/MAGICC7_AR6.R

@@ -31,7 +31,7 @@ library(remindClimateAssessment)
 if (!exists("source_include")) {
   # Define arguments that can be read from command line
   outputdir <- "."
-  lucode2::readArgs("outputdir", "gdxName", "gdx_ref_name", "gdx_refpolicycost_name")
+  lucode2::readArgs("outputdir")
 }
 
 runTimes <- c()

scripts/output/single/fixOnRef.R

@@ -113,7 +113,7 @@ fixOnMif <- function(outputdir) {
   if (! isTRUE(fixeddata) && isTRUE(envi$cfg$fixOnRefAuto %in% c(TRUE, "TRUE"))) {
     d <- fixeddata
     update <- "data from reference run because cfg$fixOnRefAuto=TRUE."
-  } else if (! isTRUE(fixeddata) && exists("flags") && isTRUE("--interactive" %in% flags)) {
+  } else if (! isTRUE(fixeddata) && exists("interactive")) {
     message("\nDo you want to fix that by overwriting ", title, " mif with reference run ",
             refname, " for t < ", startyear, "?\nType: y/N")
     if (tolower(gms::getLine()) %in% c("y", "yes")) {

scripts/output/single/reportLogit.R

@@ -15,19 +15,16 @@ library(reshape2)
 
 if(!exists("source_include")) {
   #Define arguments that can be read from command line
-  output_folder <- "dummy"
-  readArgs("output_folder")
-
-} else {
-  output_folder <- outputdir
+  outputdir <- "dummy"
+  readArgs("outputdir")
 }
 
 #---------------READ-IN DATA ----------------------------
 filename = "Logit_buildings.csv"
 if (file.exists(filename)) {
   df = read.csv(filename, stringsAsFactors = F)
-}else if (file.exists(file.path(output_folder,filename))) {
-  df = read.csv(file.path(output_folder,filename), stringsAsFactors = F)
+}else if (file.exists(file.path(outputdir,filename))) {
+  df = read.csv(file.path(outputdir,filename), stringsAsFactors = F)
 
 } else {
   stop("No capital_unit.csv file found - please perform postprocessing first!")

scripts/start/run.R

@@ -252,18 +252,24 @@ run <- function() {
   source_include <- TRUE
 
   # Postprocessing / Output Generation
-  output    <- cfg$output
-  outputdir <- cfg$results_folder
 
   # make sure the renv used for the run is also used for generating output
   if (!is.null(renv::project())) {
-    if (normalizePath(renv::project()) != normalizePath(outputdir)) {
-      warning("loaded renv=", normalizePath(renv::project()), " and outputdir=", normalizePath(outputdir), " must be equal.")
+    if (normalizePath(renv::project()) != normalizePath(cfg$results_folder)) {
+      warning("loaded renv=", normalizePath(renv::project()), " and outputdir=", normalizePath(cfg$results_folder), " must be equal.")
     }
     message("Using ", normalizePath(renv::project()), " as renv project")
     argv <- c(get0("argv"), paste0("--renv=", renv::project()))
   }
 
+  # these arguments are read by output.R to replace the command line interface
+  # this is not more explicit yet due to renv doubts
+  passedArgs <- list(
+    outputdirs = cfg$results_folder,
+    output    = cfg$output,
+    comp      = "single",
+    test      = FALSE
+  )
   sys.source("output.R",envir=new.env())
   # get runtime for output
   timeOutputEnd <- Sys.time()

scripts/vs/run_validateScenarios.R

@@ -7,12 +7,12 @@
 
 library(piamValidation)
 
-lucode2::readArgs("outputDirs", "validationConfig")
+lucode2::readArgs("outputdirs", "validationConfig")
 
 # working directory is assumed to be the remind directory
-outputDirs <- normalizePath(outputDirs, mustWork = TRUE)
-mifPath <- remind2::getMifScenPath(outputDirs, mustWork = TRUE)
-histPath <- remind2::getMifHistPath(outputDirs[1], mustWork = TRUE)
+outputdirs <- normalizePath(outputdirs, mustWork = TRUE)
+mifPath <- remind2::getMifScenPath(outputdirs, mustWork = TRUE)
+histPath <- remind2::getMifHistPath(outputdirs[1], mustWork = TRUE)
 
 # option 1: HTML validation Report
 piamValidation::validationReport(c(mifPath, histPath), validationConfig)

tests/testthat/test_06-output.R

@@ -8,7 +8,7 @@ skipIfFast()
 skipIfPreviousFailed()
 
 test_that("output.R -> single -> reporting works", {
-  output <- localSystem2("Rscript", c("output.R", "comp=single", "output=reporting", "outputdir=output/testOneRegi",
+  output <- localSystem2("Rscript", c("output.R", "--comp=single", "--output=reporting", "--outputdirs=output/testOneRegi",
                                       "slurmConfig='--qos=priority --mem=8000 --wait --time=120'"))
   printIfFailed(output)
   expectSuccessStatus(output)
@@ -20,8 +20,8 @@ test_that("output.R -> single -> reporting works", {
 test_that("output.R -> export -> xlsx_IIASA works", {
   exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*"))
   unlink(exportfiles)
-  output <- localSystem2("Rscript", c("output.R", "project=TESTTHAT", "filename_prefix=TESTTHAT",
-                                      "comp=export", "output=xlsx_IIASA", "outputdir=output/testOneRegi"))
+  output <- localSystem2("Rscript", c("output.R", "--project=TESTTHAT", "--filename_prefix=TESTTHAT",
+                                      "--comp=export", "--output=xlsx_IIASA", "--outputdirs=output/testOneRegi"))
   printIfFailed(output)
   exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*"))
   expect_true(sum(grepl("REMIND_TESTTHAT.*xlsx$", exportfiles)) == 1)

tutorials/05_AnalysingModelOutputs.md

@@ -173,9 +173,9 @@ Another useful script it `validateScenarios`, which performs an automated check
 You can also specify the parameters in the command line, for example starting a `compareScenario2` run without any prefix as:
 
 ``` bash
-Rscript output.R comp=comparison filename_prefix= output=compareScenarios2 slurmConfig=priority
+Rscript output.R --comp=comparison --filename_prefix= --output=compareScenarios2 --slurmConfig=priority
 ```
-If you want to compare runs from different REMIND folders, add `remind_dir=.,../otherremind` as a parameter.
+If you want to compare runs from different REMIND folders, add `--remind_dir=.,../otherremind` as a parameter.
 
 How to create new plots is described in the tutorial [8_Advanced_AnalysingModelOutputs.Rmd](./08_Advanced_AnalysingModelOutputs.Rmd).
 Another useful and compatible resource for generating plots (e.g. box plots) from REMIND results is UTokyo's *mipplot* R package: https://github.com/UTokyo-mip/mipplot.

tutorials/13_Submit_to_IIASA_database.md

@@ -37,12 +37,12 @@ The script requires the inputs as above, expect that it lets you select the `mif
 
 You can specify the information above in two ways: Either edit [`xlsx_IIASA.R`](../scripts/output/export/xlsx_IIASA.R) and add a project in a similar way to `NGFS` or `ENGAGE`. You can then start the scripts with:
 ```
-Rscript output.R comp=export output=xlsx_IIASA project=NGFS
+Rscript output.R --comp=export --output=xlsx_IIASA --project=NGFS
 ```
 You do not need to specify `comp` and `output` in the command line, you can just wait to be asked for it.
 An alternative is to specify everything individually as command-line arguments:
 ```
-Rscript output.R comp=export output=xlsx_IIASA model="REMIND 3.2" mapping=AR6,AR6_NGFS addToScen=whatever removeFromScen=C_ filename_prefix=test
+Rscript output.R --comp=export --output=xlsx_IIASA --model="REMIND 3.2" --mapping=AR6,AR6_NGFS --addToScen=whatever --removeFromScen=C_ --filename_prefix=test
 ```
 All the information printed to you during the run will also be present in the logfile whose path will be told you at the end.
 

tutorials/14_Calculate_policy_costs.md

@@ -46,7 +46,7 @@ scenarioAndReference=( "${scenarioAndReference[@]/#/output/C_}" )
 scenarioAndReference=( "${scenarioAndReference[@]/%/-rem-$remnr}" )
 scenarioAndReference=( $( IFS="," ; echo "${scenarioAndReference[*]}") )
 
-Rscript output.R comp=T output=policyCosts outputdir=$scenarioAndReference
+Rscript output.R --comp=T --output=policyCosts --outputdirs=$scenarioAndReference
 ```
 
 Short version if you have a single base run for all your scenarios and want to automatically start a compareScenario2 and an IIASA export:
@@ -64,6 +64,6 @@ Rscript scripts/output/comparison/policyCosts.R outputdirs=$outputstringpc speci
 outputarraycs=( "${runs[@]/%/-rem-${remnr}}" )
 outputarraycs=( "${outputarraycs[@]/#/output/C_}" )
 outputstringcs="$(IFS=,; echo "${outputarraycs[*]}")"
-Rscript output.R comp=export output=xlsx_IIASA outputdir=$outputstringcs project=NGFS_v4 filename_prefix=NGFS_v4
-Rscript output.R comp=comparison output=compareScenarios2 outputdir=$outputstringcs filename_prefix=NGFS_v4 slurmConfig=priority profileNames=REMIND-MAgPIE
+Rscript output.R --comp=export --output=xlsx_IIASA --outputdirs=$outputstringcs --project=NGFS_v4 --filename_prefix=NGFS_v4
+Rscript output.R --comp=comparison --output=compareScenarios2 --outputdirs=$outputstringcs --filename_prefix=NGFS_v4 --slurmConfig=priority --profileNames=REMIND-MAgPIE
 ```