From a698814e6a531fc83a71570c22466f8a586f47cc Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Mon, 27 Oct 2025 10:15:10 -0400 Subject: [PATCH 1/8] testing export of timestamp variable to prevent restarts --- genpipes/core/scheduler.py | 4 +++- genpipes/pipelines/dnaseq/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/genpipes/core/scheduler.py b/genpipes/core/scheduler.py index 80c4886e1..712988f87 100755 --- a/genpipes/core/scheduler.py +++ b/genpipes/core/scheduler.py @@ -333,7 +333,8 @@ def job2json_project_tracking(self, pipeline, job, job_status): -j \\"{job_name}\\" \\{metrics} -o \\"{json_outfile}\\" \\ -f {status} -export PT_JSON_OUTFILE=\\"{json_outfile}\\" {command_separator} +export PT_JSON_OUTFILE=\\"{json_outfile}\\" +export TIMESTAMP=\\"{timestamp}\\" {command_separator} """.format( job2json_project_tracking_script="genpipes tools job2json_project_tracking", samples=",".join([sample.name for sample in job.samples]), @@ -341,6 +342,7 @@ def job2json_project_tracking(self, pipeline, job, job_status): job_name=job.name, metrics=('\n -m \\"' + ','.join(job.metrics) + '\\" \\') if job.metrics else '', json_outfile=json_outfile, + timestamp=pipeline.timestamp, status=job_status, command_separator="&&" if (job_status=='\\"RUNNING\\"') else "" ) if json_outfile else "" diff --git a/genpipes/pipelines/dnaseq/__init__.py b/genpipes/pipelines/dnaseq/__init__.py index 739125357..1bc61ac60 100755 --- a/genpipes/pipelines/dnaseq/__init__.py +++ b/genpipes/pipelines/dnaseq/__init__.py @@ -2822,7 +2822,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_{self.timestamp}.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIMESTAMP.o") jobs.append( concat_jobs( [ From 65da90d4e93fa7356c619474505491a821ca9441 Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Mon, 27 Oct 2025 10:37:43 -0400 Subject: [PATCH 2/8] testing export of timestamp variable to prevent restarts --- genpipes/core/scheduler.py | 2 +- genpipes/pipelines/dnaseq/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/genpipes/core/scheduler.py b/genpipes/core/scheduler.py index 712988f87..5203d841f 100755 --- a/genpipes/core/scheduler.py +++ b/genpipes/core/scheduler.py @@ -334,7 +334,7 @@ def job2json_project_tracking(self, pipeline, job, job_status): -o \\"{json_outfile}\\" \\ -f {status} export PT_JSON_OUTFILE=\\"{json_outfile}\\" -export TIMESTAMP=\\"{timestamp}\\" {command_separator} +export TIME_STAMP=\\"{timestamp}\\" {command_separator} """.format( job2json_project_tracking_script="genpipes tools job2json_project_tracking", samples=",".join([sample.name for sample in job.samples]), diff --git a/genpipes/pipelines/dnaseq/__init__.py b/genpipes/pipelines/dnaseq/__init__.py index 1bc61ac60..d221d2a47 100755 --- a/genpipes/pipelines/dnaseq/__init__.py +++ b/genpipes/pipelines/dnaseq/__init__.py @@ -2822,7 +2822,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIMESTAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") jobs.append( concat_jobs( [ From d2a2d431ce627e6cfd1e332a80fda9c31075d6dc Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Mon, 27 Oct 2025 10:49:43 -0400 Subject: [PATCH 3/8] testing export of timestamp variable to prevent restarts --- genpipes/pipelines/dnaseq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genpipes/pipelines/dnaseq/__init__.py b/genpipes/pipelines/dnaseq/__init__.py index d221d2a47..8e6bf4627 100755 --- a/genpipes/pipelines/dnaseq/__init__.py +++ b/genpipes/pipelines/dnaseq/__init__.py @@ -2822,7 +2822,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_\$TIME_STAMP.o") jobs.append( concat_jobs( [ From 2757d04f4b2caf9cc6d7d1c03f553c22a4eb3bf9 Mon Sep 17 00:00:00 2001 From: Paul Stretenowich Date: Mon, 27 Oct 2025 12:16:27 -0400 Subject: [PATCH 4/8] Changing timestamp to env var to patch auto restart --- genpipes/pipelines/dnaseq/__init__.py | 4 ++-- genpipes/pipelines/longread_dnaseq/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/genpipes/pipelines/dnaseq/__init__.py b/genpipes/pipelines/dnaseq/__init__.py index 8e6bf4627..a97497aee 100755 --- a/genpipes/pipelines/dnaseq/__init__.py +++ b/genpipes/pipelines/dnaseq/__init__.py @@ -2822,7 +2822,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_\$TIME_STAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") jobs.append( concat_jobs( [ @@ -2960,7 +2960,7 @@ def report_pcgr(self): samples = [tumor_pair.normal, tumor_pair.tumor] if self.project_tracking_json: - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_{self.timestamp}.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") jobs.append( concat_jobs( [ diff --git a/genpipes/pipelines/longread_dnaseq/__init__.py b/genpipes/pipelines/longread_dnaseq/__init__.py index 39efbe482..6dceddf48 100644 --- a/genpipes/pipelines/longread_dnaseq/__init__.py +++ b/genpipes/pipelines/longread_dnaseq/__init__.py @@ -1911,7 +1911,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_{self.timestamp}.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") jobs.append( concat_jobs( [ @@ -2024,7 +2024,7 @@ def report_pcgr(self): samples = [tumor_pair.normal, tumor_pair.tumor] if self.project_tracking_json: - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_{self.timestamp}.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") jobs.append( concat_jobs( [ From 959e861db6a794c67fac5d133be84c055d316d89 Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Tue, 28 Oct 2025 11:01:44 -0400 Subject: [PATCH 5/8] formatting --- genpipes/core/scheduler.py | 2 +- genpipes/pipelines/dnaseq/__init__.py | 4 ++-- genpipes/pipelines/longread_dnaseq/__init__.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/genpipes/core/scheduler.py b/genpipes/core/scheduler.py index 5203d841f..712988f87 100755 --- a/genpipes/core/scheduler.py +++ b/genpipes/core/scheduler.py @@ -334,7 +334,7 @@ def job2json_project_tracking(self, pipeline, job, job_status): -o \\"{json_outfile}\\" \\ -f {status} export PT_JSON_OUTFILE=\\"{json_outfile}\\" -export TIME_STAMP=\\"{timestamp}\\" {command_separator} +export TIMESTAMP=\\"{timestamp}\\" {command_separator} """.format( job2json_project_tracking_script="genpipes tools job2json_project_tracking", samples=",".join([sample.name for sample in job.samples]), diff --git a/genpipes/pipelines/dnaseq/__init__.py b/genpipes/pipelines/dnaseq/__init__.py index a97497aee..73c3ad3e3 100755 --- a/genpipes/pipelines/dnaseq/__init__.py +++ b/genpipes/pipelines/dnaseq/__init__.py @@ -2822,7 +2822,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIMESTAMP.o") jobs.append( concat_jobs( [ @@ -2960,7 +2960,7 @@ def report_pcgr(self): samples = [tumor_pair.normal, tumor_pair.tumor] if self.project_tracking_json: - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIMESTAMP.o") jobs.append( concat_jobs( [ diff --git a/genpipes/pipelines/longread_dnaseq/__init__.py b/genpipes/pipelines/longread_dnaseq/__init__.py index 6dceddf48..9152eea6c 100644 --- a/genpipes/pipelines/longread_dnaseq/__init__.py +++ b/genpipes/pipelines/longread_dnaseq/__init__.py @@ -1911,7 +1911,7 @@ def report_pcgr(self): if self.project_tracking_json: samples = [sample] - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIMESTAMP.o") jobs.append( concat_jobs( [ @@ -2024,7 +2024,7 @@ def report_pcgr(self): samples = [tumor_pair.normal, tumor_pair.tumor] if self.project_tracking_json: - pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIME_STAMP.o") + pcgr_output_file = os.path.join(self.output_dir, "job_output", "report_pcgr", f"{job_name}_$TIMESTAMP.o") jobs.append( concat_jobs( [ From 446cbfd07ee8b119976a8a6635bed4c9ad97a6fb Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Tue, 28 Oct 2025 12:51:19 -0400 Subject: [PATCH 6/8] fix for kallisto adding other options twice, error introduced by Mareike recently --- genpipes/pipelines/rnaseq_light/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genpipes/pipelines/rnaseq_light/__init__.py b/genpipes/pipelines/rnaseq_light/__init__.py index 4c7c86d73..9f4ca8fb7 100644 --- a/genpipes/pipelines/rnaseq_light/__init__.py +++ b/genpipes/pipelines/rnaseq_light/__init__.py @@ -77,7 +77,7 @@ def kallisto(self): transcriptome_file = global_conf.global_get('kallisto', 'transcriptome_idx', param_type="filepath") tx2genes_file = global_conf.global_get('kallisto', 'transcript2genes', param_type="filepath") bootstraps = global_conf.global_get('kallisto', 'bootstraps') - other_param = global_conf.global_get('kallisto', 'other_options', required=False) + other_param = global_conf.global_get('kallisto', 'other_parameters', required=False) jobs = [] From 93eb394090cc05e1c5642ef3a05f3aa93890aab7 Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Tue, 28 Oct 2025 13:35:28 -0400 Subject: [PATCH 7/8] fix input dependencies for cnvkit_batch cna --- genpipes/pipelines/dnaseq/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genpipes/pipelines/dnaseq/__init__.py b/genpipes/pipelines/dnaseq/__init__.py index 73c3ad3e3..edc759426 100755 --- a/genpipes/pipelines/dnaseq/__init__.py +++ b/genpipes/pipelines/dnaseq/__init__.py @@ -3958,7 +3958,7 @@ def cnvkit_batch(self): name=f"cnvkit_batch.cna.{sample_name}", samples=samples, readsets=readsets, - input_dependency=[input_cna, output_cna], + input_dependency=[input_cna], output_dependency=[header, output_cna_body, output_cna] ) ) @@ -4197,7 +4197,7 @@ def cnvkit_batch(self): name=f"cnvkit_batch.cna.{sample_name}", samples=[tumor_pair.normal, tumor_pair.tumor], readsets=[*list(tumor_pair.normal.readsets), *list(tumor_pair.tumor.readsets)], - input_dependency=[input_cna, output_cna], + input_dependency=[input_cna], output_dependency=[header, output_cna_body, output_cna], removable_files=[header, output_cna_body] ) From 917cbbce262dda87a21205cc967ddf3b63f03eae Mon Sep 17 00:00:00 2001 From: Mareike Janiak Date: Tue, 28 Oct 2025 14:00:00 -0400 Subject: [PATCH 8/8] change shebang line in prologue and epilogue to prevent python version conflict --- genpipes/core/epilogue.py | 2 +- genpipes/core/prologue.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/genpipes/core/epilogue.py b/genpipes/core/epilogue.py index 83d9714f7..7e8a4c09c 100755 --- a/genpipes/core/epilogue.py +++ b/genpipes/core/epilogue.py @@ -1,4 +1,4 @@ -#!/cvmfs/soft.mugqic/CentOS6/software/python/Python-3.12.2/bin/python +#!/usr/bin/env python import csv import os diff --git a/genpipes/core/prologue.py b/genpipes/core/prologue.py index 390a5322b..8c39e05dd 100755 --- a/genpipes/core/prologue.py +++ b/genpipes/core/prologue.py @@ -1,4 +1,4 @@ -#!/cvmfs/soft.mugqic/CentOS6/software/python/Python-3.12.2/bin/python +#!/usr/bin/env python import csv import os