diff --git a/.github/workflows/sprocket-test.yaml b/.github/workflows/sprocket-test.yaml index 5800f6887..9049815b7 100644 --- a/.github/workflows/sprocket-test.yaml +++ b/.github/workflows/sprocket-test.yaml @@ -38,4 +38,4 @@ jobs: env: RUNNER: ${{ matrix.runner }} run: | - sprocket dev test --filter-tag reference --filter-tag slow --filter-tag high_mem ${{ matrix.file }} \ No newline at end of file + sprocket dev test --exclude-tag reference --exclude-tag slow --exclude-tag high_mem ${{ matrix.file }} \ No newline at end of file diff --git a/data_structures/flag_filter.wdl b/data_structures/flag_filter.wdl index dcbf73e8f..ece629955 100644 --- a/data_structures/flag_filter.wdl +++ b/data_structures/flag_filter.wdl @@ -83,6 +83,8 @@ task validate_string_is_12bit_int { } command <<< + set -euo pipefail + if [[ "~{number}" =~ ^[1-9][0-9]*$ ]]; then # number is in decimal if [ "~{number}" -lt 4096 ]; then diff --git a/data_structures/read_group.wdl b/data_structures/read_group.wdl index bde09af24..d2c0d774c 100644 --- a/data_structures/read_group.wdl +++ b/data_structures/read_group.wdl @@ -127,6 +127,8 @@ task get_read_groups { Int disk_size_gb = ceil(bam_size) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + python3 /scripts/data_structures/get_read_groups.py \ "~{bam}" \ read_groups.json @@ -181,6 +183,8 @@ task validate_read_group { ] command <<< + set -euo pipefail + exit_code=0 if ~{restrictive}; then if [[ "~{read_group.ID}" =~ ~{restrictive_pattern} ]] @@ -396,6 +400,8 @@ task inner_read_group_to_string { else " " command <<< + set -euo pipefail + if ~{format_as_sam_record}; then echo -n "@RG~{delimiter}" > out.txt fi diff --git a/scripts/data_structures/get_read_groups.py b/scripts/data_structures/get_read_groups.py index 9e164a025..6bb0d0daf 100644 --- a/scripts/data_structures/get_read_groups.py +++ b/scripts/data_structures/get_read_groups.py @@ -1,19 +1,19 @@ -import pysam import json +import pysam + def main(bam_path, outfile_path): sam = pysam.AlignmentFile(bam_path, "rb") - out_file = open(outfile_path, "w") - header = sam.header.to_dict()["RG"] - modified_header = [] - for read_group in sorted(header, key=lambda d: d["ID"]): - modified_header.append( - {k: v.upper() if k == "PL" else v for k, v in read_group.items()} - ) - json.dump(modified_header, out_file) - out_file.close() + with open(outfile_path, "w") as out_file: + header = sam.header.to_dict()["RG"] + modified_header = [] + for read_group in sorted(header, key=lambda d: d["ID"]): + modified_header.append( + {k: v.upper() if k == "PL" else v for k, v in read_group.items()} + ) + json.dump(modified_header, out_file) if __name__ == "__main__": diff --git a/scripts/htseq/calc_tpm.py b/scripts/htseq/calc_tpm.py index 5798dfbb6..5e0f8ee2d 100644 --- a/scripts/htseq/calc_tpm.py +++ b/scripts/htseq/calc_tpm.py @@ -1,35 +1,32 @@ def main(counts_name_path, feature_lengths_path, outfile_path, has_header): - counts_file = open(counts_name_path, "r") - counts = {} - if has_header: - counts_file.readline() - for line in counts_file: - gene, count = line.split("\t") - if gene[0:2] == "__": - break - counts[gene.strip()] = int(count.strip()) - counts_file.close() + with open(counts_name_path, "r") as counts_file: + counts = {} + if has_header: + counts_file.readline() + for line in counts_file: + gene, count = line.split("\t") + if gene[0:2] == "__": + break + counts[gene.strip()] = int(count.strip()) - lengths_file = open(feature_lengths_path, "r") - rpks = {} # Reads Per Kilobase - tot_rpk = 0 - lengths_file.readline() # discard header - for line in lengths_file: - gene, length = line.split("\t") - rpk = counts[gene.strip()] / int(length.strip()) * 1000 - tot_rpk += rpk - rpks[gene.strip()] = rpk - lengths_file.close() + with open(feature_lengths_path, "r") as lengths_file: + rpks = {} # Reads Per Kilobase + tot_rpk = 0 + lengths_file.readline() # discard header + for line in lengths_file: + gene, length = line.split("\t") + rpk = counts[gene.strip()] / int(length.strip()) * 1000 + tot_rpk += rpk + rpks[gene.strip()] = rpk scaling_factor = tot_rpk / 1000000 sample_name = ".".join(outfile_path.split(".")[:-2]) # assumed to end in `.TPM.txt` - outfile = open(outfile_path, "w") - print(f"feature\t{sample_name}", file=outfile) - for gene, rpk in sorted(rpks.items()): - tpm = rpk / scaling_factor - print(f"{gene}\t{tpm:.3f}", file=outfile) - outfile.close() + with open(outfile_path, "w") as outfile: + print(f"feature\t{sample_name}", file=outfile) + for gene, rpk in sorted(rpks.items()): + tpm = rpk / scaling_factor + print(f"{gene}\t{tpm:.3f}", file=outfile) if __name__ == "__main__": diff --git a/scripts/methylation/combine.py b/scripts/methylation/combine.py index 06cc8d46f..cd960e983 100644 --- a/scripts/methylation/combine.py +++ b/scripts/methylation/combine.py @@ -1,4 +1,5 @@ import argparse + import pandas as pd diff --git a/scripts/methylation/filter.py b/scripts/methylation/filter.py index de3aaf396..aeeed3b34 100644 --- a/scripts/methylation/filter.py +++ b/scripts/methylation/filter.py @@ -1,7 +1,8 @@ +import argparse import csv -import pandas as pd + import numpy as np -import argparse +import pandas as pd def get_args(): diff --git a/scripts/methylation/generate_umap.py b/scripts/methylation/generate_umap.py index 0b4bb6abe..31539fecf 100644 --- a/scripts/methylation/generate_umap.py +++ b/scripts/methylation/generate_umap.py @@ -1,6 +1,7 @@ +import argparse + import pandas as pd import umap -import argparse def get_args(): diff --git a/scripts/methylation/plot_umap.py b/scripts/methylation/plot_umap.py index 4e30f8d33..e8d6add1d 100644 --- a/scripts/methylation/plot_umap.py +++ b/scripts/methylation/plot_umap.py @@ -1,7 +1,8 @@ -import pandas as pd -import matplotlib.pyplot as plt import argparse +import matplotlib.pyplot as plt +import pandas as pd + def get_args(): parser = argparse.ArgumentParser(description="Plot UMAP coordinates.") diff --git a/scripts/util/calc_feature_lengths.py b/scripts/util/calc_feature_lengths.py index 0b17c3a11..07e39db27 100644 --- a/scripts/util/calc_feature_lengths.py +++ b/scripts/util/calc_feature_lengths.py @@ -1,14 +1,15 @@ +from collections import defaultdict + import gtfparse import numpy as np -from collections import defaultdict def main(gtf_path, outfile_path, id_attr): gtf = gtfparse.read_gtf(gtf_path) only_exons = gtf[gtf["feature"] == "exon"] - exon_starts = defaultdict(lambda: []) - exon_ends = defaultdict(lambda: []) + exon_starts = defaultdict(list) + exon_ends = defaultdict(list) gene_start_offset = {} gene_end_offset = {} gene_exon_intersection = {} @@ -43,16 +44,14 @@ def main(gtf_path, outfile_path, id_attr): - gene_start_offset[feature_id] ] = True - outfile = open(outfile_path, "w") - print("feature\tlength", file=outfile) - for gene, exonic_intersection in sorted(gene_exon_intersection.items()): - # np.count_nonzero() is faster than sum - # np.count_nonzero() evaluates the "truthfulness" of - # of all elements (by calling their '.__bool__()' method) - length = np.count_nonzero(exonic_intersection) - print(f"{gene}\t{length}", file=outfile) - - outfile.close() + with open(outfile_path, "w") as outfile: + print("feature\tlength", file=outfile) + for gene, exonic_intersection in sorted(gene_exon_intersection.items()): + # np.count_nonzero() is faster than sum + # np.count_nonzero() evaluates the "truthfulness" of + # of all elements (by calling their '.__bool__()' method) + length = np.count_nonzero(exonic_intersection) + print(f"{gene}\t{length}", file=outfile) if __name__ == "__main__": diff --git a/scripts/util/calc_global_phred_scores.py b/scripts/util/calc_global_phred_scores.py index 60825cdcb..f2558ac07 100644 --- a/scripts/util/calc_global_phred_scores.py +++ b/scripts/util/calc_global_phred_scores.py @@ -84,135 +84,136 @@ def main(bam_path, prefix, fast_mode): middle_mapped_quals[middle_score] += 1 last_mapped_quals[last_score] += 1 - outfile = open(prefix + ".global_PHRED_scores.tsv", "w") - - # print header - header = ["sample"] - if not fast_mode: + with open(prefix + ".global_PHRED_scores.tsv", "w") as outfile: + # print header + header = ["sample"] + if not fast_mode: + header += [ + "total average", + "total median", + "total stdev", + "mapped average", + "mapped median", + "mapped stdev", + "unmapped average", + "unmapped median", + "unmapped stdev", + ] header += [ - "total average", - "total median", - "total stdev", - "mapped average", - "mapped median", - "mapped stdev", - "unmapped average", - "unmapped median", - "unmapped stdev", + "first position total average", + "first position total median", + "first position total stdev", + "first position mapped average", + "first position mapped median", + "first position mapped stdev", + "first position unmapped average", + "first position unmapped median", + "first position unmapped stdev", + "middle position total average", + "middle position total median", + "middle position total stdev", + "middle position mapped average", + "middle position mapped median", + "middle position mapped stdev", + "middle position unmapped average", + "middle position unmapped median", + "middle position unmapped stdev", + "last position total average", + "last position total median", + "last position total stdev", + "last position mapped average", + "last position mapped median", + "last position mapped stdev", + "last position unmapped average", + "last position unmapped median", + "last position unmapped stdev", ] - header += [ - "first position total average", - "first position total median", - "first position total stdev", - "first position mapped average", - "first position mapped median", - "first position mapped stdev", - "first position unmapped average", - "first position unmapped median", - "first position unmapped stdev", - "middle position total average", - "middle position total median", - "middle position total stdev", - "middle position mapped average", - "middle position mapped median", - "middle position mapped stdev", - "middle position unmapped average", - "middle position unmapped median", - "middle position unmapped stdev", - "last position total average", - "last position total median", - "last position total stdev", - "last position mapped average", - "last position mapped median", - "last position mapped stdev", - "last position unmapped average", - "last position unmapped median", - "last position unmapped stdev", - ] - print( - "\t".join(header), - file=outfile, - ) - print(prefix, file=outfile, end="\t") - - if not fast_mode: - tot_avg, tot_median, tot_stdev = stats_from_dict(tot_quals) - print(f"{tot_avg}", file=outfile, end="\t") - print(f"{tot_median}", file=outfile, end="\t") - print(f"{tot_stdev}", file=outfile, end="\t") - - mapped_avg, mapped_median, mapped_stdev = stats_from_dict(mapped_quals) - print(f"{mapped_avg}", file=outfile, end="\t") - print(f"{mapped_median}", file=outfile, end="\t") - print(f"{mapped_stdev}", file=outfile, end="\t") - - unmapped_avg, unmapped_median, unmapped_stdev = stats_from_dict(unmapped_quals) - print(f"{unmapped_avg}", file=outfile, end="\t") - print(f"{unmapped_median}", file=outfile, end="\t") - print(f"{unmapped_stdev}", file=outfile, end="\t") - - first_tot_avg, first_tot_median, first_tot_stdev = stats_from_dict(first_tot_quals) - print(f"{first_tot_avg}", file=outfile, end="\t") - print(f"{first_tot_median}", file=outfile, end="\t") - print(f"{first_tot_stdev}", file=outfile, end="\t") - - first_mapped_avg, first_mapped_median, first_mapped_stdev = stats_from_dict( - first_mapped_quals - ) - print(f"{first_mapped_avg}", file=outfile, end="\t") - print(f"{first_mapped_median}", file=outfile, end="\t") - print(f"{first_mapped_stdev}", file=outfile, end="\t") - - first_unmapped_avg, first_unmapped_median, first_unmapped_stdev = stats_from_dict( - first_unmapped_quals - ) - print(f"{first_unmapped_avg}", file=outfile, end="\t") - print(f"{first_unmapped_median}", file=outfile, end="\t") - print(f"{first_unmapped_stdev}", file=outfile, end="\t") - - middle_tot_avg, middle_tot_median, middle_tot_stdev = stats_from_dict( - middle_tot_quals - ) - print(f"{middle_tot_avg}", file=outfile, end="\t") - print(f"{middle_tot_median}", file=outfile, end="\t") - print(f"{middle_tot_stdev}", file=outfile, end="\t") - - middle_mapped_avg, middle_mapped_median, middle_mapped_stdev = stats_from_dict( - middle_mapped_quals - ) - print(f"{middle_mapped_avg}", file=outfile, end="\t") - print(f"{middle_mapped_median}", file=outfile, end="\t") - print(f"{middle_mapped_stdev}", file=outfile, end="\t") - - ( - middle_unmapped_avg, - middle_unmapped_median, - middle_unmapped_stdev, - ) = stats_from_dict(middle_unmapped_quals) - print(f"{middle_unmapped_avg}", file=outfile, end="\t") - print(f"{middle_unmapped_median}", file=outfile, end="\t") - print(f"{middle_unmapped_stdev}", file=outfile, end="\t") - - last_tot_avg, last_tot_median, last_tot_stdev = stats_from_dict(last_tot_quals) - print(f"{last_tot_avg}", file=outfile, end="\t") - print(f"{last_tot_median}", file=outfile, end="\t") - print(f"{last_tot_stdev}", file=outfile, end="\t") - - last_mapped_avg, last_mapped_median, last_mapped_stdev = stats_from_dict( - last_mapped_quals - ) - print(f"{last_mapped_avg}", file=outfile, end="\t") - print(f"{last_mapped_median}", file=outfile, end="\t") - print(f"{last_mapped_stdev}", file=outfile, end="\t") + print( + "\t".join(header), + file=outfile, + ) + print(prefix, file=outfile, end="\t") - last_unmapped_avg, last_unmapped_median, last_unmapped_stdev = stats_from_dict( - last_unmapped_quals - ) - print(f"{last_unmapped_avg}", file=outfile, end="\t") - print(f"{last_unmapped_median}", file=outfile, end="\t") - print(f"{last_unmapped_stdev}", file=outfile) # end="\n" - - outfile.close() + if not fast_mode: + tot_avg, tot_median, tot_stdev = stats_from_dict(tot_quals) + print(f"{tot_avg}", file=outfile, end="\t") + print(f"{tot_median}", file=outfile, end="\t") + print(f"{tot_stdev}", file=outfile, end="\t") + + mapped_avg, mapped_median, mapped_stdev = stats_from_dict(mapped_quals) + print(f"{mapped_avg}", file=outfile, end="\t") + print(f"{mapped_median}", file=outfile, end="\t") + print(f"{mapped_stdev}", file=outfile, end="\t") + + unmapped_avg, unmapped_median, unmapped_stdev = stats_from_dict( + unmapped_quals + ) + print(f"{unmapped_avg}", file=outfile, end="\t") + print(f"{unmapped_median}", file=outfile, end="\t") + print(f"{unmapped_stdev}", file=outfile, end="\t") + + first_tot_avg, first_tot_median, first_tot_stdev = stats_from_dict( + first_tot_quals + ) + print(f"{first_tot_avg}", file=outfile, end="\t") + print(f"{first_tot_median}", file=outfile, end="\t") + print(f"{first_tot_stdev}", file=outfile, end="\t") + + first_mapped_avg, first_mapped_median, first_mapped_stdev = stats_from_dict( + first_mapped_quals + ) + print(f"{first_mapped_avg}", file=outfile, end="\t") + print(f"{first_mapped_median}", file=outfile, end="\t") + print(f"{first_mapped_stdev}", file=outfile, end="\t") + + first_unmapped_avg, first_unmapped_median, first_unmapped_stdev = ( + stats_from_dict(first_unmapped_quals) + ) + print(f"{first_unmapped_avg}", file=outfile, end="\t") + print(f"{first_unmapped_median}", file=outfile, end="\t") + print(f"{first_unmapped_stdev}", file=outfile, end="\t") + + middle_tot_avg, middle_tot_median, middle_tot_stdev = stats_from_dict( + middle_tot_quals + ) + print(f"{middle_tot_avg}", file=outfile, end="\t") + print(f"{middle_tot_median}", file=outfile, end="\t") + print(f"{middle_tot_stdev}", file=outfile, end="\t") + + middle_mapped_avg, middle_mapped_median, middle_mapped_stdev = stats_from_dict( + middle_mapped_quals + ) + print(f"{middle_mapped_avg}", file=outfile, end="\t") + print(f"{middle_mapped_median}", file=outfile, end="\t") + print(f"{middle_mapped_stdev}", file=outfile, end="\t") + + ( + middle_unmapped_avg, + middle_unmapped_median, + middle_unmapped_stdev, + ) = stats_from_dict(middle_unmapped_quals) + print(f"{middle_unmapped_avg}", file=outfile, end="\t") + print(f"{middle_unmapped_median}", file=outfile, end="\t") + print(f"{middle_unmapped_stdev}", file=outfile, end="\t") + + last_tot_avg, last_tot_median, last_tot_stdev = stats_from_dict(last_tot_quals) + print(f"{last_tot_avg}", file=outfile, end="\t") + print(f"{last_tot_median}", file=outfile, end="\t") + print(f"{last_tot_stdev}", file=outfile, end="\t") + + last_mapped_avg, last_mapped_median, last_mapped_stdev = stats_from_dict( + last_mapped_quals + ) + print(f"{last_mapped_avg}", file=outfile, end="\t") + print(f"{last_mapped_median}", file=outfile, end="\t") + print(f"{last_mapped_stdev}", file=outfile, end="\t") + + last_unmapped_avg, last_unmapped_median, last_unmapped_stdev = stats_from_dict( + last_unmapped_quals + ) + print(f"{last_unmapped_avg}", file=outfile, end="\t") + print(f"{last_unmapped_median}", file=outfile, end="\t") + print(f"{last_unmapped_stdev}", file=outfile) # end="\n" if __name__ == "__main__": diff --git a/scripts/util/check_FQs_and_RGs.py b/scripts/util/check_FQs_and_RGs.py index c7be88610..d5df10d9f 100644 --- a/scripts/util/check_FQs_and_RGs.py +++ b/scripts/util/check_FQs_and_RGs.py @@ -15,13 +15,12 @@ import argparse import os -from typing import List def validate( - read1_fastqs: List[str], - read2_fastqs: List[str], - rgids: List[str], + read1_fastqs: list[str], + read2_fastqs: list[str], + rgids: list[str], ) -> None: """Ensure that each FASTQ basename contains the expected RG ID and that each ID is unique. @@ -70,11 +69,10 @@ def validate( for fq in args.read_two_fastqs.split(",") ] - if read2_fastqs != []: - if len(read1_fastqs) != len(read2_fastqs): - raise SystemExit( - "Must have the same number of read one FASTQs as read two FASTQs" - ) + if read2_fastqs != [] and len(read1_fastqs) != len(read2_fastqs): + raise SystemExit( + "Must have the same number of read one FASTQs as read two FASTQs" + ) rg_records = [rg.strip().strip("'") for rg in args.read_groups.split(",")] diff --git a/tools/arriba.wdl b/tools/arriba.wdl index e5a0506bb..ee0de3c74 100644 --- a/tools/arriba.wdl +++ b/tools/arriba.wdl @@ -205,6 +205,8 @@ task arriba { Int memory_gb = bam_size_gb + modify_memory_gb command <<< + set -euo pipefail + arriba \ -x "~{bam}" \ ~{"-c '" + chimeric_sam + "'"} \ @@ -359,6 +361,8 @@ task arriba_extract_fusion_supporting_alignments { Int disk_size_gb = ceil(input_size_gb) + 5 + modify_disk_size_gb command <<< + set -euo pipefail + extract_fusion-supporting_alignments.sh \ "~{fusions}" \ "~{bam}" \ diff --git a/tools/fq.wdl b/tools/fq.wdl index e2d5c566a..2177d55c9 100755 --- a/tools/fq.wdl +++ b/tools/fq.wdl @@ -72,6 +72,8 @@ task fqlint { Int disk_size_gb = ceil((read1_size + read2_size) * 2) + modify_disk_size_gb command <<< + set -euo pipefail + fq lint \ ~{sep(" ", prefix("--disable-validator ", squote(disable_validator_codes)))} \ --single-read-validation-level "~{single_read_validation_level}" \ @@ -148,6 +150,8 @@ task subsample { String r2_dst = prefix + ".R2.subsampled.fastq.gz" command <<< + set -euo pipefail + # shellcheck disable=SC2086 fq subsample \ ~{probability_arg} \ diff --git a/tools/gatk4.wdl b/tools/gatk4.wdl index dfc858605..6181bfead 100644 --- a/tools/gatk4.wdl +++ b/tools/gatk4.wdl @@ -118,10 +118,12 @@ task base_recalibrator { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + # shellcheck disable=SC2102 gatk \ --java-options \ - "-XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10 -Xms4000m -Xmx~{java_heap_size}g" \ + "-XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10 -Xms4000m -Xmx~{java_heap_size}g -XX:-UseContainerSupport" \ BaseRecalibratorSpark \ -R "~{fasta}" \ -I "~{bam}" \ @@ -189,7 +191,7 @@ task apply_bqsr { # shellcheck disable=SC2102 gatk \ --java-options \ - "-XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10 -Xms3000m -Xmx~{java_heap_size}g" \ + "-XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10 -Xms3000m -Xmx~{java_heap_size}g -XX:-UseContainerSupport" \ ApplyBQSRSpark \ --spark-master local[~{ncpu}] \ -I "~{bam}" \ @@ -271,6 +273,8 @@ task haplotype_caller { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + gatk \ --java-options \ "-Xms6000m -Xmx~{java_heap_size}g -XX:GCTimeLimit=50 -XX:GCHeapFreeLimit=10" \ @@ -356,6 +360,8 @@ task variant_filtration { Int disk_size_gb = ceil(size(vcf, "GB") * 2) + 30 + modify_disk_size_gb command <<< + set -euo pipefail + gatk VariantFiltration \ --R "~{fasta}" \ --V "~{vcf}" \ @@ -468,7 +474,7 @@ task mark_duplicates_spark { # shellcheck disable=SC2102 gatk MarkDuplicatesSpark \ - --java-options "-Xmx~{java_heap_size}g" \ + --java-options "-Xmx~{java_heap_size}g -XX:-UseContainerSupport" \ -I "~{bam}" \ -M "~{prefix}.metrics.txt" \ -O "~{if create_bam diff --git a/tools/htseq.wdl b/tools/htseq.wdl index f81adf188..df7799d5d 100755 --- a/tools/htseq.wdl +++ b/tools/htseq.wdl @@ -187,6 +187,8 @@ task calc_tpm { String outfile_name = prefix + ".TPM.txt" command <<< + set -euo pipefail + python3 /scripts/htseq/calc_tpm.py \ "~{counts}" \ "~{feature_lengths}" \ diff --git a/tools/md5sum.wdl b/tools/md5sum.wdl index 47f07e21b..141811058 100755 --- a/tools/md5sum.wdl +++ b/tools/md5sum.wdl @@ -25,6 +25,8 @@ task compute_checksum { String outfile_name = basename(file) + ".md5" command <<< + set -euo pipefail + md5sum "~{file}" > "~{outfile_name}" >>> diff --git a/tools/ngsderive.wdl b/tools/ngsderive.wdl index f7856be9a..97ee53d8b 100644 --- a/tools/ngsderive.wdl +++ b/tools/ngsderive.wdl @@ -239,6 +239,8 @@ task encoding { Int disk_size_gb = ceil(files_size) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + ngsderive encoding --verbose \ -n ~{num_reads} \ ~{sep(" ", squote(ngs_files))} \ @@ -406,6 +408,8 @@ task endedness { Int disk_size_gb = ceil(bam_size) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + ngsderive endedness --verbose \ ~{if lenient then "--lenient" diff --git a/tools/picard.wdl b/tools/picard.wdl index 870c566c2..af690da08 100755 --- a/tools/picard.wdl +++ b/tools/picard.wdl @@ -543,6 +543,8 @@ task collect_wgs_metrics { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + picard -Xmx~{java_heap_size}g CollectWgsMetrics \ -I "~{bam}" \ -R "~{reference_fasta}" \ @@ -606,6 +608,8 @@ task collect_alignment_summary_metrics { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + picard -Xmx~{java_heap_size}g CollectAlignmentSummaryMetrics \ -I "~{bam}" \ --VALIDATION_STRINGENCY "~{validation_stringency}" \ @@ -674,6 +678,8 @@ task collect_gc_bias_metrics { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + picard -Xmx~{java_heap_size}g CollectGcBiasMetrics \ -I "~{bam}" \ -R "~{reference_fasta}" \ @@ -740,6 +746,8 @@ task collect_insert_size_metrics { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + picard -Xmx~{java_heap_size}g CollectInsertSizeMetrics \ -I "~{bam}" \ --VALIDATION_STRINGENCY "~{validation_stringency}" \ @@ -799,6 +807,8 @@ task quality_score_distribution { Int java_heap_size = ceil(memory_gb * 0.9) command <<< + set -euo pipefail + picard -Xmx~{java_heap_size}g QualityScoreDistribution \ --VALIDATION_STRINGENCY "~{validation_stringency}" \ -I "~{bam}" \ @@ -846,6 +856,8 @@ task merge_vcfs { Int disk_size_gb = ceil(size(vcfs, "GB") * 2) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + picard -Xms2000m \ MergeVcfs \ ~{sep(" ", prefix("--INPUT ", squote(vcfs)))} \ diff --git a/tools/sambamba.wdl b/tools/sambamba.wdl index f5e0eced0..8ecdc613b 100644 --- a/tools/sambamba.wdl +++ b/tools/sambamba.wdl @@ -209,6 +209,8 @@ task markdup { Int disk_size_gb = ceil((bam_size * 2) + 10) + modify_disk_size_gb command <<< + set -euo pipefail + sambamba markdup \ --nthreads ~{ncpu} \ ~{if remove_duplicates diff --git a/tools/samtools.wdl b/tools/samtools.wdl index d0ad37742..9f176cd51 100755 --- a/tools/samtools.wdl +++ b/tools/samtools.wdl @@ -24,6 +24,8 @@ task quickcheck { Int disk_size_gb = ceil(bam_size) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + samtools quickcheck "~{bam}" >>> diff --git a/tools/test/gatk4.yaml b/tools/test/gatk4.yaml index bfb256ae6..7fb370451 100644 --- a/tools/test/gatk4.yaml +++ b/tools/test/gatk4.yaml @@ -9,6 +9,8 @@ apply_bqsr: - bams/test_rnaseq_variant.bam.bai recalibration_report: - test_rnaseq_variant.recal.txt + ncpu: + - 1 base_recalibrator: - name: works inputs: @@ -34,6 +36,8 @@ base_recalibrator: - [ reference/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz ] known_indels_sites_indices: - [ reference/Mills_and_1000G_gold_standard.indels.hg38.vcf.gz.tbi ] + ncpu: + - 1 haplotype_caller: - name: works tags: [ slow ] @@ -98,4 +102,6 @@ mark_duplicates_spark: - bams/test.bwa_aln_pe.chrY_chrM.bam - bams/Aligned.sortedByCoord.chr9_chr22.bam - bams/test_rnaseq_variant.bam - - bams/test.bam \ No newline at end of file + - bams/test.bam + ncpu: + - 1 \ No newline at end of file diff --git a/tools/util.wdl b/tools/util.wdl index 2a82680c8..d848dd6de 100644 --- a/tools/util.wdl +++ b/tools/util.wdl @@ -114,6 +114,8 @@ task calc_feature_lengths { Int disk_size_gb = ceil(gtf_size * 2) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + python3 /scripts/util/calc_feature_lengths.py \ --id_attr "~{idattr}" \ "~{gtf}" \ @@ -152,6 +154,8 @@ task compression_integrity { Int disk_size_gb = ceil(file_size) + 10 + modify_disk_size_gb command <<< + set -euo pipefail + bgzip -t "~{bgzipped_file}" >>> @@ -338,6 +342,8 @@ task global_phred_scores { String outfile_name = prefix + ".global_PHRED_scores.tsv" command <<< + set -euo pipefail + python3 /scripts/util/calc_global_phred_scores.py \ ~{if fast_mode then "--fast_mode" @@ -394,6 +400,8 @@ task check_fastq_and_rg_concordance { ]) command <<< + set -euo pipefail + python3 /scripts/util/check_FQs_and_RGs.py \ --read-one-fastqs "~{sep(",", squote(read_one_names))}" \ ~{if length(read_twos) > 0 diff --git a/workflows/dnaseq/dnaseq-standard.wdl b/workflows/dnaseq/dnaseq-standard.wdl index faa9be835..13c9db8de 100644 --- a/workflows/dnaseq/dnaseq-standard.wdl +++ b/workflows/dnaseq/dnaseq-standard.wdl @@ -154,6 +154,8 @@ task parse_input { } command <<< + set -euo pipefail + if [ "~{aligner}" != "mem" ] \ && [ "~{aligner}" != "aln" ] then diff --git a/workflows/methylation/methylation-cohort.wdl b/workflows/methylation/methylation-cohort.wdl index 652ab9c0d..c10d388c4 100644 --- a/workflows/methylation/methylation-cohort.wdl +++ b/workflows/methylation/methylation-cohort.wdl @@ -179,6 +179,8 @@ task combine_data { Int disk_size_gb = ceil(size(files_to_combine, "GB") * 2) + 2 command <<< + set -euo pipefail + python /scripts/methylation/combine.py \ --output-name "~{combined_file_name}" \ ~{if simple_merge @@ -235,6 +237,8 @@ task filter_probes { Int disk_size_gb = ceil(size(beta_values, "GB") * 2) + 2 command <<< + set -euo pipefail + python /scripts/methylation/filter.py \ --output-name "~{prefix}.beta.csv" \ --filtered-probes "~{prefix}.probes.csv" \ @@ -282,6 +286,8 @@ task generate_umap { Int disk_size_gb = ceil(size(filtered_beta_values, "GB") * 2) + 2 command <<< + set -euo pipefail + python /scripts/methylation/generate_umap.py \ --beta "~{filtered_beta_values}" \ --output-name "~{prefix}.csv" @@ -319,6 +325,8 @@ task plot_umap { } command <<< + set -euo pipefail + python /scripts/methylation/plot_umap.py \ --umap "~{umap}" \ --output-name "~{plot_file}" diff --git a/workflows/qc/quality-check-standard.wdl b/workflows/qc/quality-check-standard.wdl index 8d80458c1..0e6aa1242 100644 --- a/workflows/qc/quality-check-standard.wdl +++ b/workflows/qc/quality-check-standard.wdl @@ -567,6 +567,8 @@ task parse_input { Int coverage_labels_len = length(coverage_labels) command <<< + set -euo pipefail + EXITCODE=0 if ~{rna} && ! ~{gtf_provided}; then diff --git a/workflows/rnaseq/rnaseq-standard.wdl b/workflows/rnaseq/rnaseq-standard.wdl index d100a9af7..c4bfaa711 100755 --- a/workflows/rnaseq/rnaseq-standard.wdl +++ b/workflows/rnaseq/rnaseq-standard.wdl @@ -168,6 +168,8 @@ task parse_input { } command <<< + set -euo pipefail + if [ -n "~{strand}" ] \ && [ "~{strand}" != "Stranded-Reverse" ] \ && [ "~{strand}" != "Stranded-Forward" ] \