From ed9c127221d85c07bf67a0a0df3fc200a64c9f7e Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 9 Dec 2024 17:04:17 +1100 Subject: [PATCH 01/88] add sigrap modules --- modules/local/sigrap/chord/main.nf | 39 ++++++++++++++++++++++++++ modules/local/sigrap/hrdetect/main.nf | 40 +++++++++++++++++++++++++++ modules/local/sigrap/mutpat/main.nf | 38 +++++++++++++++++++++++++ workflows/sash.nf | 4 +++ 4 files changed, 121 insertions(+) create mode 100644 modules/local/sigrap/chord/main.nf create mode 100644 modules/local/sigrap/hrdetect/main.nf create mode 100644 modules/local/sigrap/mutpat/main.nf diff --git a/modules/local/sigrap/chord/main.nf b/modules/local/sigrap/chord/main.nf new file mode 100644 index 00000000..f8b8b25e --- /dev/null +++ b/modules/local/sigrap/chord/main.nf @@ -0,0 +1,39 @@ +process SIGRAP_CHORD { + tag "${meta.id}" + label 'process_low' + + container 'docker.io/qclayssen/sigrap:0.1.1' + + input: + tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) + + output: + path 'output/chord.json.gz' , emit: chord_json + path 'versions.yml' , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + + """ + sigrap chord \\ + --sample ${meta.subject_id} \\ + --snv ${smlv_somatic_vcf} \\ + --sv ${sv_somatic_vcf} \\ + --out output/ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + sigrap: \$(sigrap --version | sed 's/^.*version //') + END_VERSIONS + """ + + stub: + """ + mkdir -p output/ + touch output/chord.json.gz + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml + """ +} diff --git a/modules/local/sigrap/hrdetect/main.nf b/modules/local/sigrap/hrdetect/main.nf new file mode 100644 index 00000000..e02c2d4d --- /dev/null +++ b/modules/local/sigrap/hrdetect/main.nf @@ -0,0 +1,40 @@ +process SIGRAP_HRDETECT { + tag "${meta.id}" + label 'process_low' + + container 'docker.io/qclayssen/sigrap:0.1.1' + + input: + tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) + + output: + path 'output/hrdetect.json.gz' , emit: hrdetect_json + path 'versions.yml' , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + + """ + sigrap hrdetect \\ + --sample ${meta.subject_id} \\ + --snv ${smlv_somatic_vcf} \\ + --sv ${sv_somatic_vcf} \\ + --cnv ${cnv_somatic_tsv} \\ + --out output/ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + sigrap: \$(sigrap --version | sed 's/^.*version //') + END_VERSIONS + """ + + stub: + """ + mkdir -p output/ + touch output/hrdetect.json.gz + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml + """ +} diff --git a/modules/local/sigrap/mutpat/main.nf b/modules/local/sigrap/mutpat/main.nf new file mode 100644 index 00000000..ab2bb5ac --- /dev/null +++ b/modules/local/sigrap/mutpat/main.nf @@ -0,0 +1,38 @@ +process SIGRAP_MUTPAT { + tag "${meta.id}" + label 'process_low' + + container 'docker.io/qclayssen/sigrap:0.1.1' + + input: + tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) + + output: + path 'output/' , emit: mutpat_output + path 'versions.yml' , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + + """ + sigrap mutpat \\ + --sample ${meta.subject_id} \\ + --snv ${smlv_somatic_vcf} \\ + --out output/ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + sigrap: \$(sigrap --version | sed 's/^.*version //') + END_VERSIONS + """ + + stub: + """ + mkdir -p output/ + touch output/chord.json.gz + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml + """ +} diff --git a/workflows/sash.nf b/workflows/sash.nf index b963867a..30e6edf4 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -47,6 +47,10 @@ include { BOLT_SMLV_SOMATIC_RESCUE } from '../modules/local/bolt/smlv_somatic/ include { BOLT_SV_SOMATIC_ANNOTATE } from '../modules/local/bolt/sv_somatic/annotate/main' include { BOLT_SV_SOMATIC_PRIORITISE } from '../modules/local/bolt/sv_somatic/prioritise/main' include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' +include { SIGRAP_CHORD } from '../modules/local/sigvrap/chord/main' +include { SIGRAP_HRDETECT } from '../modules/local/sigvrap/hrdetect/main' +include { SIGRAP_MUTPAT } from '../modules/local/sigvrap/mutpat/main' + include { GRIPSS_FILTERING } from '../subworkflows/local/gripss_filtering' include { LINX_ANNOTATION } from '../subworkflows/local/linx_annotation' From 8914f8e75698badd6e51208901265f39285ab13a Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 21 Feb 2025 09:45:10 +1100 Subject: [PATCH 02/88] changes images modules --- modules/local/bolt/other/cancer_report/main.nf | 2 +- modules/local/sigrap/chord/main.nf | 5 +++-- modules/local/sigrap/hrdetect/main.nf | 2 +- modules/local/sigrap/mutpat/main.nf | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index 4690eda8..3000f4f6 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-gpgr' + container 'docker.io/qclayssen/bolt:gpgr-input-sigrap-data' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) diff --git a/modules/local/sigrap/chord/main.nf b/modules/local/sigrap/chord/main.nf index f8b8b25e..80199d2d 100644 --- a/modules/local/sigrap/chord/main.nf +++ b/modules/local/sigrap/chord/main.nf @@ -2,10 +2,11 @@ process SIGRAP_CHORD { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/sigrap:0.1.1' + container 'ghcr.io/umccr/sigrap:0.2.0' input: - tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) + tuple val(meta), path(smlv_somatic_vcf) + tuple val(meta), path(sv_somatic_vcf) output: path 'output/chord.json.gz' , emit: chord_json diff --git a/modules/local/sigrap/hrdetect/main.nf b/modules/local/sigrap/hrdetect/main.nf index e02c2d4d..425819f6 100644 --- a/modules/local/sigrap/hrdetect/main.nf +++ b/modules/local/sigrap/hrdetect/main.nf @@ -2,7 +2,7 @@ process SIGRAP_HRDETECT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/sigrap:0.1.1' + container 'ghcr.io/umccr/sigrap:0.2.0' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) diff --git a/modules/local/sigrap/mutpat/main.nf b/modules/local/sigrap/mutpat/main.nf index ab2bb5ac..6173fc7d 100644 --- a/modules/local/sigrap/mutpat/main.nf +++ b/modules/local/sigrap/mutpat/main.nf @@ -2,13 +2,13 @@ process SIGRAP_MUTPAT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/sigrap:0.1.1' + container 'ghcr.io/umccr/sigrap:0.2.0' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) output: - path 'output/' , emit: mutpat_output + path 'output/' , emit: mutpat_output path 'versions.yml' , emit: versions when: From 03c2cf337b337da4006f96523808307f6899b263 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 21 Feb 2025 09:46:01 +1100 Subject: [PATCH 03/88] uptade workflow with modules a channels --- workflows/sash.nf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/workflows/sash.nf b/workflows/sash.nf index 30e6edf4..33c878cd 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -473,6 +473,21 @@ workflow SASH { return [meta, hrd] } + SIGRAP_CHORD( + ch_smlv_somatic_out, + ch_sv_somatic_sv_vcf_out + ) + SIGRAP_HRDETECT( + ch_smlv_somatic_out, + ch_sv_somatic_sv_vcf_out, + ch_sv_somatic_cnv_tsv_out, + + + ) + SIGRAP_MUTPAT( + ch_smlv_somatic_out + ) + // channel: [ meta_bolt, smlv_somatic_vcf, smlv_somatic_bcftools_stats, smlv_somatic_counts_process, sv_tsv, sv_vcf, cnv_tsv, af_global, af_keygenes, purple_baf_circos_plot, purple_dir, virusbreakend_dir, dragen_hrd ] ch_cancer_report_inputs = WorkflowSash.groupByMeta( ch_smlv_somatic_out, @@ -487,6 +502,9 @@ workflow SASH { PURPLE_CALLING.out.purple_dir, ch_virusbreakend, ch_input_hrd, + SIGRAP_MUTPAT.out.mutpat_output, + SIGRAP_HRDETECT.out.hrdetect_json, + SIGRAP_CHORD.out.chord_json ) .map { def meta = it[0] From e8bf9d3a584bc4eac20b3d0bb575dcdad5f1ec5c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 21 Feb 2025 16:34:11 +1100 Subject: [PATCH 04/88] fix typo --- workflows/sash.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 33c878cd..fe8d9b2f 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -47,9 +47,9 @@ include { BOLT_SMLV_SOMATIC_RESCUE } from '../modules/local/bolt/smlv_somatic/ include { BOLT_SV_SOMATIC_ANNOTATE } from '../modules/local/bolt/sv_somatic/annotate/main' include { BOLT_SV_SOMATIC_PRIORITISE } from '../modules/local/bolt/sv_somatic/prioritise/main' include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' -include { SIGRAP_CHORD } from '../modules/local/sigvrap/chord/main' -include { SIGRAP_HRDETECT } from '../modules/local/sigvrap/hrdetect/main' -include { SIGRAP_MUTPAT } from '../modules/local/sigvrap/mutpat/main' +include { SIGRAP_CHORD } from '../modules/local/sigrap/chord/main' +include { SIGRAP_HRDETECT } from '../modules/local/sigrap/hrdetect/main' +include { SIGRAP_MUTPAT } from '../modules/local/sigrap/mutpat/main' include { GRIPSS_FILTERING } from '../subworkflows/local/gripss_filtering' From ab0d3b910eaad0c281027e542bac5629ae004d20 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 21 Feb 2025 16:38:37 +1100 Subject: [PATCH 05/88] fix input module --- modules/local/sigrap/hrdetect/main.nf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/local/sigrap/hrdetect/main.nf b/modules/local/sigrap/hrdetect/main.nf index 425819f6..1bc175d3 100644 --- a/modules/local/sigrap/hrdetect/main.nf +++ b/modules/local/sigrap/hrdetect/main.nf @@ -5,7 +5,9 @@ process SIGRAP_HRDETECT { container 'ghcr.io/umccr/sigrap:0.2.0' input: - tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) + path(smlv_somatic_vcf) + path(sv_somatic_vcf) + path(cnv_somatic_tsv) output: path 'output/hrdetect.json.gz' , emit: hrdetect_json From 0011e478d73e5662b0b0a3783ee5ebec74950960 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 29 Apr 2025 14:16:15 +1000 Subject: [PATCH 06/88] replace gridss by esvee data --- workflows/sash.nf | 50 ++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index b963867a..a978f77f 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -95,14 +95,29 @@ workflow SASH { // channel: [ meta, cobalt_dir ] ch_cobalt = ch_inputs.map { meta -> [meta, file(meta.oncoanalyser_dir).toUriString() + '/cobalt/'] } - // channel: [ meta, gripss_somatic_vcf, gripss_somatic_tbi ] - ch_gridss = ch_inputs + // channel: [ meta, esvee_somatic_vcf, esvee_somatic_tbi ] + ch_esvee_somatic = ch_inputs .map { meta -> - def subpath = "/gridss/${meta.tumor_id}.gridss.vcf.gz" + def subpath = "/esvee/caller/${meta.tumor_id}.esvee.somatic.vcf.gz" def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath return [meta, vcf] } + // channel: [ meta, esvee_germline_vcf, esvee_germline_tbi ] + ch_esvee_germline = ch_inputs + .map { meta -> + def subpath = "/esvee/caller/${meta.tumor_id}.esvee.germline.vcf.gz" + def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath + return [meta, vcf] + } + + // channel: [ meta, esvee_somatic_unfiltred_vcf, esvee_somatic_unfiltred_tbi ] + ch_esvee_somatic_unfiltred = ch_inputs + .map { meta -> + def subpath = "/esvee/caller/${meta.tumor_id}.esvee.unfiltered.vcf.g" + def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath + return [meta, vcf] + } // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] ch_sage_somatic = ch_inputs .map { meta -> @@ -246,27 +261,6 @@ workflow SASH { ch_smlv_germline_out = WorkflowSash.restoreMeta(BOLT_SMLV_GERMLINE_PREPARE.out.vcf, ch_inputs) - - - - // - // Somatic structural variants - // - GRIPSS_FILTERING( - ch_inputs, - ch_gridss, - genome.fasta, - genome.version, - genome.fai, - hmf_data.gridss_pon_breakends, - hmf_data.gridss_pon_breakpoints, - umccr_data.known_fusions, - hmf_data.repeatmasker_annotations, - ) - - - - // // CNV calling using UMCCR postprocessed variants // @@ -284,11 +278,9 @@ workflow SASH { // * https://github.com/hartwigmedical/hmftools/blob/a2f82e5/purple/src/main/java/com/hartwig/hmftools/purple/germline/GermlineGenotypeEnrichment.java#L63 //ch_smlv_germline_out, ch_smlv_germline_out.map { meta, vcf -> return [meta, []] }, - - - GRIPSS_FILTERING.out.somatic, - GRIPSS_FILTERING.out.germline, - GRIPSS_FILTERING.out.somatic_unfiltered, + ch_esvee_somatic, + ch_esvee_germline, + ch_esvee_somatic_unfiltred, genome.fasta, genome.version, genome.fai, From 9da48e7adfa4e1bfda0bf8e6c1cd3eb90f435203 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 10:58:46 +1000 Subject: [PATCH 07/88] change input channel files in subworkflow --- subworkflows/local/prepare_input.nf | 57 ++++++++++++++++++++ workflows/sash.nf | 81 +++++------------------------ 2 files changed, 70 insertions(+), 68 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 0e32f978..8ea361f8 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -40,6 +40,63 @@ workflow PREPARE_INPUT { return meta } + // map oncoanalyser assets and DRAGEN outputs into channels + def ch_amber = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + return [meta, "${base}/amber/"] + } + def ch_cobalt = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + return [meta, "${base}/cobalt/"] + } + def ch_esvee_somatic = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.somatic.vcf.gz" + return [meta, vcf] + } + def ch_esvee_germline = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.germline.vcf.gz" + return [meta, vcf] + } + def ch_esvee_somatic_unfiltered = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz" + return [meta, vcf] + } + def ch_sage_somatic = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + def vcf = "${base}/sage/somatic/${meta.tumor_id}.sage.somatic.vcf.gz" + return [meta, vcf, "${vcf}.tbi"] + } + def ch_virusbreakend = ch_metas.map { meta -> + def base = file(meta.oncoanalyser_dir).toUriString() + return [meta, "${base}/virusbreakend/"] + } + def ch_input_hrd = ch_metas.map { meta -> + def base = file(meta.dragen_somatic_dir).toUriString() + return [meta, "${base}/${meta.tumor_id}.hrdscore.csv"] + } + def ch_input_vcf_germline = ch_metas.map { meta -> + def base = file(meta.dragen_germline_dir).toUriString() + return [meta, "${base}/${meta.normal_id}.hard-filtered.vcf.gz"] + } + def ch_input_vcf_somatic = ch_metas.map { meta -> + def base = file(meta.dragen_somatic_dir).toUriString() + def vcf = "${base}/${meta.tumor_id}.hard-filtered.vcf.gz" + return [meta, vcf, "${vcf}.tbi"] + } + emit: metas = ch_metas + amber = ch_amber + cobalt = ch_cobalt + esvee_somatic = ch_esvee_somatic + esvee_germline = ch_esvee_germline + esvee_somatic_unfiltered = ch_esvee_somatic_unfiltered + sage_somatic = ch_sage_somatic + virusbreakend = ch_virusbreakend + hrd = ch_input_hrd + vcf_germline = ch_input_vcf_germline + vcf_somatic = ch_input_vcf_somatic } diff --git a/workflows/sash.nf b/workflows/sash.nf index a978f77f..2336e5d2 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -87,52 +87,16 @@ workflow SASH { ) ch_inputs = PREPARE_INPUT.out.metas - // TODO(SW): place into subworkflow; simplify with map of variable name -> subpath - - // channel: [ meta, amber_dir ] - ch_amber = ch_inputs.map { meta -> [meta, file(meta.oncoanalyser_dir).toUriString() + '/amber/'] } - - // channel: [ meta, cobalt_dir ] - ch_cobalt = ch_inputs.map { meta -> [meta, file(meta.oncoanalyser_dir).toUriString() + '/cobalt/'] } - - // channel: [ meta, esvee_somatic_vcf, esvee_somatic_tbi ] - ch_esvee_somatic = ch_inputs - .map { meta -> - def subpath = "/esvee/caller/${meta.tumor_id}.esvee.somatic.vcf.gz" - def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath - return [meta, vcf] - } - - // channel: [ meta, esvee_germline_vcf, esvee_germline_tbi ] - ch_esvee_germline = ch_inputs - .map { meta -> - def subpath = "/esvee/caller/${meta.tumor_id}.esvee.germline.vcf.gz" - def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath - return [meta, vcf] - } - - // channel: [ meta, esvee_somatic_unfiltred_vcf, esvee_somatic_unfiltred_tbi ] - ch_esvee_somatic_unfiltred = ch_inputs - .map { meta -> - def subpath = "/esvee/caller/${meta.tumor_id}.esvee.unfiltered.vcf.g" - def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath - return [meta, vcf] - } - // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] - ch_sage_somatic = ch_inputs - .map { meta -> - def subpath = "/sage/somatic/${meta.tumor_id}.sage.somatic.vcf.gz" - def vcf = file(meta.oncoanalyser_dir).toUriString() + subpath - return [meta, vcf, "${vcf}.tbi"] - } - - // channel: [ meta, virusbreakend_dir ] - ch_virusbreakend = ch_inputs - .map { meta -> - def subpath = '/virusbreakend/' - def virusbreakend_dir = file(meta.oncoanalyser_dir).toUriString() + subpath - return [meta, virusbreakend_dir] - } + ch_amber = PREPARE_INPUT.out.amber + ch_cobalt = PREPARE_INPUT.out.cobalt + ch_esvee_somatic = PREPARE_INPUT.out.esvee_somatic + ch_esvee_germline = PREPARE_INPUT.out.esvee_germline + ch_esvee_somatic_unfiltered = PREPARE_INPUT.out.esvee_somatic_unfiltered + ch_sage_somatic = PREPARE_INPUT.out.sage_somatic + ch_virusbreakend = PREPARE_INPUT.out.virusbreakend + ch_input_hrd = PREPARE_INPUT.out.hrd + ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline + ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic @@ -155,12 +119,6 @@ workflow SASH { // Somatic small variants // - // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] - ch_input_vcf_somatic = ch_inputs - .map { meta -> - def vcf = file(meta.dragen_somatic_dir).toUriString() + "/${meta.tumor_id}.hard-filtered.vcf.gz" - return [meta, vcf, "${vcf}.tbi"] - } // channel: [ meta_bolt, dragen_somatic_vcf, dragen_somatic_tbi, sage_somatic_vcf, sage_somatic_tbi ] @@ -176,7 +134,7 @@ workflow SASH { tumor_id: meta.tumor_id, normal_id: meta.normal_id, ] - return [meta_bolt, *it[1..-1]] + return [meta_bolt] + it[1..-1] } BOLT_SMLV_SOMATIC_RESCUE( @@ -230,14 +188,6 @@ workflow SASH { // Germline small variants // - // channel: [ meta, dragen_germline_vcf ] - ch_input_vcf_germline = ch_inputs - .map { meta -> - def vcf = file(meta.dragen_germline_dir).toUriString() + "/${meta.normal_id}.hard-filtered.vcf.gz" - return [meta, vcf] - } - - // channel: [ meta_bolt, dragen_germline_vcf ] ch_smlv_germline_prepare_inputs = ch_input_vcf_germline .map { meta, dragen_vcf -> @@ -280,7 +230,7 @@ workflow SASH { ch_smlv_germline_out.map { meta, vcf -> return [meta, []] }, ch_esvee_somatic, ch_esvee_germline, - ch_esvee_somatic_unfiltred, + ch_esvee_somatic_unfiltered, genome.fasta, genome.version, genome.fai, @@ -455,11 +405,6 @@ workflow SASH { // // channel: [ meta, dragen_hrd ] - ch_input_hrd = ch_inputs - .map { meta -> - def hrd = file(meta.dragen_somatic_dir).toUriString() + "/${meta.tumor_id}.hrdscore.csv" - return [meta, hrd] - } // channel: [ meta_bolt, smlv_somatic_vcf, smlv_somatic_bcftools_stats, smlv_somatic_counts_process, sv_tsv, sv_vcf, cnv_tsv, af_global, af_keygenes, purple_baf_circos_plot, purple_dir, virusbreakend_dir, dragen_hrd ] ch_cancer_report_inputs = WorkflowSash.groupByMeta( @@ -484,7 +429,7 @@ workflow SASH { subject_id: meta.subject_id, tumor_id: meta.tumor_id, ] - return [meta_bolt, *it[1..-1]] + return [meta_bolt] + it[1..-1] } BOLT_OTHER_CANCER_REPORT( From 033af833111568ba64d7d11cc2e870a314129291 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 11:02:20 +1000 Subject: [PATCH 08/88] fix meta channel --- subworkflows/local/prepare_input.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 8ea361f8..802c92f8 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -3,7 +3,7 @@ workflow PREPARE_INPUT { ch_samplesheet main: - ch_metas = Channel.of(ch_samplesheet) + def ch_metas = Channel.of(ch_samplesheet) .splitCsv(header: true) .map { [it.id, it] } .groupTuple() From c9136cf1da9564664f764fb4beed05de1fb67d98 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 11:11:55 +1000 Subject: [PATCH 09/88] fix add tbi esvee files --- subworkflows/local/prepare_input.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 802c92f8..92c97af9 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -52,17 +52,17 @@ workflow PREPARE_INPUT { def ch_esvee_somatic = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.somatic.vcf.gz" - return [meta, vcf] + return [meta, vcf, "${vcf}.tbi"] } def ch_esvee_germline = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.germline.vcf.gz" - return [meta, vcf] + return [meta, vcf, "${vcf}.tbi"] } def ch_esvee_somatic_unfiltered = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz" - return [meta, vcf] + return [meta, vcf, "${vcf}.tbi"] } def ch_sage_somatic = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() From 3e5399ae08b815f22e9abe48d136b83375c33b06 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 14:27:21 +1000 Subject: [PATCH 10/88] add esvee module --- modules/local/esvee/call/environment.yml | 7 ++ modules/local/esvee/call/main.nf | 81 ++++++++++++++++++++++++ modules/local/esvee/call/meta.yml | 73 +++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 modules/local/esvee/call/environment.yml create mode 100644 modules/local/esvee/call/main.nf create mode 100644 modules/local/esvee/call/meta.yml diff --git a/modules/local/esvee/call/environment.yml b/modules/local/esvee/call/environment.yml new file mode 100644 index 00000000..01355c11 --- /dev/null +++ b/modules/local/esvee/call/environment.yml @@ -0,0 +1,7 @@ +name: esvee_call +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::hmftools-esvee=1.0.3 diff --git a/modules/local/esvee/call/main.nf b/modules/local/esvee/call/main.nf new file mode 100644 index 00000000..a6480a18 --- /dev/null +++ b/modules/local/esvee/call/main.nf @@ -0,0 +1,81 @@ +process ESVEE_CALL { + tag "${meta.id}" + label 'process_high' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/hmftools-esvee:1.0.3--hdfd78af_0' : + 'biocontainers/hmftools-esvee:1.0.3--hdfd78af_0' }" + + input: + tuple val(meta), path(ref_depth_vcf), path(prep_dir) + path genome_fasta + val genome_ver + path pon_breakends + path pon_breakpoints + path known_fusions + path repeatmasker_annotations + + output: + tuple val(meta), path("caller/") , emit: caller_dir + tuple val(meta), path("caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz"), path("caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz.tbi"), emit: unfiltered_vcf + tuple val(meta), path("caller/${meta.tumor_id}.esvee.somatic.vcf.gz"), path("caller/${meta.tumor_id}.esvee.somatic.vcf.gz.tbi") , emit: somatic_vcf + tuple val(meta), path("caller/${meta.tumor_id}.esvee.germline.vcf.gz"), path("caller/${meta.tumor_id}.esvee.germline.vcf.gz.tbi") , emit: germline_vcf, optional: true + path 'versions.yml' , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + + def xmx_mod = task.ext.xmx_mod ?: 0.95 + + def reference_arg = meta.normal_id != null ? "-reference ${meta.normal_id}" : '' + + """ + mkdir -p caller/ + + esvee com.hartwig.hmftools.esvee.caller.CallerApplication \\ + -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ + ${args} \\ + -sample ${meta.tumor_id} \\ + ${reference_arg} \\ + -input_vcf ${ref_depth_vcf} \\ + -esvee_prep_dir ${prep_dir}/ \\ + -ref_genome_version ${genome_ver} \\ + -known_hotspot_file ${known_fusions} \\ + -pon_sgl_file ${pon_breakends} \\ + -pon_sv_file ${pon_breakpoints} \\ + -repeat_mask_file ${repeatmasker_annotations} \\ + -output_dir caller/ \\ + -log_level DEBUG + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + esvee: \$(esvee -version | sed -n '/^Esvee version/ { s/^.* //p }') + END_VERSIONS + """ + + stub: + """ + mkdir -p caller/ + + vcf_template='##fileformat=VCFv4.1 + ##contig= + #CHROM POS ID REF ALT QUAL FILTER INFO + . . . . . . . + ' + + echo \${vcf_template} | gzip -c > caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz + echo \${vcf_template} | gzip -c > caller/${meta.tumor_id}.esvee.somatic.vcf.gz + + touch caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz.tbi + touch caller/${meta.tumor_id}.esvee.somatic.vcf.gz.tbi + + ${ (meta.normal_id != null) ? "touch caller/${meta.tumor_id}.esvee.germline.vcf.gz" : '' } + ${ (meta.normal_id != null) ? "touch caller/${meta.tumor_id}.esvee.germline.vcf.gz.tbi" : '' } + + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml + """ +} diff --git a/modules/local/esvee/call/meta.yml b/modules/local/esvee/call/meta.yml new file mode 100644 index 00000000..38979eb8 --- /dev/null +++ b/modules/local/esvee/call/meta.yml @@ -0,0 +1,73 @@ +name: esvee_call +description: Call somatic SVs with ESVEE +keywords: + - calling + - sv +tools: + - esvee: + description: Structural variant (SV) calling + homepage: https://github.com/hartwigmedical/hmftools/tree/master/esvee + documentation: https://github.com/hartwigmedical/hmftools/tree/master/esvee + licence: ["GPL >=3"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [id: 'sample_id'] + - ref_depth_vcf: + type: file + description: ESVEE depth annotated VCF file + pattern: "*.{vcf.gz}" + - prep_dir: + type: directory + description: ESVEE prep output directory + - genome_fasta: + type: file + description: Reference genome assembly FASTA file + pattern: "*.{fa,fasta}" + - genome_ver: + type: string + description: Reference genome version + - pon_breakend: + type: file + description: GRIDSS breakend PON file + pattern: "*.{bed.gz}" + - pon_breakpoints: + type: file + description: GRIDSS breakpoint PON file + pattern: "*.{bedpe.gz}" + - known_fusions: + type: file + description: HMF Known Fusions file + pattern: "*.{bedpe}" + - repeatmasker_annotations: + type: file + description: RepeatMasker annotations file +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [id: 'sample_id'] + - caller_dir: + type: directory + description: ESVEE call output directory + - unfiltered_vcf: + type: list + description: Unfiltered VCF file + pattern: "*.{vcf.gz,vcf.gz.tbi}" + - somatic_vcf: + type: list + description: Somatic VCF file + pattern: "*.{vcf.gz,vcf.gz.tbi}" + - germline_vcf: + type: list + description: Germline VCF file (optional) + pattern: "*.{vcf.gz,vcf.gz.tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@scwatts" From 4e34cbe85e727a04c8034d0827b6623008946505 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 14:27:35 +1000 Subject: [PATCH 11/88] update purple --- modules/local/purple/main.nf | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/local/purple/main.nf b/modules/local/purple/main.nf index 477d92ec..999e921b 100644 --- a/modules/local/purple/main.nf +++ b/modules/local/purple/main.nf @@ -1,8 +1,11 @@ process PURPLE { tag "${meta.id}" - label 'process_low' + label 'process_medium' - container 'docker.io/scwatts/purple:4.0.2--1' + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/hmftools-purple:4.1--hdfd78af_0' : + 'biocontainers/hmftools-purple:4.1--hdfd78af_0' }" input: tuple val(meta), path(amber), path(cobalt), path(sv_tumor_vcf), path(sv_tumor_tbi), path(sv_tumor_unfiltered_vcf), path(sv_tumor_unfiltered_tbi), path(sv_normal_vcf), path(sv_normal_tbi), path(smlv_tumor_vcf), path(smlv_normal_vcf) @@ -30,6 +33,9 @@ process PURPLE { script: def args = task.ext.args ?: '' + // allow custom heap fraction + def xmx_mod = task.ext.xmx_mod ?: 0.75 + def reference_arg = meta.containsKey('normal_id') ? "-reference ${meta.normal_id}" : '' def sv_tumor_vcf_arg = sv_tumor_vcf ? "-somatic_sv_vcf ${sv_tumor_vcf}" : '' @@ -65,7 +71,7 @@ process PURPLE { fi; purple \\ - -Xmx${Math.round(task.memory.bytes * 0.95)} \\ + -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ ${args} \\ -tumor ${meta.tumor_id} \\ ${reference_arg} \\ @@ -73,7 +79,6 @@ process PURPLE { -cobalt ${cobalt} \\ ${sv_tumor_vcf_arg} \\ ${sv_normal_vcf_arg} \\ - ${sv_tumor_recovery_vcf_arg} \\ ${smlv_tumor_vcf_arg} \\ ${smlv_normal_vcf_arg} \\ -ref_genome ${genome_fasta} \\ @@ -99,7 +104,7 @@ process PURPLE { cat <<-END_VERSIONS > versions.yml "${task.process}": - purple: \$(purple -version | sed 's/^.* //') + purple: \$(purple -version | sed -n '/^Purple version / { s/^.* //p }') END_VERSIONS """ From 7f2abbdd0ff31ae73ddcd18f46141756efa5c126 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 15:01:01 +1000 Subject: [PATCH 12/88] fix image purple --- modules/local/purple/main.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/local/purple/main.nf b/modules/local/purple/main.nf index 999e921b..706130eb 100644 --- a/modules/local/purple/main.nf +++ b/modules/local/purple/main.nf @@ -3,9 +3,7 @@ process PURPLE { label 'process_medium' conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/hmftools-purple:4.1--hdfd78af_0' : - 'biocontainers/hmftools-purple:4.1--hdfd78af_0' }" + container 'docker.io/qclayssen/purple:4.1.0--1' input: tuple val(meta), path(amber), path(cobalt), path(sv_tumor_vcf), path(sv_tumor_tbi), path(sv_tumor_unfiltered_vcf), path(sv_tumor_unfiltered_tbi), path(sv_normal_vcf), path(sv_normal_tbi), path(smlv_tumor_vcf), path(smlv_normal_vcf) @@ -79,6 +77,7 @@ process PURPLE { -cobalt ${cobalt} \\ ${sv_tumor_vcf_arg} \\ ${sv_normal_vcf_arg} \\ + ${sv_tumor_recovery_vcf_arg} \\ ${smlv_tumor_vcf_arg} \\ ${smlv_normal_vcf_arg} \\ -ref_genome ${genome_fasta} \\ From 6a2a0312c2e303474131b09ed904532c194762bf Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 15:23:57 +1000 Subject: [PATCH 13/88] upgrade docker image and add env --- modules/local/linx/Dockerfile | 2 +- modules/local/linx/germline/environment.yml | 7 +++ modules/local/linx/germline/main.nf | 9 ++- modules/local/linx/somatic/environment.yml | 7 +++ modules/local/linx/somatic/main.nf | 13 +++-- modules/local/linx/visualiser/environment.yml | 7 +++ modules/local/linx/visualiser/main.nf | 57 ++++++++++++++++--- modules/local/linxreport/Dockerfile | 2 +- modules/local/linxreport/environment.yml | 7 +++ modules/local/linxreport/main.nf | 15 +++-- modules/local/purple/Dockerfile | 2 +- modules/local/purple/environment.yml | 7 +++ 12 files changed, 110 insertions(+), 25 deletions(-) create mode 100644 modules/local/linx/germline/environment.yml create mode 100644 modules/local/linx/somatic/environment.yml create mode 100644 modules/local/linx/visualiser/environment.yml create mode 100644 modules/local/linxreport/environment.yml create mode 100644 modules/local/purple/environment.yml diff --git a/modules/local/linx/Dockerfile b/modules/local/linx/Dockerfile index eb57d724..bb54ea71 100644 --- a/modules/local/linx/Dockerfile +++ b/modules/local/linx/Dockerfile @@ -7,7 +7,7 @@ RUN \ zlib && \ micromamba clean --all --yes -FROM quay.io/biocontainers/hmftools-linx:1.25--hdfd78af_0 +FROM quay.io/biocontainers/hmftools-linx:2.0--hdfd78af_0 COPY --from=build /env/ /env/ diff --git a/modules/local/linx/germline/environment.yml b/modules/local/linx/germline/environment.yml new file mode 100644 index 00000000..57471f2c --- /dev/null +++ b/modules/local/linx/germline/environment.yml @@ -0,0 +1,7 @@ +name: linx_germline +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::hmftools-linx=2.0 diff --git a/modules/local/linx/germline/main.nf b/modules/local/linx/germline/main.nf index 8a5f4ac9..c3b436db 100644 --- a/modules/local/linx/germline/main.nf +++ b/modules/local/linx/germline/main.nf @@ -2,7 +2,7 @@ process LINX_GERMLINE { tag "${meta.id}" label 'process_low' - container 'docker.io/scwatts/hmftools-linx:1.25--0' + container 'docker.io/qclayssen/hmftools-linx:2.0--10' input: tuple val(meta), path(sv_vcf) @@ -20,9 +20,11 @@ process LINX_GERMLINE { script: def args = task.ext.args ?: '' + def xmx_mod = task.ext.xmx_mod ?: 0.75 + """ linx \\ - -Xmx${Math.round(task.memory.bytes * 0.95)} \\ + -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ ${args} \\ -sample ${meta.sample_id} \\ -sv_vcf ${sv_vcf} \\ @@ -34,7 +36,7 @@ process LINX_GERMLINE { cat <<-END_VERSIONS > versions.yml "${task.process}": - linx: \$(linx -version | sed 's/^.* //') + linx: \$(linx -version | sed -n '/^Linx version / { s/^.* //p }') END_VERSIONS """ @@ -42,6 +44,7 @@ process LINX_GERMLINE { """ mkdir linx_germline/ touch linx_germline/placeholder + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ } diff --git a/modules/local/linx/somatic/environment.yml b/modules/local/linx/somatic/environment.yml new file mode 100644 index 00000000..4d244fae --- /dev/null +++ b/modules/local/linx/somatic/environment.yml @@ -0,0 +1,7 @@ +name: linx_somatic +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::hmftools-linx=2.0 diff --git a/modules/local/linx/somatic/main.nf b/modules/local/linx/somatic/main.nf index c0504829..e9d5426d 100644 --- a/modules/local/linx/somatic/main.nf +++ b/modules/local/linx/somatic/main.nf @@ -2,7 +2,7 @@ process LINX_SOMATIC { tag "${meta.id}" label 'process_low' - container 'docker.io/scwatts/hmftools-linx:1.25--0' + container 'docker.io/qclayssen/hmftools-linx:2.0--1' input: tuple val(meta), path(purple_dir) @@ -10,7 +10,6 @@ process LINX_SOMATIC { path ensembl_data_resources path known_fusion_data path driver_gene_panel - path gene_id_file output: tuple val(meta), path('linx_somatic/'), emit: annotation_dir @@ -21,26 +20,27 @@ process LINX_SOMATIC { script: def args = task.ext.args ?: '' - def gene_id_file_arg = gene_id_file ? "-gene_id_file ${gene_id_file}" : '' + + def xmx_mod = task.ext.xmx_mod ?: 0.75 """ linx \\ - -Xmx${Math.round(task.memory.bytes * 0.95)} \\ + -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ ${args} \\ -sample ${meta.sample_id} \\ -sv_vcf ${purple_dir}/${meta.sample_id}.purple.sv.vcf.gz \\ -purple_dir ${purple_dir} \\ - ${gene_id_file_arg} \\ -ref_genome_version ${genome_ver} \\ -ensembl_data_dir ${ensembl_data_resources} \\ -known_fusion_file ${known_fusion_data} \\ -driver_gene_panel ${driver_gene_panel} \\ -write_vis_data \\ + -write_neo_epitopes \\ -output_dir linx_somatic/ cat <<-END_VERSIONS > versions.yml "${task.process}": - linx: \$(linx -version | sed 's/^.* //') + linx: \$(linx -version | sed -n '/^Linx version / { s/^.* //p }') END_VERSIONS """ @@ -48,6 +48,7 @@ process LINX_SOMATIC { """ mkdir linx_somatic/ touch linx_somatic/placeholder + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ } diff --git a/modules/local/linx/visualiser/environment.yml b/modules/local/linx/visualiser/environment.yml new file mode 100644 index 00000000..5dadb70e --- /dev/null +++ b/modules/local/linx/visualiser/environment.yml @@ -0,0 +1,7 @@ +name: linx_visualiser +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::hmftools-linx=2.0 diff --git a/modules/local/linx/visualiser/main.nf b/modules/local/linx/visualiser/main.nf index 01e28f2e..a40646a3 100644 --- a/modules/local/linx/visualiser/main.nf +++ b/modules/local/linx/visualiser/main.nf @@ -2,7 +2,7 @@ process LINX_VISUALISER { tag "${meta.id}" label 'process_medium' - container 'docker.io/scwatts/hmftools-linx:1.25--0' + container 'docker.io/qclayssen/hmftools-linx:2.0--1' input: tuple val(meta), path(linx_annotation_dir) @@ -18,8 +18,19 @@ process LINX_VISUALISER { script: def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' + + def xmx_mod = task.ext.xmx_mod ?: 0.75 """ + # NOTE(SW): the output plot directories are always required for ORANGE, which is straightfoward to handle with POSIX + # fs but more involved with FusionFS since it will not write empty directories to S3. A placeholder file can't be + # used in the plot directory to force FusionFS to create the directory as ORANGE will treat the placeholder as a PNG + # and fail. Optional outputs are possible but requires further channel logic and output to detect when complete. + # Instead I place the two plot output directories under a parent directory, only operating on that to allow use of a + # placeholder and support empty outputs when using FusionFS. Handling missing/non-existent directories are deferred + # to downstream processes, bypassing the need to implement further channel operations. + mkdir -p plots/ # NOTE(SW): LINX v1.24.1 require trailing slashes for the -plot_out and -data_out arguments since no filesystem @@ -28,8 +39,10 @@ process LINX_VISUALISER { # https://github.com/hartwigmedical/hmftools/blob/linx-v1.24.1/linx/src/main/java/com/hartwig/hmftools/linx/visualiser/circos/ChromosomeRangeExecution.java#L22-L29 # https://github.com/hartwigmedical/hmftools/blob/linx-v1.24.1/linx/src/main/java/com/hartwig/hmftools/linx/visualiser/circos/FusionExecution.java#L18-L23 + # Generate all chromosome and cluster plots by default + linx \\ - -Xmx${Math.round(task.memory.bytes * 0.95)} \\ + -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ com.hartwig.hmftools.linx.visualiser.SvVisualiser \\ ${args} \\ -sample ${meta.sample_id} \\ @@ -38,19 +51,49 @@ process LINX_VISUALISER { -ensembl_data_dir ${ensembl_data_resources} \\ -circos \$(which circos) \\ -threads ${task.cpus} \\ - -plot_out plots/ \\ - -data_out data/ + -plot_out plots/all/ \\ + -data_out data/all/ + + # Rerun LINX to render only reportable cluster plots in a separate directory. While this is regenerating existing + # cluster plots, the number of reportable plots is generally very small and I prefer to rely on the internal LINX + # logic to determine whether a cluster is reportable rather than attempting to infer manually to copy out target + # plot files. + + # The ORANGE report receives only reportable clusters while the gpgr LINX report receives chromosome and all cluster + # plots. + + # https://github.com/hartwigmedical/hmftools/blob/linx-v1.24.1/linx/src/main/java/com/hartwig/hmftools/linx/visualiser/SampleData.java#L220-L236 + + linx \\ + -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ + com.hartwig.hmftools.linx.visualiser.SvVisualiser \\ + ${args2} \\ + -sample ${meta.sample_id} \\ + -vis_file_dir ${linx_annotation_dir} \\ + -ref_genome_version ${genome_ver} \\ + -ensembl_data_dir ${ensembl_data_resources} \\ + -circos \$(which circos) \\ + -plot_reportable \\ + -threads ${task.cpus} \\ + -plot_out plots/reportable/ \\ + -data_out data/reportable/ + + # Create placeholders to force FusionFS to create parent plot directory on S3 + if [[ \$(ls plots/ | wc -l) -eq 0 ]]; then + touch plots/.keep; + fi; cat <<-END_VERSIONS > versions.yml "${task.process}": - linx: \$(linx -version | sed 's/^.* //') + linx: \$(linx -version | sed -n '/^Linx version / { s/^.* //p }') END_VERSIONS """ stub: """ - mkdir -p plots/ - touch plots/placeholder + mkdir -p plots/{all,reportable}/ + touch plots/{all,reportable}/placeholder + echo -e '${task.process}:\n stub: noversions\n' > versions.yml """ } diff --git a/modules/local/linxreport/Dockerfile b/modules/local/linxreport/Dockerfile index 963f2f5d..60850abb 100644 --- a/modules/local/linxreport/Dockerfile +++ b/modules/local/linxreport/Dockerfile @@ -7,7 +7,7 @@ RUN \ zlib && \ micromamba clean --all --yes -FROM quay.io/biocontainers/r-linxreport:1.0.0--r43hdfd78af_0 +FROM quay.io/biocontainers/r-linxreport:1.1.0--r44hdfd78af_0 COPY --from=build /env/ /env/ diff --git a/modules/local/linxreport/environment.yml b/modules/local/linxreport/environment.yml new file mode 100644 index 00000000..71113340 --- /dev/null +++ b/modules/local/linxreport/environment.yml @@ -0,0 +1,7 @@ +name: linxreport +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::r-linxreport=1.1.0 diff --git a/modules/local/linxreport/main.nf b/modules/local/linxreport/main.nf index fc28fcfd..4b9ab63a 100644 --- a/modules/local/linxreport/main.nf +++ b/modules/local/linxreport/main.nf @@ -2,7 +2,7 @@ process LINXREPORT { tag "${meta.id}" label 'process_single' - container 'docker.io/scwatts/r-linxreport:1.0.0--0' + container 'docker.io/scwatts/r-linxreport:1.1.0--0' input: tuple val(meta), path(linx_annotation_dir), path(linx_visualiser_dir) @@ -17,22 +17,24 @@ process LINXREPORT { script: def args = task.ext.args ?: '' + def plot_dir = linx_visualiser_dir.resolve('all/').toUriString() + """ - # Create input directory if it doesn't exist for linxreport - if [[ ! -e ${linx_visualiser_dir} ]]; then - mkdir -p ${linx_visualiser_dir}; + # Set input plot directory and create it doesn't exist. See the LINX visualiser module for further info. + if [[ ! -e ${plot_dir} ]]; then + mkdir -p ${plot_dir}; fi; linxreport.R \\ ${args} \\ --sample ${meta.sample_id} \\ - --plot ${linx_visualiser_dir} \\ + --plot ${plot_dir} \\ --table ${linx_annotation_dir} \\ --out ${meta.sample_id}_linx.html cat <<-END_VERSIONS > versions.yml "${task.process}": - R: \$(R --version | head -n1 | sed 's/^R version \\([0-9.]\\+\\).\\+/\\1/') + r: \$(R --version | head -n1 | sed 's/^R version \\([0-9.]\\+\\).\\+/\\1/') linxreport: \$(linxreport.R --version) END_VERSIONS """ @@ -40,6 +42,7 @@ process LINXREPORT { stub: """ touch ${meta.sample_id}_linx.html + echo -e '${task.process}:\n stub: noversions\n' > versions.yml """ } diff --git a/modules/local/purple/Dockerfile b/modules/local/purple/Dockerfile index 26879a24..1cda1921 100644 --- a/modules/local/purple/Dockerfile +++ b/modules/local/purple/Dockerfile @@ -7,7 +7,7 @@ RUN \ zlib && \ micromamba clean --all --yes -FROM quay.io/biocontainers/hmftools-purple:4.0.2--hdfd78af_0 +FROM quay.io/biocontainers/hmftools-purple:4.1--hdfd78af_0 # NOTE(SW): copy over compatibile build of BCFtools COPY --from=quay.io/biocontainers/bcftools:1.19--h8b25389_0 /usr/local/ /usr/local/ diff --git a/modules/local/purple/environment.yml b/modules/local/purple/environment.yml new file mode 100644 index 00000000..fb18f1e6 --- /dev/null +++ b/modules/local/purple/environment.yml @@ -0,0 +1,7 @@ +name: purple +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::hmftools-purple=4.1 From ab5a21034736a36a926abfe49526a9c724304ac9 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 30 Apr 2025 16:32:16 +1000 Subject: [PATCH 14/88] fix linx somatic --- modules/local/linx/somatic/main.nf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/local/linx/somatic/main.nf b/modules/local/linx/somatic/main.nf index e9d5426d..c6d7041d 100644 --- a/modules/local/linx/somatic/main.nf +++ b/modules/local/linx/somatic/main.nf @@ -10,6 +10,8 @@ process LINX_SOMATIC { path ensembl_data_resources path known_fusion_data path driver_gene_panel + path gene_id_file + output: tuple val(meta), path('linx_somatic/'), emit: annotation_dir @@ -20,6 +22,8 @@ process LINX_SOMATIC { script: def args = task.ext.args ?: '' + def gene_id_file_arg = gene_id_file ? "-gene_id_file ${gene_id_file}" : '' + def xmx_mod = task.ext.xmx_mod ?: 0.75 @@ -30,6 +34,7 @@ process LINX_SOMATIC { -sample ${meta.sample_id} \\ -sv_vcf ${purple_dir}/${meta.sample_id}.purple.sv.vcf.gz \\ -purple_dir ${purple_dir} \\ + ${gene_id_file_arg} \\ -ref_genome_version ${genome_ver} \\ -ensembl_data_dir ${ensembl_data_resources} \\ -known_fusion_file ${known_fusion_data} \\ From cd22e5ec3644944bb30e09f209152c815f9c6301 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 1 May 2025 11:40:48 +1000 Subject: [PATCH 15/88] fix purple input doesn't take sv_tumor_unfiltered_vcf --- modules/local/purple/main.nf | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/local/purple/main.nf b/modules/local/purple/main.nf index 706130eb..f267b00c 100644 --- a/modules/local/purple/main.nf +++ b/modules/local/purple/main.nf @@ -39,8 +39,6 @@ process PURPLE { def sv_tumor_vcf_arg = sv_tumor_vcf ? "-somatic_sv_vcf ${sv_tumor_vcf}" : '' def sv_normal_vcf_arg = sv_normal_vcf ? "-germline_sv_vcf ${sv_normal_vcf}" : '' - def sv_tumor_recovery_vcf_arg = sv_tumor_unfiltered_vcf ? "-sv_recovery_vcf ${sv_tumor_unfiltered_vcf}" : '' - // NOTE(SW): use of 'smlv_tumor.vcf.gz' is intended here; see comment below in script block def smlv_tumor_vcf_arg = smlv_tumor_vcf ? "-somatic_vcf smlv_tumor.vcf.gz" : '' def smlv_normal_vcf_arg = smlv_normal_vcf ? "-germline_vcf ${smlv_normal_vcf}" : '' @@ -77,7 +75,6 @@ process PURPLE { -cobalt ${cobalt} \\ ${sv_tumor_vcf_arg} \\ ${sv_normal_vcf_arg} \\ - ${sv_tumor_recovery_vcf_arg} \\ ${smlv_tumor_vcf_arg} \\ ${smlv_normal_vcf_arg} \\ -ref_genome ${genome_fasta} \\ From a0a2bbdb1f857ac318673f9effff077a9a4d0a23 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 1 May 2025 15:30:49 +1000 Subject: [PATCH 16/88] change dicker image bolt --- modules/local/bolt/sv_somatic/prioritise/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bolt/sv_somatic/prioritise/main.nf b/modules/local/bolt/sv_somatic/prioritise/main.nf index d2c0b7e9..4ac07c7e 100644 --- a/modules/local/bolt/sv_somatic/prioritise/main.nf +++ b/modules/local/bolt/sv_somatic/prioritise/main.nf @@ -2,7 +2,7 @@ process BOLT_SV_SOMATIC_PRIORITISE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13' + container 'docker.io/qclayssen/bolt:oncoanalyser-v2' input: tuple val(meta), path(sv_vcf) From 736c478ab0e39ea2c4f59be67f8b66e51037d59b Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 1 May 2025 15:52:55 +1000 Subject: [PATCH 17/88] fix docker path --- modules/local/linx/germline/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/linx/germline/main.nf b/modules/local/linx/germline/main.nf index c3b436db..88c8c56e 100644 --- a/modules/local/linx/germline/main.nf +++ b/modules/local/linx/germline/main.nf @@ -2,7 +2,7 @@ process LINX_GERMLINE { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/hmftools-linx:2.0--10' + container 'docker.io/qclayssen/hmftools-linx:2.0--1' input: tuple val(meta), path(sv_vcf) From 48a153c244eeb124e1c00b266d925b05329f1c3c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 1 May 2025 15:59:48 +1000 Subject: [PATCH 18/88] chane docker path linx report --- modules/local/linxreport/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/linxreport/main.nf b/modules/local/linxreport/main.nf index 4b9ab63a..957407c1 100644 --- a/modules/local/linxreport/main.nf +++ b/modules/local/linxreport/main.nf @@ -2,7 +2,7 @@ process LINXREPORT { tag "${meta.id}" label 'process_single' - container 'docker.io/scwatts/r-linxreport:1.1.0--0' + container 'docker.io/qclayssen/r-linxreport:1.1.0--0' input: tuple val(meta), path(linx_annotation_dir), path(linx_visualiser_dir) From af2e69c6077b94838212dba87c74c9b3a6c9a554 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 5 May 2025 09:42:44 +1000 Subject: [PATCH 19/88] change add eSVee call --- conf/modules.config | 8 +++++++ subworkflows/local/prepare_input.nf | 22 +++++++------------ workflows/sash.nf | 33 ++++++++++++++++++++++++----- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 04c86aec..da15bd78 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -171,6 +171,14 @@ process { ] } + withName: 'ESVEE_CALL' { + publishDir = [ + path: { "${params.outdir}/esvee/" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/${filename}" } + ] + } + withName: '.*:LINX_ANNOTATION:(?:GERMLINE|SOMATIC)|.*:LINX_PLOTTING:VISUALISER' { ext.jarPath = '/opt/linx/linx.jar' } diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 92c97af9..8c46a015 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -49,21 +49,17 @@ workflow PREPARE_INPUT { def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/cobalt/"] } - def ch_esvee_somatic = ch_metas.map { meta -> + def ref_depth_vcf = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() - def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.somatic.vcf.gz" + def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } - def ch_esvee_germline = ch_metas.map { meta -> + def prep_dir = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() - def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.germline.vcf.gz" - return [meta, vcf, "${vcf}.tbi"] - } - def ch_esvee_somatic_unfiltered = ch_metas.map { meta -> - def base = file(meta.oncoanalyser_dir).toUriString() - def vcf = "${base}/esvee/caller/${meta.tumor_id}.esvee.unfiltered.vcf.gz" - return [meta, vcf, "${vcf}.tbi"] + def dir = "${base}/esvee/prep/" + return [meta, dir] } + def ch_sage_somatic = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/sage/somatic/${meta.tumor_id}.sage.somatic.vcf.gz" @@ -86,17 +82,15 @@ workflow PREPARE_INPUT { def vcf = "${base}/${meta.tumor_id}.hard-filtered.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } - emit: metas = ch_metas amber = ch_amber cobalt = ch_cobalt - esvee_somatic = ch_esvee_somatic - esvee_germline = ch_esvee_germline - esvee_somatic_unfiltered = ch_esvee_somatic_unfiltered sage_somatic = ch_sage_somatic virusbreakend = ch_virusbreakend hrd = ch_input_hrd vcf_germline = ch_input_vcf_germline vcf_somatic = ch_input_vcf_somatic + ref_depth_vcf = ref_depth_vcf + prep_dir = prep_dir } diff --git a/workflows/sash.nf b/workflows/sash.nf index 2336e5d2..50844365 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -47,8 +47,8 @@ include { BOLT_SMLV_SOMATIC_RESCUE } from '../modules/local/bolt/smlv_somatic/ include { BOLT_SV_SOMATIC_ANNOTATE } from '../modules/local/bolt/sv_somatic/annotate/main' include { BOLT_SV_SOMATIC_PRIORITISE } from '../modules/local/bolt/sv_somatic/prioritise/main' include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' +include { ESVEE_CALL } from '../modules/local/esvee/call/main' -include { GRIPSS_FILTERING } from '../subworkflows/local/gripss_filtering' include { LINX_ANNOTATION } from '../subworkflows/local/linx_annotation' include { LINX_PLOTTING } from '../subworkflows/local/linx_plotting' include { PREPARE_INPUT } from '../subworkflows/local/prepare_input' @@ -89,14 +89,13 @@ workflow SASH { ch_amber = PREPARE_INPUT.out.amber ch_cobalt = PREPARE_INPUT.out.cobalt - ch_esvee_somatic = PREPARE_INPUT.out.esvee_somatic - ch_esvee_germline = PREPARE_INPUT.out.esvee_germline - ch_esvee_somatic_unfiltered = PREPARE_INPUT.out.esvee_somatic_unfiltered ch_sage_somatic = PREPARE_INPUT.out.sage_somatic ch_virusbreakend = PREPARE_INPUT.out.virusbreakend ch_input_hrd = PREPARE_INPUT.out.hrd ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic + ch_ref_depth_vcf = PREPARE_INPUT.out.ref_depth_vcf + ch_prep_dir = PREPARE_INPUT.out.prep_dir @@ -243,8 +242,33 @@ workflow SASH { hmf_data.purple_germline_del, ) + // + // Somatic structural variants + + // + // MODULE: ESVEE call somatic structural variants + // + // Create process input channel + // channel: [meta_esvee, ref_depth_vcf, prep_dir] + ch_call_inputs = WorkflowOncoanalyser.groupByMeta( + ch_ref_depth_vcf, + ch_prep_dir, + ) + //TODO update ref data + ESVEE_CALL( + ch_call_inputs, + genome.fasta, + genome.version, + hmf_data.gridss_pon_breakends, + hmf_data.gridss_pon_breakpoints, + umccr_data.known_fusions, + hmf_data.repeatmasker_annotations, + ) + ch_esvee_somatic = ESVEE_CALL.out.somatic_vcf + ch_esvee_germline = ESVEE_CALL.out.germline_vcf + ch_esvee_somatic_unfiltered = ESVEE_CALL.out.unfiltered_vcf // // Small variant reporting (PCGR, CPSR, stats) @@ -370,7 +394,6 @@ workflow SASH { - // // Generate custom PURPLE β-allele frequency circos plot // From 4492f569d29e6a8a518d9582f6cd33d777807f7d Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 6 May 2025 17:05:43 +1000 Subject: [PATCH 20/88] fix name channel --- subworkflows/local/prepare_input.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 8c46a015..6d1e9eba 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -49,12 +49,12 @@ workflow PREPARE_INPUT { def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/cobalt/"] } - def ref_depth_vcf = ch_metas.map { meta -> + def ch_ref_depth_vcf = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } - def prep_dir = ch_metas.map { meta -> + def chprep_dir = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def dir = "${base}/esvee/prep/" return [meta, dir] @@ -91,6 +91,6 @@ workflow PREPARE_INPUT { hrd = ch_input_hrd vcf_germline = ch_input_vcf_germline vcf_somatic = ch_input_vcf_somatic - ref_depth_vcf = ref_depth_vcf - prep_dir = prep_dir + ref_depth_vcf = ch_ref_depth_vcf + prep_dir = ch_prep_dir } From c084de20112f40a001dad86ddb93a52698135304 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 6 May 2025 17:09:35 +1000 Subject: [PATCH 21/88] fix typo --- subworkflows/local/prepare_input.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 6d1e9eba..8002a969 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -54,7 +54,7 @@ workflow PREPARE_INPUT { def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } - def chprep_dir = ch_metas.map { meta -> + def ch_prep_dir = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def dir = "${base}/esvee/prep/" return [meta, dir] From d27f6ee5920c5b137792121e839c61eb4d7beff1 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 6 May 2025 17:16:23 +1000 Subject: [PATCH 22/88] fix fonction call --- workflows/sash.nf | 55 +++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 50844365..fdafe4a6 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -209,6 +209,33 @@ workflow SASH { // channel: [ meta, smlv_germline_vcf ] ch_smlv_germline_out = WorkflowSash.restoreMeta(BOLT_SMLV_GERMLINE_PREPARE.out.vcf, ch_inputs) + // + // Somatic structural variants + + // + // MODULE: ESVEE call somatic structural variants + // + // Create process input channel + // channel: [meta_esvee, ref_depth_vcf, prep_dir] + ch_call_inputs = WorkflowSash.groupByMeta( + ch_ref_depth_vcf, + ch_prep_dir, + ) + + //TODO update ref data + ESVEE_CALL( + ch_call_inputs, + genome.fasta, + genome.version, + hmf_data.gridss_pon_breakends, + hmf_data.gridss_pon_breakpoints, + umccr_data.known_fusions, + hmf_data.repeatmasker_annotations, + ) + + ch_esvee_somatic = ESVEE_CALL.out.somatic_vcf + ch_esvee_germline = ESVEE_CALL.out.germline_vcf + ch_esvee_somatic_unfiltered = ESVEE_CALL.out.unfiltered_vcf // // CNV calling using UMCCR postprocessed variants @@ -242,34 +269,6 @@ workflow SASH { hmf_data.purple_germline_del, ) - // - // Somatic structural variants - - // - // MODULE: ESVEE call somatic structural variants - // - // Create process input channel - // channel: [meta_esvee, ref_depth_vcf, prep_dir] - ch_call_inputs = WorkflowOncoanalyser.groupByMeta( - ch_ref_depth_vcf, - ch_prep_dir, - ) - - //TODO update ref data - ESVEE_CALL( - ch_call_inputs, - genome.fasta, - genome.version, - hmf_data.gridss_pon_breakends, - hmf_data.gridss_pon_breakpoints, - umccr_data.known_fusions, - hmf_data.repeatmasker_annotations, - ) - - ch_esvee_somatic = ESVEE_CALL.out.somatic_vcf - ch_esvee_germline = ESVEE_CALL.out.germline_vcf - ch_esvee_somatic_unfiltered = ESVEE_CALL.out.unfiltered_vcf - // // Small variant reporting (PCGR, CPSR, stats) // From 1d239057d38016a68b552877c06ecaaa2ba09397 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 6 May 2025 17:38:23 +1000 Subject: [PATCH 23/88] remove index esvee channel --- subworkflows/local/prepare_input.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 8002a969..d9cdbc6d 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -52,7 +52,7 @@ workflow PREPARE_INPUT { def ch_ref_depth_vcf = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" - return [meta, vcf, "${vcf}.tbi"] + return [meta, vcf] } def ch_prep_dir = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() From ae3becdfa40b1c5c0dfa90e01c7f6514ebaa6d83 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 7 May 2025 09:09:30 +1000 Subject: [PATCH 24/88] add dockerfile eSVee --- modules/local/esvee/call/Dockerfile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 modules/local/esvee/call/Dockerfile diff --git a/modules/local/esvee/call/Dockerfile b/modules/local/esvee/call/Dockerfile new file mode 100644 index 00000000..bd0796ca --- /dev/null +++ b/modules/local/esvee/call/Dockerfile @@ -0,0 +1,15 @@ +FROM mambaorg/micromamba:1.5.7 as build + +USER root + +RUN \ + micromamba create -y -p /env/ -c bioconda -c conda-forge \ + zlib && \ + micromamba clean --all --yes + +FROM quay.io/biocontainers/hmftools-esvee:1.0.3--hdfd78af_0 + +COPY --from=build /env/ /env/ + +ENV PATH="/env/bin:${PATH}" +ENV LD_LIBRARY_PATH="/env/lib/:${LD_LIBRARY_PATH}" From 015d5ab8d27ffd3e9d1f66ee21b53cc3174507eb Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 7 May 2025 09:14:26 +1000 Subject: [PATCH 25/88] change container esvee --- modules/local/esvee/call/main.nf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/local/esvee/call/main.nf b/modules/local/esvee/call/main.nf index a6480a18..1447e79c 100644 --- a/modules/local/esvee/call/main.nf +++ b/modules/local/esvee/call/main.nf @@ -2,10 +2,8 @@ process ESVEE_CALL { tag "${meta.id}" label 'process_high' - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/hmftools-esvee:1.0.3--hdfd78af_0' : - 'biocontainers/hmftools-esvee:1.0.3--hdfd78af_0' }" + container 'docker.io/qclayssen/esvee:1.0.3' + input: tuple val(meta), path(ref_depth_vcf), path(prep_dir) From 6bf2d129a3a0f93285d6a406d4c2147e1661a12e Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 7 May 2025 09:21:59 +1000 Subject: [PATCH 26/88] remove gripss filtering subworkflow and modules --- conf/modules.config | 4 -- subworkflows/local/gripss_filtering.nf | 90 -------------------------- 2 files changed, 94 deletions(-) delete mode 100644 subworkflows/local/gripss_filtering.nf diff --git a/conf/modules.config b/conf/modules.config index da15bd78..5b78059e 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -98,10 +98,6 @@ process { ] } - withName: '.*:GRIPSS_FILTERING:(?:GERMLINE|SOMATIC)' { - ext.jarPath = '/opt/gripss/gripss.jar' - } - withName: 'BOLT_SV_SOMATIC_ANNOTATE' { publishDir = [ path: { "${params.outdir}" }, diff --git a/subworkflows/local/gripss_filtering.nf b/subworkflows/local/gripss_filtering.nf deleted file mode 100644 index cb76a0c3..00000000 --- a/subworkflows/local/gripss_filtering.nf +++ /dev/null @@ -1,90 +0,0 @@ -// -// GRIPSS performs SV filtering. -// - -include { GRIPSS_GERMLINE as GERMLINE } from '../../modules/local/gripss/germline/main' -include { GRIPSS_SOMATIC as SOMATIC } from '../../modules/local/gripss/somatic/main' - -workflow GRIPSS_FILTERING { - take: - // Sample inputs - ch_inputs // channel: [mandatory] [ meta ] - ch_gridss // channel: [mandatory] [ meta, gridss_vcf ] - - // Reference data - genome_fasta // channel: [mandatory] /path/to/genome_fasta - genome_version // channel: [mandatory] genome version - genome_fai // channel: [mandatory] /path/to/genome_fai - breakend_pon // channel: [mandatory] /path/to/breakend_pon - breakpoint_pon // channel: [mandatory] /path/to/breakpoint_pon - known_fusions // channel: [mandatory] /path/to/known_fusions - repeatmasker_annotations // channel: [mandatory] /path/to/repeatmasker_annotations - - main: - // Channel for version.yml files - // channel: [ versions.yml ] - ch_versions = Channel.empty() - - // Create process input channel - // channel: [ meta_gripss, gridss_vcf ] - ch_gripss_inputs = ch_gridss - .map { meta, gridss_vcf -> - - def meta_gripss = [ - key: meta.id, - id: meta.id, - tumor_id: meta.tumor_id, - normal_id: meta.normal_id, - ] - - return [meta_gripss, gridss_vcf] - } - - // - // MODULE: GRIPSS germline - // - GERMLINE( - ch_gripss_inputs, - genome_fasta, - genome_version, - genome_fai, - breakend_pon, - breakpoint_pon, - known_fusions, - repeatmasker_annotations, - ) - - ch_versions = ch_versions.mix(GERMLINE.out.versions) - - // - // MODULE: GRIPSS somatic - // - SOMATIC( - ch_gripss_inputs, - genome_fasta, - genome_version, - genome_fai, - breakend_pon, - breakpoint_pon, - known_fusions, - repeatmasker_annotations, - [], - ) - - ch_versions = ch_versions.mix(SOMATIC.out.versions) - - // Set outputs, restoring original meta - // channel: [ meta, gripss_vcf, gripss_tbi ] - ch_somatic_out = WorkflowSash.restoreMeta(SOMATIC.out.vcf, ch_inputs) - ch_somatic_unfiltered_out = WorkflowSash.restoreMeta(SOMATIC.out.vcf_unfiltered, ch_inputs) - ch_germline_out = WorkflowSash.restoreMeta(GERMLINE.out.vcf, ch_inputs) - ch_germline_unfiltered_out = WorkflowSash.restoreMeta(GERMLINE.out.vcf_unfiltered, ch_inputs) - - emit: - somatic = ch_somatic_out // channel: [ meta, gripss_vcf, gripss_tbi ] - germline = ch_germline_out // channel: [ meta, gripss_vcf, gripss_tbi ] - somatic_unfiltered = ch_somatic_unfiltered_out // channel: [ meta, gripss_vcf, gripss_tbi ] - germline_unfiltered = ch_germline_unfiltered_out // channel: [ meta, gripss_vcf, gripss_tbi ] - - versions = ch_versions // channel: [ versions.yml ] -} From 3996a977ef4e4eaa49e7091e64ad70eac503dd64 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 7 May 2025 13:51:44 +1000 Subject: [PATCH 27/88] change docker image cancer report --- modules/local/bolt/other/cancer_report/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index 4690eda8..847013e7 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-gpgr' + container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) From 81ebfe512192cb5fea4efa02dca5a0865f632e1b Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 7 May 2025 13:51:57 +1000 Subject: [PATCH 28/88] fix output path esvee --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 5b78059e..5f92b07a 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -171,7 +171,7 @@ process { publishDir = [ path: { "${params.outdir}/esvee/" }, mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/${filename}" } + saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/call/" } ] } @@ -195,7 +195,7 @@ process { ] } - withName: '.*:LINX_PLOTTING:VISUALISER' { + withName: '.*:LINX_PLOTTING:VISUALISER' { ext.circosPath = '/opt/conda/bin/circos' publishDir = [ path: { "${params.outdir}" }, From 3b9e0ab05e6f8742a1d1e96420fcc1dd1a876110 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 9 May 2025 11:31:29 +1000 Subject: [PATCH 29/88] fix output path esvee --- conf/modules.config | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 5f92b07a..5a19d566 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -171,7 +171,10 @@ process { publishDir = [ path: { "${params.outdir}/esvee/" }, mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/call/" } + saveAs: { fp -> + def fp_out = fp.replaceFirst(/caller\//, '') + return fp.equals('versions.yml') ? null : "${meta.key}/esvee/call/${fp_out}" + }, ] } From db8a53f658373fa6fca8a6f8909b510bfc935e61 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 9 May 2025 15:01:56 +1000 Subject: [PATCH 30/88] fix output path eSVee --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 5a19d566..d2696252 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -169,7 +169,7 @@ process { withName: 'ESVEE_CALL' { publishDir = [ - path: { "${params.outdir}/esvee/" }, + path: { "${params.outdir}" }, mode: params.publish_dir_mode, saveAs: { fp -> def fp_out = fp.replaceFirst(/caller\//, '') From b97feb565960507478572fdc6f48f64dc8626f58 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 9 May 2025 15:38:52 +1000 Subject: [PATCH 31/88] fix name output path esvee --- conf/modules.config | 5 +---- subworkflows/local/purple_calling.nf | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index d2696252..c8777ec1 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -171,10 +171,7 @@ process { publishDir = [ path: { "${params.outdir}" }, mode: params.publish_dir_mode, - saveAs: { fp -> - def fp_out = fp.replaceFirst(/caller\//, '') - return fp.equals('versions.yml') ? null : "${meta.key}/esvee/call/${fp_out}" - }, + saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/${filename}" } ] } diff --git a/subworkflows/local/purple_calling.nf b/subworkflows/local/purple_calling.nf index 6bd79424..1db09041 100644 --- a/subworkflows/local/purple_calling.nf +++ b/subworkflows/local/purple_calling.nf @@ -12,9 +12,9 @@ workflow PURPLE_CALLING { ch_cobalt // channel: [mandatory] [ meta, cobalt_dir ] ch_smlv_somatic // channel: [optional] [ meta, pave_vcf ] ch_smlv_germline // channel: [optional] [ meta, pave_vcf ] - ch_sv_somatic // channel: [optional] [ meta, gripss_vcf, gripss_tbi ] - ch_sv_germline // channel: [optional] [ meta, gripss_vcf, gripss_tbi ] - ch_sv_somatic_unfiltered // channel: [optional] [ meta, gripss_vcf, gripss_tbi ] + ch_sv_somatic // channel: [optional] [ meta, esvee_vcf, esvee_tbi ] + ch_sv_germline // channel: [optional] [ meta, esvee_vcf, esvee_tbi ] + ch_sv_somatic_unfiltered // channel: [optional] [ meta, esvee_vcf, esvee_tbi ] // Reference data genome_fasta // channel: [mandatory] /path/to/genome_fasta From 9bd7ad2954de4df5127f7d128198cf62c0df304d Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 01:11:32 +0000 Subject: [PATCH 32/88] latest docker images --- modules/local/bolt/other/cancer_report/main.nf | 2 +- modules/local/bolt/sv_somatic/prioritise/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index 847013e7..778ad080 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee' + container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-2' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) diff --git a/modules/local/bolt/sv_somatic/prioritise/main.nf b/modules/local/bolt/sv_somatic/prioritise/main.nf index 4ac07c7e..0a59abe6 100644 --- a/modules/local/bolt/sv_somatic/prioritise/main.nf +++ b/modules/local/bolt/sv_somatic/prioritise/main.nf @@ -2,7 +2,7 @@ process BOLT_SV_SOMATIC_PRIORITISE { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:oncoanalyser-v2' + container 'docker.io/qclayssen/bolt:oncoanalyser-v2-2' input: tuple val(meta), path(sv_vcf) From 82a6d2e6a4cd19644db49e9112c889cffe13fceb Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 13:21:22 +1000 Subject: [PATCH 33/88] remove purple conda env, and deprecated input --- modules/local/esvee/call/environment.yml | 7 ------- modules/local/purple/main.nf | 3 +-- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 modules/local/esvee/call/environment.yml diff --git a/modules/local/esvee/call/environment.yml b/modules/local/esvee/call/environment.yml deleted file mode 100644 index 01355c11..00000000 --- a/modules/local/esvee/call/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: esvee_call -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::hmftools-esvee=1.0.3 diff --git a/modules/local/purple/main.nf b/modules/local/purple/main.nf index f267b00c..29512207 100644 --- a/modules/local/purple/main.nf +++ b/modules/local/purple/main.nf @@ -2,11 +2,10 @@ process PURPLE { tag "${meta.id}" label 'process_medium' - conda "${moduleDir}/environment.yml" container 'docker.io/qclayssen/purple:4.1.0--1' input: - tuple val(meta), path(amber), path(cobalt), path(sv_tumor_vcf), path(sv_tumor_tbi), path(sv_tumor_unfiltered_vcf), path(sv_tumor_unfiltered_tbi), path(sv_normal_vcf), path(sv_normal_tbi), path(smlv_tumor_vcf), path(smlv_normal_vcf) + tuple val(meta), path(amber), path(cobalt), path(sv_tumor_vcf), path(sv_tumor_tbi), path(sv_normal_vcf), path(sv_normal_tbi), path(smlv_tumor_vcf), path(smlv_normal_vcf) path genome_fasta val genome_ver path genome_fai From 7e9398e6d8d48935754fdaedfb1e6b6935420f14 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 13:21:42 +1000 Subject: [PATCH 34/88] tidy up --- workflows/sash.nf | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index fdafe4a6..ada5dc28 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -133,7 +133,7 @@ workflow SASH { tumor_id: meta.tumor_id, normal_id: meta.normal_id, ] - return [meta_bolt] + it[1..-1] + return [meta_bolt], *it[1..-1] } BOLT_SMLV_SOMATIC_RESCUE( @@ -222,7 +222,6 @@ workflow SASH { ch_prep_dir, ) - //TODO update ref data ESVEE_CALL( ch_call_inputs, genome.fasta, @@ -235,7 +234,6 @@ workflow SASH { ch_esvee_somatic = ESVEE_CALL.out.somatic_vcf ch_esvee_germline = ESVEE_CALL.out.germline_vcf - ch_esvee_somatic_unfiltered = ESVEE_CALL.out.unfiltered_vcf // // CNV calling using UMCCR postprocessed variants @@ -254,9 +252,8 @@ workflow SASH { // * https://github.com/hartwigmedical/hmftools/blob/a2f82e5/purple/src/main/java/com/hartwig/hmftools/purple/germline/GermlineGenotypeEnrichment.java#L63 //ch_smlv_germline_out, ch_smlv_germline_out.map { meta, vcf -> return [meta, []] }, - ch_esvee_somatic, - ch_esvee_germline, - ch_esvee_somatic_unfiltered, + ESVEE_CALL.out.somatic_vcf, + ESVEE_CALL.out.germline_vcfe, genome.fasta, genome.version, genome.fai, @@ -451,7 +448,7 @@ workflow SASH { subject_id: meta.subject_id, tumor_id: meta.tumor_id, ] - return [meta_bolt] + it[1..-1] + return [meta_bolt], *it[1..-1] } BOLT_OTHER_CANCER_REPORT( From 03ca27ddd97875c5ec869b9a8e78e46814b8c62c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 13:21:58 +1000 Subject: [PATCH 35/88] fix subworkflow deprecated input purple --- subworkflows/local/purple_calling.nf | 2 -- 1 file changed, 2 deletions(-) diff --git a/subworkflows/local/purple_calling.nf b/subworkflows/local/purple_calling.nf index 1db09041..ef73140c 100644 --- a/subworkflows/local/purple_calling.nf +++ b/subworkflows/local/purple_calling.nf @@ -14,7 +14,6 @@ workflow PURPLE_CALLING { ch_smlv_germline // channel: [optional] [ meta, pave_vcf ] ch_sv_somatic // channel: [optional] [ meta, esvee_vcf, esvee_tbi ] ch_sv_germline // channel: [optional] [ meta, esvee_vcf, esvee_tbi ] - ch_sv_somatic_unfiltered // channel: [optional] [ meta, esvee_vcf, esvee_tbi ] // Reference data genome_fasta // channel: [mandatory] /path/to/genome_fasta @@ -41,7 +40,6 @@ workflow PURPLE_CALLING { ch_cobalt, // Optional inputs ch_sv_somatic, - ch_sv_somatic_unfiltered, ch_sv_germline, ch_smlv_somatic, ch_smlv_germline, From 9f1528e1a41ef8e5c93b95944a0d878936df644d Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 13:27:35 +1000 Subject: [PATCH 36/88] linting --- workflows/sash.nf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index ada5dc28..e9025eae 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -46,8 +46,8 @@ include { BOLT_SMLV_SOMATIC_REPORT } from '../modules/local/bolt/smlv_somatic/ include { BOLT_SMLV_SOMATIC_RESCUE } from '../modules/local/bolt/smlv_somatic/rescue/main' include { BOLT_SV_SOMATIC_ANNOTATE } from '../modules/local/bolt/sv_somatic/annotate/main' include { BOLT_SV_SOMATIC_PRIORITISE } from '../modules/local/bolt/sv_somatic/prioritise/main' -include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' include { ESVEE_CALL } from '../modules/local/esvee/call/main' +include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' include { LINX_ANNOTATION } from '../subworkflows/local/linx_annotation' include { LINX_PLOTTING } from '../subworkflows/local/linx_plotting' @@ -91,11 +91,11 @@ workflow SASH { ch_cobalt = PREPARE_INPUT.out.cobalt ch_sage_somatic = PREPARE_INPUT.out.sage_somatic ch_virusbreakend = PREPARE_INPUT.out.virusbreakend - ch_input_hrd = PREPARE_INPUT.out.hrd + ch_input_hrd = PREPARE_INPUT.out.hrd ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic - ch_ref_depth_vcf = PREPARE_INPUT.out.ref_depth_vcf - ch_prep_dir = PREPARE_INPUT.out.prep_dir + ch_ref_depth_vcf = PREPARE_INPUT.out.ref_depth_vcf + ch_prep_dir = PREPARE_INPUT.out.prep_dir @@ -106,10 +106,10 @@ workflow SASH { // channel: [ meta ] PREPARE_REFERENCE() - genome = PREPARE_REFERENCE.out.genome - umccr_data = PREPARE_REFERENCE.out.umccr_data - hmf_data = PREPARE_REFERENCE.out.hmf_data - misc_data = PREPARE_REFERENCE.out.misc_data + genome = PREPARE_REFERENCE.out.genome + umccr_data = PREPARE_REFERENCE.out.umccr_data + hmf_data = PREPARE_REFERENCE.out.hmf_data + misc_data = PREPARE_REFERENCE.out.misc_data From 754e7c3e83367bd4bcdf0ca5216bcf69aa593e5c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 13:59:56 +1000 Subject: [PATCH 37/88] tidy up --- workflows/sash.nf | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index e9025eae..ae77b71b 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -209,14 +209,13 @@ workflow SASH { // channel: [ meta, smlv_germline_vcf ] ch_smlv_germline_out = WorkflowSash.restoreMeta(BOLT_SMLV_GERMLINE_PREPARE.out.vcf, ch_inputs) + + + // // Somatic structural variants // - // MODULE: ESVEE call somatic structural variants - // - // Create process input channel - // channel: [meta_esvee, ref_depth_vcf, prep_dir] ch_call_inputs = WorkflowSash.groupByMeta( ch_ref_depth_vcf, ch_prep_dir, @@ -232,9 +231,6 @@ workflow SASH { hmf_data.repeatmasker_annotations, ) - ch_esvee_somatic = ESVEE_CALL.out.somatic_vcf - ch_esvee_germline = ESVEE_CALL.out.germline_vcf - // // CNV calling using UMCCR postprocessed variants // From aad37a72a60babf4bd2d933444c87c4afa0d6e7a Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 14:29:37 +1000 Subject: [PATCH 38/88] Uptade linx report for sash --- modules/local/linx/somatic/main.nf | 2 -- modules/local/linx/visualiser/main.nf | 24 ------------------------ 2 files changed, 26 deletions(-) diff --git a/modules/local/linx/somatic/main.nf b/modules/local/linx/somatic/main.nf index c6d7041d..e96e5d8e 100644 --- a/modules/local/linx/somatic/main.nf +++ b/modules/local/linx/somatic/main.nf @@ -12,7 +12,6 @@ process LINX_SOMATIC { path driver_gene_panel path gene_id_file - output: tuple val(meta), path('linx_somatic/'), emit: annotation_dir path 'versions.yml' , emit: versions @@ -24,7 +23,6 @@ process LINX_SOMATIC { def args = task.ext.args ?: '' def gene_id_file_arg = gene_id_file ? "-gene_id_file ${gene_id_file}" : '' - def xmx_mod = task.ext.xmx_mod ?: 0.75 """ diff --git a/modules/local/linx/visualiser/main.nf b/modules/local/linx/visualiser/main.nf index a40646a3..02b17d98 100644 --- a/modules/local/linx/visualiser/main.nf +++ b/modules/local/linx/visualiser/main.nf @@ -54,30 +54,6 @@ process LINX_VISUALISER { -plot_out plots/all/ \\ -data_out data/all/ - # Rerun LINX to render only reportable cluster plots in a separate directory. While this is regenerating existing - # cluster plots, the number of reportable plots is generally very small and I prefer to rely on the internal LINX - # logic to determine whether a cluster is reportable rather than attempting to infer manually to copy out target - # plot files. - - # The ORANGE report receives only reportable clusters while the gpgr LINX report receives chromosome and all cluster - # plots. - - # https://github.com/hartwigmedical/hmftools/blob/linx-v1.24.1/linx/src/main/java/com/hartwig/hmftools/linx/visualiser/SampleData.java#L220-L236 - - linx \\ - -Xmx${Math.round(task.memory.bytes * xmx_mod)} \\ - com.hartwig.hmftools.linx.visualiser.SvVisualiser \\ - ${args2} \\ - -sample ${meta.sample_id} \\ - -vis_file_dir ${linx_annotation_dir} \\ - -ref_genome_version ${genome_ver} \\ - -ensembl_data_dir ${ensembl_data_resources} \\ - -circos \$(which circos) \\ - -plot_reportable \\ - -threads ${task.cpus} \\ - -plot_out plots/reportable/ \\ - -data_out data/reportable/ - # Create placeholders to force FusionFS to create parent plot directory on S3 if [[ \$(ls plots/ | wc -l) -eq 0 ]]; then touch plots/.keep; From ee35c4ebfae23bce277f2af8fe5f19dc0a6dca9b Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 18:32:58 +1000 Subject: [PATCH 39/88] fix typo --- workflows/sash.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index ae77b71b..f8cc9779 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -133,7 +133,7 @@ workflow SASH { tumor_id: meta.tumor_id, normal_id: meta.normal_id, ] - return [meta_bolt], *it[1..-1] + return [meta_bolt], *it[1..-1]] } BOLT_SMLV_SOMATIC_RESCUE( @@ -444,7 +444,7 @@ workflow SASH { subject_id: meta.subject_id, tumor_id: meta.tumor_id, ] - return [meta_bolt], *it[1..-1] + return [meta_bolt], *it[1..-1]] } BOLT_OTHER_CANCER_REPORT( From 80081b143acca629e420638c0c481067a0dca53a Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 18:37:14 +1000 Subject: [PATCH 40/88] same typo --- workflows/sash.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index f8cc9779..9ff3c89f 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -133,7 +133,7 @@ workflow SASH { tumor_id: meta.tumor_id, normal_id: meta.normal_id, ] - return [meta_bolt], *it[1..-1]] + return [[meta_bolt], *it[1..-1]] } BOLT_SMLV_SOMATIC_RESCUE( @@ -444,7 +444,7 @@ workflow SASH { subject_id: meta.subject_id, tumor_id: meta.tumor_id, ] - return [meta_bolt], *it[1..-1]] + return [[meta_bolt], *it[1..-1]] } BOLT_OTHER_CANCER_REPORT( From 3a0cae80801373807ec29a202d27b3ef33d8b27c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 12 May 2025 18:38:59 +1000 Subject: [PATCH 41/88] still ... --- workflows/sash.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 9ff3c89f..4da00a7d 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -133,7 +133,7 @@ workflow SASH { tumor_id: meta.tumor_id, normal_id: meta.normal_id, ] - return [[meta_bolt], *it[1..-1]] + return [meta_bolt, *it[1..-1]] } BOLT_SMLV_SOMATIC_RESCUE( @@ -444,7 +444,7 @@ workflow SASH { subject_id: meta.subject_id, tumor_id: meta.tumor_id, ] - return [[meta_bolt], *it[1..-1]] + return [meta_bolt, *it[1..-1]] } BOLT_OTHER_CANCER_REPORT( From 78d9e3c6198fc203cac4ba7ec2b592bd45b0f05a Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 09:55:31 +1000 Subject: [PATCH 42/88] typo --- workflows/sash.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 4da00a7d..ac34d065 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -249,7 +249,7 @@ workflow SASH { //ch_smlv_germline_out, ch_smlv_germline_out.map { meta, vcf -> return [meta, []] }, ESVEE_CALL.out.somatic_vcf, - ESVEE_CALL.out.germline_vcfe, + ESVEE_CALL.out.germline_vcf, genome.fasta, genome.version, genome.fai, From 1bf7155957922fdd1f3c5cce166aaa2e9a705f73 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 11:55:30 +1000 Subject: [PATCH 43/88] touch file for pcgr stub run --- modules/local/bolt/smlv_somatic/report/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/local/bolt/smlv_somatic/report/main.nf b/modules/local/bolt/smlv_somatic/report/main.nf index e293cd4c..c5bbf6b9 100644 --- a/modules/local/bolt/smlv_somatic/report/main.nf +++ b/modules/local/bolt/smlv_somatic/report/main.nf @@ -60,6 +60,7 @@ process BOLT_SMLV_SOMATIC_REPORT { stub: """ mkdir -p output/pcgr/ + touch output/pcgr/pcgr.stub touch output/af_tumor.txt touch output/af_tumor_keygenes.txt touch output/${meta.tumor_id}.somatic.variant_counts_type.yaml From f5852b0882489f30ef4f150b9be821702dfe50b7 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 13:33:06 +1000 Subject: [PATCH 44/88] touch stub file multiqc --- modules/local/bolt/other/multiqc_report/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/local/bolt/other/multiqc_report/main.nf b/modules/local/bolt/other/multiqc_report/main.nf index 5dfb9b11..1f2b3305 100644 --- a/modules/local/bolt/other/multiqc_report/main.nf +++ b/modules/local/bolt/other/multiqc_report/main.nf @@ -35,6 +35,7 @@ process BOLT_OTHER_MULTIQC_REPORT { stub: """ mkdir -p multiqc_report/multiqc_data/ + touch multiqc_report/multiqc.stub touch ${meta.tumor_id}.multiqc.html echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ From 2497ce5c667b88794ad9428ac279ccd9e38f3a59 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 14:07:56 +1000 Subject: [PATCH 45/88] tidy up --- modules/local/bolt/other/multiqc_report/main.nf | 2 +- modules/local/linx/visualiser/main.nf | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/local/bolt/other/multiqc_report/main.nf b/modules/local/bolt/other/multiqc_report/main.nf index 1f2b3305..d35be097 100644 --- a/modules/local/bolt/other/multiqc_report/main.nf +++ b/modules/local/bolt/other/multiqc_report/main.nf @@ -35,7 +35,7 @@ process BOLT_OTHER_MULTIQC_REPORT { stub: """ mkdir -p multiqc_report/multiqc_data/ - touch multiqc_report/multiqc.stub + touch multiqc_report/multiqc_data/multiqc.stub touch ${meta.tumor_id}.multiqc.html echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ diff --git a/modules/local/linx/visualiser/main.nf b/modules/local/linx/visualiser/main.nf index 02b17d98..8eb1ae96 100644 --- a/modules/local/linx/visualiser/main.nf +++ b/modules/local/linx/visualiser/main.nf @@ -18,7 +18,6 @@ process LINX_VISUALISER { script: def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' def xmx_mod = task.ext.xmx_mod ?: 0.75 From d977f2bb4c7a08ff14e34183b481df2aa50a773f Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 14:13:14 +1000 Subject: [PATCH 46/88] test fix out put file eSVee --- workflows/sash.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index ac34d065..c02d7779 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -216,11 +216,11 @@ workflow SASH { // Somatic structural variants // - ch_call_inputs = WorkflowSash.groupByMeta( + call_inputs = WorkflowSash.groupByMeta( ch_ref_depth_vcf, ch_prep_dir, ) - + ch_call_inputs = WorkflowSash.restoreMeta(call_inputs, ch_inputs) ESVEE_CALL( ch_call_inputs, genome.fasta, From 29c0f62e1cbaa3b1b35c9e259d9ecf6adf922a31 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 14:36:28 +1000 Subject: [PATCH 47/88] add more stubs --- modules/local/bolt/other/cancer_report/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index 778ad080..b2c3194d 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -60,6 +60,7 @@ process BOLT_OTHER_CANCER_REPORT { stub: """ mkdir -p output/ + mkdir -p output/cancer_report.stub touch ${meta.tumor_id}.cancer_report.html echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ From 8b837b5427276529d221fba89e2e88ca9fdea4c7 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 14:50:10 +1000 Subject: [PATCH 48/88] test fix ouput path esvee --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c8777ec1..32a24956 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -167,11 +167,11 @@ process { ] } - withName: 'ESVEE_CALL' { + withName: '.*:ESVEE_CALL' { publishDir = [ path: { "${params.outdir}" }, mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/${filename}" } + saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.id}/esvee/${filename}" } ] } From 9611cc68a021535ec7d4b052226b23b82274ab64 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 15:03:47 +1000 Subject: [PATCH 49/88] fix stub ... --- modules/local/bolt/other/cancer_report/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index b2c3194d..feb542be 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -60,7 +60,7 @@ process BOLT_OTHER_CANCER_REPORT { stub: """ mkdir -p output/ - mkdir -p output/cancer_report.stub + touch -p output/cancer_report.stub touch ${meta.tumor_id}.cancer_report.html echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ From f0325e9d5872050431fd9ade11d4f47c365740d0 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 15:14:55 +1000 Subject: [PATCH 50/88] fix typo --- conf/modules.config | 2 +- modules/local/bolt/other/cancer_report/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 32a24956..c67f481e 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -167,7 +167,7 @@ process { ] } - withName: '.*:ESVEE_CALL' { + withName: 'ESVEE_CALL' { publishDir = [ path: { "${params.outdir}" }, mode: params.publish_dir_mode, diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index feb542be..ab41d0bf 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -60,7 +60,7 @@ process BOLT_OTHER_CANCER_REPORT { stub: """ mkdir -p output/ - touch -p output/cancer_report.stub + touch output/cancer_report.stub touch ${meta.tumor_id}.cancer_report.html echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ From 2a096878bd30ab68ccb7e7c72655d7013c92ecbc Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 15:16:39 +1000 Subject: [PATCH 51/88] fix channel --- workflows/sash.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index c02d7779..16d20b54 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -216,11 +216,11 @@ workflow SASH { // Somatic structural variants // - call_inputs = WorkflowSash.groupByMeta( + ch_call_inputs = WorkflowSash.groupByMeta( ch_ref_depth_vcf, - ch_prep_dir, + ch_prep_dir ) - ch_call_inputs = WorkflowSash.restoreMeta(call_inputs, ch_inputs) + ESVEE_CALL( ch_call_inputs, genome.fasta, From 210c5ca1cdb0efa43a0fbf29e89ff71496d4eb4b Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 15:36:13 +1000 Subject: [PATCH 52/88] change in prepare input and add doc --- conf/modules.config | 1 + subworkflows/local/prepare_input.nf | 42 +++++++++++++++++------------ workflows/sash.nf | 27 ++++++++++--------- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c67f481e..821bf08b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -171,6 +171,7 @@ process { publishDir = [ path: { "${params.outdir}" }, mode: params.publish_dir_mode, + // Use meta.id as it's always defined, unlike meta.key which was causing null issues saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.id}/esvee/${filename}" } ] } diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index d9cdbc6d..6ea4bcaa 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -45,19 +45,17 @@ workflow PREPARE_INPUT { def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/amber/"] } + def ch_cobalt = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/cobalt/"] } - def ch_ref_depth_vcf = ch_metas.map { meta -> - def base = file(meta.oncoanalyser_dir).toUriString() - def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" - return [meta, vcf] - } - def ch_prep_dir = ch_metas.map { meta -> + + def ch_call_inputs = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def dir = "${base}/esvee/prep/" - return [meta, dir] + def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" + return [meta, dir, vcf] } def ch_sage_somatic = ch_metas.map { meta -> @@ -65,32 +63,42 @@ workflow PREPARE_INPUT { def vcf = "${base}/sage/somatic/${meta.tumor_id}.sage.somatic.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } + def ch_virusbreakend = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/virusbreakend/"] } + def ch_input_hrd = ch_metas.map { meta -> def base = file(meta.dragen_somatic_dir).toUriString() return [meta, "${base}/${meta.tumor_id}.hrdscore.csv"] } + def ch_input_vcf_germline = ch_metas.map { meta -> def base = file(meta.dragen_germline_dir).toUriString() return [meta, "${base}/${meta.normal_id}.hard-filtered.vcf.gz"] } + def ch_input_vcf_somatic = ch_metas.map { meta -> def base = file(meta.dragen_somatic_dir).toUriString() def vcf = "${base}/${meta.tumor_id}.hard-filtered.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } emit: - metas = ch_metas - amber = ch_amber - cobalt = ch_cobalt - sage_somatic = ch_sage_somatic - virusbreakend = ch_virusbreakend - hrd = ch_input_hrd - vcf_germline = ch_input_vcf_germline - vcf_somatic = ch_input_vcf_somatic - ref_depth_vcf = ch_ref_depth_vcf - prep_dir = ch_prep_dir + // Meta information for each sample + metas = ch_metas // channel: [ meta ] + + // Oncoanalyser inputs + amber = ch_amber // channel: [ meta, amber_dir ] + cobalt = ch_cobalt // channel: [ meta, cobalt_dir ] + sage_somatic = ch_sage_somatic // channel: [ meta, vcf, tbi ] + virusbreakend = ch_virusbreakend // channel: [ meta, virusbreakend_dir ] + + // DRAGEN inputs + hrd = ch_input_hrd // channel: [ meta, hrdscore_csv ] + vcf_germline = ch_input_vcf_germline // channel: [ meta, germline_vcf ] + vcf_somatic = ch_input_vcf_somatic // channel: [ meta, somatic_vcf, tbi ] + + // eSVee inputs + call_inputs = ch_call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] } diff --git a/workflows/sash.nf b/workflows/sash.nf index 16d20b54..0ec538d8 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -81,21 +81,24 @@ workflow SASH { // Prepare inputs from samplesheet // - // channel: [ meta ] PREPARE_INPUT( file(params.input), ) - ch_inputs = PREPARE_INPUT.out.metas - - ch_amber = PREPARE_INPUT.out.amber - ch_cobalt = PREPARE_INPUT.out.cobalt - ch_sage_somatic = PREPARE_INPUT.out.sage_somatic - ch_virusbreakend = PREPARE_INPUT.out.virusbreakend - ch_input_hrd = PREPARE_INPUT.out.hrd - ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline - ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic - ch_ref_depth_vcf = PREPARE_INPUT.out.ref_depth_vcf - ch_prep_dir = PREPARE_INPUT.out.prep_dir + ch_inputs = PREPARE_INPUT.out.metas // channel: [ meta ] + + // OncoAnalyser outputs + ch_amber = PREPARE_INPUT.out.amber // channel: [ meta, amber_dir ] + ch_cobalt = PREPARE_INPUT.out.cobalt // channel: [ meta, cobalt_dir ] + ch_sage_somatic = PREPARE_INPUT.out.sage_somatic // channel: [ meta, vcf, tbi ] + ch_virusbreakend = PREPARE_INPUT.out.virusbreakend // channel: [ meta, virusbreakend_dir ] + + // DRAGEN outputs + ch_input_hrd = PREPARE_INPUT.out.hrd // channel: [ meta, hrdscore_csv ] + ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline // channel: [ meta, germline_vcf ] + ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic // channel: [ meta, somatic_vcf, tbi ] + + // eSVee inputs + ch_call_inputs = PREPARE_INPUT.out.call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] From 22ca727cdf1b663e60d5bd2ef38aec5cb0061a2e Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 16:05:21 +1000 Subject: [PATCH 53/88] linting and version esvee --- workflows/sash.nf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/workflows/sash.nf b/workflows/sash.nf index 0ec538d8..083ea6ef 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -234,6 +234,7 @@ workflow SASH { hmf_data.repeatmasker_annotations, ) + ch_versions = ch_versions.mix(ESVEE_CALL.out.versions) // // CNV calling using UMCCR postprocessed variants // @@ -251,6 +252,7 @@ workflow SASH { // * https://github.com/hartwigmedical/hmftools/blob/a2f82e5/purple/src/main/java/com/hartwig/hmftools/purple/germline/GermlineGenotypeEnrichment.java#L63 //ch_smlv_germline_out, ch_smlv_germline_out.map { meta, vcf -> return [meta, []] }, + ESVEE_CALL.out.somatic_vcf, ESVEE_CALL.out.germline_vcf, genome.fasta, @@ -265,6 +267,9 @@ workflow SASH { hmf_data.purple_germline_del, ) + + + // // Small variant reporting (PCGR, CPSR, stats) // From fd1e84f4255e3b55c6dfb7db492518714fe1c4a3 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 17:53:33 +1000 Subject: [PATCH 54/88] fix deprecated channel --- workflows/sash.nf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 083ea6ef..4a1c0507 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -219,10 +219,6 @@ workflow SASH { // Somatic structural variants // - ch_call_inputs = WorkflowSash.groupByMeta( - ch_ref_depth_vcf, - ch_prep_dir - ) ESVEE_CALL( ch_call_inputs, From 73fcdc20dd5e9c7bc1d1b36a03e9bb7e2f049e54 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 17:57:58 +1000 Subject: [PATCH 55/88] uniform doc --- workflows/sash.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 4a1c0507..753b560c 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -86,13 +86,13 @@ workflow SASH { ) ch_inputs = PREPARE_INPUT.out.metas // channel: [ meta ] - // OncoAnalyser outputs + // OncoAnalyser inputs ch_amber = PREPARE_INPUT.out.amber // channel: [ meta, amber_dir ] ch_cobalt = PREPARE_INPUT.out.cobalt // channel: [ meta, cobalt_dir ] ch_sage_somatic = PREPARE_INPUT.out.sage_somatic // channel: [ meta, vcf, tbi ] ch_virusbreakend = PREPARE_INPUT.out.virusbreakend // channel: [ meta, virusbreakend_dir ] - // DRAGEN outputs + // DRAGEN inputs ch_input_hrd = PREPARE_INPUT.out.hrd // channel: [ meta, hrdscore_csv ] ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline // channel: [ meta, germline_vcf ] ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic // channel: [ meta, somatic_vcf, tbi ] From 83b15a728995e51a9972106732ffe4182a139650 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 13 May 2025 18:14:03 +1000 Subject: [PATCH 56/88] fix order esvee channel --- subworkflows/local/prepare_input.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 6ea4bcaa..a7f6e2b3 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -53,9 +53,9 @@ workflow PREPARE_INPUT { def ch_call_inputs = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() - def dir = "${base}/esvee/prep/" def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" - return [meta, dir, vcf] + def dir = "${base}/esvee/prep/" + return [meta, vcf, dir] } def ch_sage_somatic = ch_metas.map { meta -> From 8b7f806a16bd795f1041c46c909e3372b102b77e Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 14 May 2025 14:11:13 +1000 Subject: [PATCH 57/88] add stup block --- modules/nf-core/custom/dumpsoftwareversions/main.nf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 800a6099..87adf06a 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -21,4 +21,11 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { script: def args = task.ext.args ?: '' template 'dumpsoftwareversions.py' + + stub: + """ + touch software_versions.yml + touch software_versions_mqc.yml + touch versions.yml + """ } From ab31d5d58489885ee667b9e5c36eb27b39637bf0 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 14 May 2025 14:27:03 +1000 Subject: [PATCH 58/88] change esvee meta to use key tag --- conf/modules.config | 3 +-- subworkflows/local/prepare_input.nf | 9 ++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 821bf08b..c8777ec1 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -171,8 +171,7 @@ process { publishDir = [ path: { "${params.outdir}" }, mode: params.publish_dir_mode, - // Use meta.id as it's always defined, unlike meta.key which was causing null issues - saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.id}/esvee/${filename}" } + saveAs: { filename -> filename.equals('versions.yml') ? null : "${meta.key}/esvee/${filename}" } ] } diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index a7f6e2b3..d1040ed5 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -52,10 +52,17 @@ workflow PREPARE_INPUT { } def ch_call_inputs = ch_metas.map { meta -> + def meta_esvee = [ + key: meta.id, + id: meta.id, + tumor_id: meta.tumor_id, + normal_id: meta.normal_id, + ] + def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/esvee/depth_annotation/${meta.tumor_id}.esvee.ref_depth.vcf.gz" def dir = "${base}/esvee/prep/" - return [meta, vcf, dir] + return [meta_esvee, vcf, dir] } def ch_sage_somatic = ch_metas.map { meta -> From 6db7b5d9482c5cbd5086794571fdb3092df70e9b Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 14 May 2025 16:22:59 +1000 Subject: [PATCH 59/88] clean up --- modules/local/linx/germline/environment.yml | 7 ------- modules/local/linx/somatic/environment.yml | 7 ------- modules/local/linx/visualiser/environment.yml | 7 ------- modules/local/linxreport/environment.yml | 7 ------- modules/local/purple/environment.yml | 7 ------- modules/nf-core/custom/dumpsoftwareversions/main.nf | 2 +- subworkflows/local/prepare_input.nf | 6 +++--- workflows/sash.nf | 11 ++++++++--- 8 files changed, 12 insertions(+), 42 deletions(-) delete mode 100644 modules/local/linx/germline/environment.yml delete mode 100644 modules/local/linx/somatic/environment.yml delete mode 100644 modules/local/linx/visualiser/environment.yml delete mode 100644 modules/local/linxreport/environment.yml delete mode 100644 modules/local/purple/environment.yml diff --git a/modules/local/linx/germline/environment.yml b/modules/local/linx/germline/environment.yml deleted file mode 100644 index 57471f2c..00000000 --- a/modules/local/linx/germline/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: linx_germline -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::hmftools-linx=2.0 diff --git a/modules/local/linx/somatic/environment.yml b/modules/local/linx/somatic/environment.yml deleted file mode 100644 index 4d244fae..00000000 --- a/modules/local/linx/somatic/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: linx_somatic -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::hmftools-linx=2.0 diff --git a/modules/local/linx/visualiser/environment.yml b/modules/local/linx/visualiser/environment.yml deleted file mode 100644 index 5dadb70e..00000000 --- a/modules/local/linx/visualiser/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: linx_visualiser -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::hmftools-linx=2.0 diff --git a/modules/local/linxreport/environment.yml b/modules/local/linxreport/environment.yml deleted file mode 100644 index 71113340..00000000 --- a/modules/local/linxreport/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: linxreport -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::r-linxreport=1.1.0 diff --git a/modules/local/purple/environment.yml b/modules/local/purple/environment.yml deleted file mode 100644 index fb18f1e6..00000000 --- a/modules/local/purple/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: purple -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::hmftools-purple=4.1 diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 87adf06a..f3b3d8a8 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -26,6 +26,6 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { """ touch software_versions.yml touch software_versions_mqc.yml - touch versions.yml + echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml """ } diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index d1040ed5..49ccf207 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -98,13 +98,13 @@ workflow PREPARE_INPUT { // Oncoanalyser inputs amber = ch_amber // channel: [ meta, amber_dir ] cobalt = ch_cobalt // channel: [ meta, cobalt_dir ] - sage_somatic = ch_sage_somatic // channel: [ meta, vcf, tbi ] + sage_somatic = ch_sage_somatic // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] virusbreakend = ch_virusbreakend // channel: [ meta, virusbreakend_dir ] // DRAGEN inputs hrd = ch_input_hrd // channel: [ meta, hrdscore_csv ] - vcf_germline = ch_input_vcf_germline // channel: [ meta, germline_vcf ] - vcf_somatic = ch_input_vcf_somatic // channel: [ meta, somatic_vcf, tbi ] + vcf_germline = ch_input_vcf_germline // channel: [ meta, dragen_germline_vcf ] + vcf_somatic = ch_input_vcf_somatic // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] // eSVee inputs call_inputs = ch_call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] diff --git a/workflows/sash.nf b/workflows/sash.nf index 753b560c..1ce4d20b 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -89,13 +89,13 @@ workflow SASH { // OncoAnalyser inputs ch_amber = PREPARE_INPUT.out.amber // channel: [ meta, amber_dir ] ch_cobalt = PREPARE_INPUT.out.cobalt // channel: [ meta, cobalt_dir ] - ch_sage_somatic = PREPARE_INPUT.out.sage_somatic // channel: [ meta, vcf, tbi ] + ch_sage_somatic = PREPARE_INPUT.out.sage_somatic // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] ch_virusbreakend = PREPARE_INPUT.out.virusbreakend // channel: [ meta, virusbreakend_dir ] // DRAGEN inputs ch_input_hrd = PREPARE_INPUT.out.hrd // channel: [ meta, hrdscore_csv ] - ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline // channel: [ meta, germline_vcf ] - ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic // channel: [ meta, somatic_vcf, tbi ] + ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline // channel: [ meta, dragen_germline_vcf ] + ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] // eSVee inputs ch_call_inputs = PREPARE_INPUT.out.call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] @@ -190,6 +190,7 @@ workflow SASH { // Germline small variants // + // channel: [ meta_bolt, dragen_germline_vcf ] ch_smlv_germline_prepare_inputs = ch_input_vcf_germline .map { meta, dragen_vcf -> @@ -231,6 +232,10 @@ workflow SASH { ) ch_versions = ch_versions.mix(ESVEE_CALL.out.versions) + + + + // // CNV calling using UMCCR postprocessed variants // From 0f1748796770ce4c461211910b63b4449c4292ff Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 15 May 2025 11:39:16 +1000 Subject: [PATCH 60/88] add utils_nfcore_pipeline --- modules.json | 9 + modules/nf-core/fastqc/main.nf | 51 +++ modules/nf-core/fastqc/meta.yml | 52 +++ modules/nf-core/multiqc/main.nf | 53 +++ modules/nf-core/multiqc/meta.yml | 56 +++ .../nf-core/utils_nfcore_pipeline/main.nf | 419 ++++++++++++++++++ .../nf-core/utils_nfcore_pipeline/meta.yml | 24 + .../tests/main.function.nf.test | 126 ++++++ .../tests/main.function.nf.test.snap | 136 ++++++ .../tests/main.workflow.nf.test | 29 ++ .../tests/main.workflow.nf.test.snap | 19 + .../tests/nextflow.config | 9 + 12 files changed, 983 insertions(+) create mode 100644 modules/nf-core/fastqc/main.nf create mode 100644 modules/nf-core/fastqc/meta.yml create mode 100644 modules/nf-core/multiqc/main.nf create mode 100644 modules/nf-core/multiqc/meta.yml create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/main.nf create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/meta.yml create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap create mode 100644 subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config diff --git a/modules.json b/modules.json index f5341730..42c1b398 100644 --- a/modules.json +++ b/modules.json @@ -21,6 +21,15 @@ "installed_by": ["modules"] } } + }, + "subworkflows": { + "nf-core": { + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d", + "installed_by": ["subworkflows"] + } + } } } } diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf new file mode 100644 index 00000000..9ae58381 --- /dev/null +++ b/modules/nf-core/fastqc/main.nf @@ -0,0 +1,51 @@ +process FASTQC { + tag "$meta.id" + label 'process_medium' + + conda "bioconda::fastqc=0.11.9" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + 'quay.io/biocontainers/fastqc:0.11.9--0' }" + + input: + tuple val(meta), path(reads) + + output: + tuple val(meta), path("*.html"), emit: html + tuple val(meta), path("*.zip") , emit: zip + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + """ + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.html + touch ${prefix}.zip + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml new file mode 100644 index 00000000..4da5bb5a --- /dev/null +++ b/modules/nf-core/fastqc/meta.yml @@ -0,0 +1,52 @@ +name: fastqc +description: Run FastQC on sequenced reads +keywords: + - quality control + - qc + - adapters + - fastq +tools: + - fastqc: + description: | + FastQC gives general quality metrics about your reads. + It provides information about the quality score distribution + across your reads, the per base sequence content (%A/C/G/T). + You get information about adapter contamination and other + overrepresented sequences. + homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ + documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ + licence: ["GPL-2.0-only"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - html: + type: file + description: FastQC report + pattern: "*_{fastqc.html}" + - zip: + type: file + description: FastQC report archive + pattern: "*_{fastqc.zip}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@drpatelh" + - "@grst" + - "@ewels" + - "@FelixKrueger" diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf new file mode 100644 index 00000000..4b604749 --- /dev/null +++ b/modules/nf-core/multiqc/main.nf @@ -0,0 +1,53 @@ +process MULTIQC { + label 'process_single' + + conda "bioconda::multiqc=1.14" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" + + input: + path multiqc_files, stageAs: "?/*" + path(multiqc_config) + path(extra_multiqc_config) + path(multiqc_logo) + + output: + path "*multiqc_report.html", emit: report + path "*_data" , emit: data + path "*_plots" , optional:true, emit: plots + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def config = multiqc_config ? "--config $multiqc_config" : '' + def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + """ + multiqc \\ + --force \\ + $args \\ + $config \\ + $extra_config \\ + . + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ + + stub: + """ + touch multiqc_data + touch multiqc_plots + touch multiqc_report.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml new file mode 100644 index 00000000..f93b5ee5 --- /dev/null +++ b/modules/nf-core/multiqc/meta.yml @@ -0,0 +1,56 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +name: MultiQC +description: Aggregate results from bioinformatics analyses across many samples into a single report +keywords: + - QC + - bioinformatics tools + - Beautiful stand-alone HTML report +tools: + - multiqc: + description: | + MultiQC searches a given directory for analysis logs and compiles a HTML report. + It's a general use tool, perfect for summarising the output from numerous bioinformatics tools. + homepage: https://multiqc.info/ + documentation: https://multiqc.info/docs/ + licence: ["GPL-3.0-or-later"] + +input: + - multiqc_files: + type: file + description: | + List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. + pattern: "*.{yml,yaml}" + - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" + +output: + - report: + type: file + description: MultiQC report file + pattern: "multiqc_report.html" + - data: + type: directory + description: MultiQC data dir + pattern: "multiqc_data" + - plots: + type: file + description: Plots created by MultiQC + pattern: "*_data" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@abhi18av" + - "@bunop" + - "@drpatelh" + - "@jfy133" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf new file mode 100644 index 00000000..bfd25876 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -0,0 +1,419 @@ +// +// Subworkflow with utility functions specific to the nf-core pipeline template +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + SUBWORKFLOW DEFINITION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +workflow UTILS_NFCORE_PIPELINE { + take: + nextflow_cli_args + + main: + valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) + + emit: + valid_config +} + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + FUNCTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +// +// Warn if a -profile or Nextflow config has not been provided to run the pipeline +// +def checkConfigProvided() { + def valid_config = true as Boolean + if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { + log.warn( + "[${workflow.manifest.name}] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + "Please refer to the quick start section and usage docs for the pipeline.\n " + ) + valid_config = false + } + return valid_config +} + +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error( + "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) + } + if (nextflow_cli_args[0]) { + log.warn( + "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) + } +} + +// +// Generate workflow version string +// +def getWorkflowVersion() { + def version_string = "" as String + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + ${workflow.manifest.name}: ${getWorkflowVersion()} + Nextflow: ${workflow.nextflow.version} + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(Channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + summary_params + .keySet() + .each { group -> + def group_params = summary_params.get(group) + // This gets the parameters of that particular group + if (group_params) { + summary_section += "

${group}

\n" + summary_section += "
\n" + group_params + .keySet() + .sort() + .each { param -> + summary_section += "
${param}
${group_params.get(param) ?: 'N/A'}
\n" + } + summary_section += "
\n" + } + } + + def yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" as String + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + +// +// ANSII colours used for terminal logging +// +def logColours(monochrome_logs=true) { + def colorcodes = [:] as Map + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes +} + +// Return a single report from an object that may be a Path or List +// +def getSingleReport(multiqc_reports) { + if (multiqc_reports instanceof Path) { + return multiqc_reports + } else if (multiqc_reports instanceof List) { + if (multiqc_reports.size() == 0) { + log.warn("[${workflow.manifest.name}] No reports found from process 'MULTIQC'") + return null + } else if (multiqc_reports.size() == 1) { + return multiqc_reports.first() + } else { + log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") + return multiqc_reports.first() + } + } else { + return null + } +} + +// +// Construct and send completion email +// +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { + + // Set up the e-mail variables + def subject = "[${workflow.manifest.name}] Successful: ${workflow.runName}" + if (!workflow.success) { + subject = "[${workflow.manifest.name}] FAILED: ${workflow.runName}" + } + + def summary = [:] + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) { + misc_fields['Pipeline repository Git URL'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['Pipeline repository Git Commit'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['Pipeline Git branch/tag'] = workflow.revision + } + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = getWorkflowVersion() + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // On success try attach the multiqc report + def mqc_report = getSingleReport(multiqc_report) + + // Check if we are only sending emails on failure + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("${workflow.projectDir}/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("${workflow.projectDir}/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit + def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] + def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + def colors = logColours(monochrome_logs) as Map + if (email_address) { + try { + if (plaintext_email) { + new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') + } + // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } + ['sendmail', '-t'].execute() << sendmail_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") + } + catch (Exception msg) { + log.debug(msg.toString()) + log.debug("Trying with mail instead of sendmail") + // Catch failures and try with plaintext + def mail_cmd = ['mail', '-s', subject, '--content-type=text/html', email_address] + mail_cmd.execute() << email_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (mail)-") + } + } + + // Write summary e-mail HTML to a file + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") + output_tf.delete() +} + +// +// Print pipeline summary on completion +// +def completionSummary(monochrome_logs=true) { + def colors = logColours(monochrome_logs) as Map + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Pipeline completed successfully${colors.reset}-") + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-") + } + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") + } +} + +// +// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack +// +def imNotification(summary_params, hook_url) { + def summary = [:] + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) { + misc_fields['repository'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['commitid'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['revision'] = workflow.revision + } + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = getWorkflowVersion() + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("${workflow.projectDir}/assets/${json_path}") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection() + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")) + def postRC = post.getResponseCode() + if (!postRC.equals(200)) { + log.warn(post.getErrorStream().getText()) + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml new file mode 100644 index 00000000..d08d2434 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFCORE_PIPELINE" +description: Subworkflow with utility functions specific to the nf-core pipeline template +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments +output: + - success: + type: boolean + description: | + Dummy output to indicate success +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test new file mode 100644 index 00000000..f117040c --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -0,0 +1,126 @@ + +nextflow_function { + + name "Test Functions" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Test Function checkConfigProvided") { + + function "checkConfigProvided" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + + when { + function { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function without logColours") { + + function "logColours" + + when { + function { + """ + input[0] = true + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function with logColours") { + function "logColours" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function getSingleReport with a single file") { + function "getSingleReport" + + when { + function { + """ + input[0] = file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.contains("test.tsv") } + ) + } + } + + test("Test Function getSingleReport with multiple files") { + function "getSingleReport" + + when { + function { + """ + input[0] = [ + file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/network.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/expression.tsv', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert function.result.contains("test.tsv") }, + { assert !function.result.contains("network.tsv") }, + { assert !function.result.contains("expression.tsv") } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 00000000..02c67014 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,136 @@ +{ + "Test Function checkProfileProvided": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" + }, + "Test Function checkConfigProvided": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" + }, + "Test Function with logColours": { + "content": [ + { + "reset": "\u001b[0m", + "bold": "\u001b[1m", + "dim": "\u001b[2m", + "underlined": "\u001b[4m", + "blink": "\u001b[5m", + "reverse": "\u001b[7m", + "hidden": "\u001b[8m", + "black": "\u001b[0;30m", + "red": "\u001b[0;31m", + "green": "\u001b[0;32m", + "yellow": "\u001b[0;33m", + "blue": "\u001b[0;34m", + "purple": "\u001b[0;35m", + "cyan": "\u001b[0;36m", + "white": "\u001b[0;37m", + "bblack": "\u001b[1;30m", + "bred": "\u001b[1;31m", + "bgreen": "\u001b[1;32m", + "byellow": "\u001b[1;33m", + "bblue": "\u001b[1;34m", + "bpurple": "\u001b[1;35m", + "bcyan": "\u001b[1;36m", + "bwhite": "\u001b[1;37m", + "ublack": "\u001b[4;30m", + "ured": "\u001b[4;31m", + "ugreen": "\u001b[4;32m", + "uyellow": "\u001b[4;33m", + "ublue": "\u001b[4;34m", + "upurple": "\u001b[4;35m", + "ucyan": "\u001b[4;36m", + "uwhite": "\u001b[4;37m", + "iblack": "\u001b[0;90m", + "ired": "\u001b[0;91m", + "igreen": "\u001b[0;92m", + "iyellow": "\u001b[0;93m", + "iblue": "\u001b[0;94m", + "ipurple": "\u001b[0;95m", + "icyan": "\u001b[0;96m", + "iwhite": "\u001b[0;97m", + "biblack": "\u001b[1;90m", + "bired": "\u001b[1;91m", + "bigreen": "\u001b[1;92m", + "biyellow": "\u001b[1;93m", + "biblue": "\u001b[1;94m", + "bipurple": "\u001b[1;95m", + "bicyan": "\u001b[1;96m", + "biwhite": "\u001b[1;97m" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 00000000..8940d32d --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 00000000..859d1030 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config new file mode 100644 index 00000000..d0a926bf --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} From a9e6512a83ec47c7030ce612b48e60738f1d5882 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 15 May 2025 11:41:47 +1000 Subject: [PATCH 61/88] change version collection --- workflows/sash.nf | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 1ce4d20b..50379776 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -49,6 +49,8 @@ include { BOLT_SV_SOMATIC_PRIORITISE } from '../modules/local/bolt/sv_somatic/pr include { ESVEE_CALL } from '../modules/local/esvee/call/main' include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' + include { LINX_ANNOTATION } from '../subworkflows/local/linx_annotation' include { LINX_PLOTTING } from '../subworkflows/local/linx_plotting' include { PREPARE_INPUT } from '../subworkflows/local/prepare_input' @@ -539,11 +541,15 @@ workflow SASH { // - // Collect software versions + // TASK: Aggregate software versions // - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) + softwareVersionsToYAML(ch_versions) + .collectFile( + storeDir: "${params.outdir}/pipeline_info", + name: 'software_versions.yml', + sort: true, + newLine: true, + ) } From 6af54578b9ef7275e9b5ab25148920bf589133c0 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 15 May 2025 14:56:08 +1000 Subject: [PATCH 62/88] fix meta --- subworkflows/local/prepare_input.nf | 1 + workflows/sash.nf | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index 49ccf207..b81e1329 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -57,6 +57,7 @@ workflow PREPARE_INPUT { id: meta.id, tumor_id: meta.tumor_id, normal_id: meta.normal_id, + sample_id: meta.tumor_id ] def base = file(meta.oncoanalyser_dir).toUriString() diff --git a/workflows/sash.nf b/workflows/sash.nf index 50379776..731b36bf 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -49,8 +49,6 @@ include { BOLT_SV_SOMATIC_PRIORITISE } from '../modules/local/bolt/sv_somatic/pr include { ESVEE_CALL } from '../modules/local/esvee/call/main' include { PAVE_SOMATIC } from '../modules/local/pave/somatic/main' -include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' - include { LINX_ANNOTATION } from '../subworkflows/local/linx_annotation' include { LINX_PLOTTING } from '../subworkflows/local/linx_plotting' include { PREPARE_INPUT } from '../subworkflows/local/prepare_input' @@ -63,7 +61,7 @@ include { PURPLE_CALLING } from '../subworkflows/local/purple_callin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -430,8 +428,6 @@ workflow SASH { // Generate the cancer report // - // channel: [ meta, dragen_hrd ] - // channel: [ meta_bolt, smlv_somatic_vcf, smlv_somatic_bcftools_stats, smlv_somatic_counts_process, sv_tsv, sv_vcf, cnv_tsv, af_global, af_keygenes, purple_baf_circos_plot, purple_dir, virusbreakend_dir, dragen_hrd ] ch_cancer_report_inputs = WorkflowSash.groupByMeta( ch_smlv_somatic_out, From 7350545a33a32af2ebec36993be6e1e3c1f13158 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 16 May 2025 15:15:46 +1000 Subject: [PATCH 63/88] fix channel meta for stub and standar run --- workflows/sash.nf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 731b36bf..f9e97ab4 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -135,6 +135,7 @@ workflow SASH { id: meta.id, tumor_id: meta.tumor_id, normal_id: meta.normal_id, + sample_id: meta.tumor_id ] return [meta_bolt, *it[1..-1]] } @@ -233,6 +234,9 @@ workflow SASH { ch_versions = ch_versions.mix(ESVEE_CALL.out.versions) + ch_esvee_somatic_out = WorkflowSash.restoreMeta(ESVEE_CALL.out.somatic_vcf, ch_inputs) + ch_esvee_germline_out = WorkflowSash.restoreMeta(ESVEE_CALL.out.germline_vcf, ch_inputs) + @@ -254,8 +258,8 @@ workflow SASH { //ch_smlv_germline_out, ch_smlv_germline_out.map { meta, vcf -> return [meta, []] }, - ESVEE_CALL.out.somatic_vcf, - ESVEE_CALL.out.germline_vcf, + ch_esvee_somatic_out, + ch_esvee_germline_out, genome.fasta, genome.version, genome.fai, @@ -268,6 +272,8 @@ workflow SASH { hmf_data.purple_germline_del, ) + ch_versions = ch_versions.mix(PURPLE_CALLING.out.versions) + From fb02a3140c4104b403ca298f6d166539a954b50c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 16 May 2025 15:16:07 +1000 Subject: [PATCH 64/88] remove deprecated modules --- .../custom/dumpsoftwareversions/main.nf | 31 ------ .../custom/dumpsoftwareversions/meta.yml | 36 ------- .../templates/dumpsoftwareversions.py | 102 ------------------ modules/nf-core/fastqc/main.nf | 51 --------- modules/nf-core/fastqc/meta.yml | 52 --------- modules/nf-core/multiqc/main.nf | 53 --------- modules/nf-core/multiqc/meta.yml | 56 ---------- 7 files changed, 381 deletions(-) delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/main.nf delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/meta.yml delete mode 100755 modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py delete mode 100644 modules/nf-core/fastqc/main.nf delete mode 100644 modules/nf-core/fastqc/meta.yml delete mode 100644 modules/nf-core/multiqc/main.nf delete mode 100644 modules/nf-core/multiqc/meta.yml diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf deleted file mode 100644 index f3b3d8a8..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ /dev/null @@ -1,31 +0,0 @@ -process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_single' - - // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.14" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" - - input: - path versions - - output: - path "software_versions.yml" , emit: yml - path "software_versions_mqc.yml", emit: mqc_yml - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - template 'dumpsoftwareversions.py' - - stub: - """ - touch software_versions.yml - touch software_versions_mqc.yml - echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml - """ -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml deleted file mode 100644 index c32657de..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ /dev/null @@ -1,36 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json -name: custom_dumpsoftwareversions -description: Custom module used to dump software versions within the nf-core pipeline template -keywords: - - custom - - dump - - version -tools: - - custom: - description: Custom module used to dump software versions within the nf-core pipeline template - homepage: https://github.com/nf-core/tools - documentation: https://github.com/nf-core/tools - licence: ["MIT"] -input: - - versions: - type: file - description: YML file containing software versions - pattern: "*.yml" - -output: - - yml: - type: file - description: Standard YML file containing software versions - pattern: "software_versions.yml" - - mqc_yml: - type: file - description: MultiQC custom content YML file containing software versions - pattern: "software_versions_mqc.yml" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" - -authors: - - "@drpatelh" - - "@grst" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py deleted file mode 100755 index e55b8d43..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - - -"""Provide functions to merge multiple versions.yml files.""" - - -import platform -from textwrap import dedent - -import yaml - - -def _make_versions_html(versions): - """Generate a tabular HTML output of all versions for MultiQC.""" - html = [ - dedent( - """\\ - - - - - - - - - - """ - ) - ] - for process, tmp_versions in sorted(versions.items()): - html.append("") - for i, (tool, version) in enumerate(sorted(tmp_versions.items())): - html.append( - dedent( - f"""\\ - - - - - - """ - ) - ) - html.append("") - html.append("
Process Name Software Version
{process if (i == 0) else ''}{tool}{version}
") - return "\\n".join(html) - - -def main(): - """Load all version files and generate merged output.""" - versions_this_module = {} - versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, - } - - with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - - # aggregate versions by the module name (derived from fully-qualified process name) - versions_by_module = {} - for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - - versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", - } - - versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), - } - - with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) - with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - - with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) - - -if __name__ == "__main__": - main() diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf deleted file mode 100644 index 9ae58381..00000000 --- a/modules/nf-core/fastqc/main.nf +++ /dev/null @@ -1,51 +0,0 @@ -process FASTQC { - tag "$meta.id" - label 'process_medium' - - conda "bioconda::fastqc=0.11.9" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'quay.io/biocontainers/fastqc:0.11.9--0' }" - - input: - tuple val(meta), path(reads) - - output: - tuple val(meta), path("*.html"), emit: html - tuple val(meta), path("*.zip") , emit: zip - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" - // Make list of old name and new name pairs to use for renaming in the bash while loop - def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } - def rename_to = old_new_pairs*.join(' ').join(' ') - def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') - """ - printf "%s %s\\n" $rename_to | while read old_name new_name; do - [ -f "\${new_name}" ] || ln -s \$old_name \$new_name - done - fastqc $args --threads $task.cpus $renamed_files - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - - stub: - def prefix = task.ext.prefix ?: "${meta.id}" - """ - touch ${prefix}.html - touch ${prefix}.zip - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml deleted file mode 100644 index 4da5bb5a..00000000 --- a/modules/nf-core/fastqc/meta.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: fastqc -description: Run FastQC on sequenced reads -keywords: - - quality control - - qc - - adapters - - fastq -tools: - - fastqc: - description: | - FastQC gives general quality metrics about your reads. - It provides information about the quality score distribution - across your reads, the per base sequence content (%A/C/G/T). - You get information about adapter contamination and other - overrepresented sequences. - homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ - documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ - licence: ["GPL-2.0-only"] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - html: - type: file - description: FastQC report - pattern: "*_{fastqc.html}" - - zip: - type: file - description: FastQC report archive - pattern: "*_{fastqc.zip}" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@grst" - - "@ewels" - - "@FelixKrueger" diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf deleted file mode 100644 index 4b604749..00000000 --- a/modules/nf-core/multiqc/main.nf +++ /dev/null @@ -1,53 +0,0 @@ -process MULTIQC { - label 'process_single' - - conda "bioconda::multiqc=1.14" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" - - input: - path multiqc_files, stageAs: "?/*" - path(multiqc_config) - path(extra_multiqc_config) - path(multiqc_logo) - - output: - path "*multiqc_report.html", emit: report - path "*_data" , emit: data - path "*_plots" , optional:true, emit: plots - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - def config = multiqc_config ? "--config $multiqc_config" : '' - def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' - """ - multiqc \\ - --force \\ - $args \\ - $config \\ - $extra_config \\ - . - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ - - stub: - """ - touch multiqc_data - touch multiqc_plots - touch multiqc_report.html - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml deleted file mode 100644 index f93b5ee5..00000000 --- a/modules/nf-core/multiqc/meta.yml +++ /dev/null @@ -1,56 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json -name: MultiQC -description: Aggregate results from bioinformatics analyses across many samples into a single report -keywords: - - QC - - bioinformatics tools - - Beautiful stand-alone HTML report -tools: - - multiqc: - description: | - MultiQC searches a given directory for analysis logs and compiles a HTML report. - It's a general use tool, perfect for summarising the output from numerous bioinformatics tools. - homepage: https://multiqc.info/ - documentation: https://multiqc.info/docs/ - licence: ["GPL-3.0-or-later"] - -input: - - multiqc_files: - type: file - description: | - List of reports / files recognised by MultiQC, for example the html and zip output of FastQC - - multiqc_config: - type: file - description: Optional config yml for MultiQC - pattern: "*.{yml,yaml}" - - extra_multiqc_config: - type: file - description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. - pattern: "*.{yml,yaml}" - - multiqc_logo: - type: file - description: Optional logo file for MultiQC - pattern: "*.{png}" - -output: - - report: - type: file - description: MultiQC report file - pattern: "multiqc_report.html" - - data: - type: directory - description: MultiQC data dir - pattern: "multiqc_data" - - plots: - type: file - description: Plots created by MultiQC - pattern: "*_data" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@abhi18av" - - "@bunop" - - "@drpatelh" - - "@jfy133" From 43d18cef634b87991ae13e72ccf6bf98bd6516c5 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 16 May 2025 15:19:30 +1000 Subject: [PATCH 65/88] upgrade gpgr container --- modules/local/bolt/other/cancer_report/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index ab41d0bf..e677c51f 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-2' + container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-3' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) From 3112089089c9ad492e39943ea04c6f148baaa49c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 22 May 2025 10:53:38 +1000 Subject: [PATCH 66/88] update changelog --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++ subworkflows/local/purple_calling.nf | 4 +-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d88a7205..49cbc8b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,3 +14,43 @@ Initial release of umccr/sash, created with the [nf-core](https://nf-co.re/) tem ### `Dependencies` ### `Deprecated` + +## [0.6.0] – 2025-05-19 + +### Added + +- _none_ + +### Changed + +- SV caller switched GRIPSS → eSVe +- Cancer-report Structural Variants plot: `SR`→`SF`, `PR`→`DF` +- Linx upgraded 1.25 → 2.0 +- Purple upgraded 4.0.1 → 4.1.0 +- Bolt & GPGR updated for the adpat to above change + +### Removed + +- Kataegis module +- CHORD HRD metrics + +### Deprecated + +- Metric aliases `SR`, `PR` + +### Fixed + +- _none_ + +### Security + +- _none_ + +### Dependencies + +| Tool | Old | New | +|------|-----|-----| +| Linx | 1.25 | 2.0 | +| Purple | 4.0.1 | 4.1.0 | +| Bolt | — | umccr/bolt#6 | +| GPGR | — | umccr/gpgr#88 | diff --git a/subworkflows/local/purple_calling.nf b/subworkflows/local/purple_calling.nf index ef73140c..444d0b68 100644 --- a/subworkflows/local/purple_calling.nf +++ b/subworkflows/local/purple_calling.nf @@ -33,7 +33,7 @@ workflow PURPLE_CALLING { ch_versions = Channel.empty() // Collect inputs - // channel: [ meta, amber_dir, cobalt_dir, sv_somatic_vcf, sv_somatic_tbi, sv_somatic_unfiltered_vcf, sv_somatic_unfiltered_tbi, sv_germline_vcf, sv_germline_tbi, smlv_somatic_vcf, smlv_germline_vcf ] + // channel: [ meta, amber_dir, cobalt_dir, sv_somatic_vcf, sv_somatic_tbi, sv_germline_vcf, sv_germline_tbi, smlv_somatic_vcf, smlv_germline_vcf ] ch_purple_inputs_source = WorkflowSash.groupByMeta( // Required inputs ch_amber, @@ -47,7 +47,7 @@ workflow PURPLE_CALLING { ) // Create process-specific meta - // channel: [ meta_purple, amber_dir, cobalt_dir, sv_somatic_vcf, sv_somatic_tbi, sv_somatic_unfiltered_vcf, sv_somatic_unfilt_vcf, sv_germline_tbi, smlv_somatic_vcf, smlv_germline_vcf ] + // channel: [ meta_purple, amber_dir, cobalt_dir, sv_somatic_vcf, sv_somatic_tbi, sv_germline_tbi, smlv_somatic_vcf, smlv_germline_vcf ] ch_purple_inputs = ch_purple_inputs_source .map { def meta = it[0] From 3afbc68bb1b5dbb6ad7a33a7aec11bd783ad3380 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 22 May 2025 10:54:48 +1000 Subject: [PATCH 67/88] remove deprecated tool versionning collection --- conf/base.config | 3 --- conf/modules.config | 8 -------- modules/nf-core/custom/dumpsoftwareversions/main.nf | 0 3 files changed, 11 deletions(-) create mode 100644 modules/nf-core/custom/dumpsoftwareversions/main.nf diff --git a/conf/base.config b/conf/base.config index 39c4800a..65e095e5 100644 --- a/conf/base.config +++ b/conf/base.config @@ -53,7 +53,4 @@ process { maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/conf/modules.config b/conf/modules.config index c8777ec1..b729e3af 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -212,12 +212,4 @@ process { ] } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - } diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf new file mode 100644 index 00000000..e69de29b From 01a0001b6a698ee2f795663a9fe7b2e3939ceae7 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 27 May 2025 10:54:27 +1000 Subject: [PATCH 68/88] upadte pave --- modules/local/pave/Dockerfile | 2 +- modules/local/pave/somatic/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/pave/Dockerfile b/modules/local/pave/Dockerfile index 47d89500..d0b4fe12 100644 --- a/modules/local/pave/Dockerfile +++ b/modules/local/pave/Dockerfile @@ -7,7 +7,7 @@ RUN \ zlib && \ micromamba clean --all --yes -FROM quay.io/biocontainers/hmftools-pave:1.6--hdfd78af_0 +FROM quay.io/biocontainers/hmftools-pave:1.7--hdfd78af_0 COPY --from=build /env/ /env/ diff --git a/modules/local/pave/somatic/main.nf b/modules/local/pave/somatic/main.nf index b17725fb..0a7c7954 100644 --- a/modules/local/pave/somatic/main.nf +++ b/modules/local/pave/somatic/main.nf @@ -2,7 +2,7 @@ process PAVE_SOMATIC { tag "${meta.id}" label 'process_medium' - container 'docker.io/scwatts/hmftools-pave:1.6--0' + container 'docker.io/qclayssen/hmftools-pave:1.7--hdfd78af_0' input: tuple val(meta), path(vcf), path(tbi) From 9cf9f5e029c57d89edb1f1f164a51ea6557b054e Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 27 May 2025 15:10:31 +1000 Subject: [PATCH 69/88] update container for esvee --- modules/local/bolt/other/cancer_report/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index e677c51f..0be6eb3d 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-3' + container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-4' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) From ee113189d839200b69e6171f9e8d2d2ad0f907cb Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 29 May 2025 09:48:13 +1000 Subject: [PATCH 70/88] update cancer report container --- modules/local/bolt/other/cancer_report/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index 0be6eb3d..cb02bbc7 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-4' + container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-5' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) From cd3f6640bc338a18a9158b4afcfd101946dedba3 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 29 May 2025 14:27:23 +1000 Subject: [PATCH 71/88] Remove deprecated module trace --- modules.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules.json b/modules.json index 42c1b398..8995535e 100644 --- a/modules.json +++ b/modules.json @@ -5,11 +5,6 @@ "https://github.com/nf-core/modules.git": { "modules": { "nf-core": { - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "76cc4938c1f6ea5c7d83fed1eeffc146787f9543", - "installed_by": ["modules"] - }, "fastqc": { "branch": "master", "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", From 3da7b3e9dcab2079bd39d9765648b1b415f09984 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 29 May 2025 14:27:34 +1000 Subject: [PATCH 72/88] Upadate changelog for MSI --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49cbc8b8..dbfbe71a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Initial release of umccr/sash, created with the [nf-core](https://nf-co.re/) tem - Kataegis module - CHORD HRD metrics +- MSI load and status from purple ### Deprecated From 14f1ab8eb7cdcc6c9a14a2c56a1366a8cb4af573 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 2 Jun 2025 12:13:28 +1000 Subject: [PATCH 73/88] fix documentation --- subworkflows/local/prepare_input.nf | 41 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index b81e1329..b6d0c8e9 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -3,7 +3,9 @@ workflow PREPARE_INPUT { ch_samplesheet main: - def ch_metas = Channel.of(ch_samplesheet) + // Parse samplesheet and group entries by sample ID + // channel: [ meta ] + ch_metas = Channel.of(ch_samplesheet) .splitCsv(header: true) .map { [it.id, it] } .groupTuple() @@ -40,18 +42,25 @@ workflow PREPARE_INPUT { return meta } - // map oncoanalyser assets and DRAGEN outputs into channels - def ch_amber = ch_metas.map { meta -> + // Map oncoanalyser assets and DRAGEN outputs into channels + + // AMBER: copy number segmentation data + // channel: [ meta, amber_dir ] + ch_amber = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/amber/"] } - def ch_cobalt = ch_metas.map { meta -> + // COBALT: read depth ratio data + // channel: [ meta, cobalt_dir ] + ch_cobalt = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/cobalt/"] } - def ch_call_inputs = ch_metas.map { meta -> + // eSVee: structural variant calling inputs + // channel: [ meta_esvee, esvee_ref_depth_vcf, esvee_prep_dir ] + ch_call_inputs = ch_metas.map { meta -> def meta_esvee = [ key: meta.id, id: meta.id, @@ -66,28 +75,38 @@ workflow PREPARE_INPUT { return [meta_esvee, vcf, dir] } - def ch_sage_somatic = ch_metas.map { meta -> + // SAGE: somatic small variant calls + // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] + ch_sage_somatic = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() def vcf = "${base}/sage/somatic/${meta.tumor_id}.sage.somatic.vcf.gz" return [meta, vcf, "${vcf}.tbi"] } - def ch_virusbreakend = ch_metas.map { meta -> + // VirusBreakend: viral integration detection + // channel: [ meta, virusbreakend_dir ] + ch_virusbreakend = ch_metas.map { meta -> def base = file(meta.oncoanalyser_dir).toUriString() return [meta, "${base}/virusbreakend/"] } - def ch_input_hrd = ch_metas.map { meta -> + // HRD: homologous recombination deficiency scores + // channel: [ meta, hrdscore_csv ] + ch_input_hrd = ch_metas.map { meta -> def base = file(meta.dragen_somatic_dir).toUriString() return [meta, "${base}/${meta.tumor_id}.hrdscore.csv"] } - def ch_input_vcf_germline = ch_metas.map { meta -> + // DRAGEN germline variants + // channel: [ meta, dragen_germline_vcf ] + ch_input_vcf_germline = ch_metas.map { meta -> def base = file(meta.dragen_germline_dir).toUriString() return [meta, "${base}/${meta.normal_id}.hard-filtered.vcf.gz"] } - def ch_input_vcf_somatic = ch_metas.map { meta -> + // DRAGEN somatic variants + // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] + ch_input_vcf_somatic = ch_metas.map { meta -> def base = file(meta.dragen_somatic_dir).toUriString() def vcf = "${base}/${meta.tumor_id}.hard-filtered.vcf.gz" return [meta, vcf, "${vcf}.tbi"] @@ -108,5 +127,5 @@ workflow PREPARE_INPUT { vcf_somatic = ch_input_vcf_somatic // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] // eSVee inputs - call_inputs = ch_call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] + call_inputs = ch_call_inputs // channel: [ meta_esvee, esvee_ref_depth_vcf, esvee_prep_dir ] } From 6023723f969833f9ea596d3284769479ff0f916d Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 2 Jun 2025 12:17:48 +1000 Subject: [PATCH 74/88] linting --- subworkflows/local/prepare_input.nf | 30 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/subworkflows/local/prepare_input.nf b/subworkflows/local/prepare_input.nf index b6d0c8e9..f2a74d70 100644 --- a/subworkflows/local/prepare_input.nf +++ b/subworkflows/local/prepare_input.nf @@ -112,20 +112,18 @@ workflow PREPARE_INPUT { return [meta, vcf, "${vcf}.tbi"] } emit: - // Meta information for each sample - metas = ch_metas // channel: [ meta ] - - // Oncoanalyser inputs - amber = ch_amber // channel: [ meta, amber_dir ] - cobalt = ch_cobalt // channel: [ meta, cobalt_dir ] - sage_somatic = ch_sage_somatic // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] - virusbreakend = ch_virusbreakend // channel: [ meta, virusbreakend_dir ] - - // DRAGEN inputs - hrd = ch_input_hrd // channel: [ meta, hrdscore_csv ] - vcf_germline = ch_input_vcf_germline // channel: [ meta, dragen_germline_vcf ] - vcf_somatic = ch_input_vcf_somatic // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] - - // eSVee inputs - call_inputs = ch_call_inputs // channel: [ meta_esvee, esvee_ref_depth_vcf, esvee_prep_dir ] + // Sample metadata + metas = ch_metas // channel: [ meta ] + + // oncoanalyser channels + amber = ch_amber // channel: [ meta, amber_dir ] + cobalt = ch_cobalt // channel: [ meta, cobalt_dir ] + sage_somatic = ch_sage_somatic // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] + virusbreakend = ch_virusbreakend // channel: [ meta, virusbreakend_dir ] + call_inputs = ch_call_inputs // channel: [ meta_esvee, esvee_ref_depth_vcf, esvee_prep_dir ] + + // DRAGEN channels + hrd = ch_input_hrd // channel: [ meta, hrdscore_csv ] + vcf_germline = ch_input_vcf_germline // channel: [ meta, dragen_germline_vcf ] + vcf_somatic = ch_input_vcf_somatic // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] } From 757c2bd8c5b042f4e8e264b0c6f3b93528a02d0a Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 2 Jun 2025 12:34:43 +1000 Subject: [PATCH 75/88] tidy up channel --- workflows/sash.nf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index f9e97ab4..596f2daa 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -91,15 +91,13 @@ workflow SASH { ch_cobalt = PREPARE_INPUT.out.cobalt // channel: [ meta, cobalt_dir ] ch_sage_somatic = PREPARE_INPUT.out.sage_somatic // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] ch_virusbreakend = PREPARE_INPUT.out.virusbreakend // channel: [ meta, virusbreakend_dir ] + ch_call_inputs = PREPARE_INPUT.out.call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] // DRAGEN inputs ch_input_hrd = PREPARE_INPUT.out.hrd // channel: [ meta, hrdscore_csv ] ch_input_vcf_germline = PREPARE_INPUT.out.vcf_germline // channel: [ meta, dragen_germline_vcf ] ch_input_vcf_somatic = PREPARE_INPUT.out.vcf_somatic // channel: [ meta, dragen_somatic_vcf, dragen_somatic_tbi ] - // eSVee inputs - ch_call_inputs = PREPARE_INPUT.out.call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] - From 35e3a349d8e8bd0e6286f651f779031707ae8f8e Mon Sep 17 00:00:00 2001 From: qclayssen Date: Mon, 2 Jun 2025 12:47:52 +1000 Subject: [PATCH 76/88] tidy up --- workflows/sash.nf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflows/sash.nf b/workflows/sash.nf index 596f2daa..12080847 100644 --- a/workflows/sash.nf +++ b/workflows/sash.nf @@ -84,14 +84,15 @@ workflow SASH { PREPARE_INPUT( file(params.input), ) - ch_inputs = PREPARE_INPUT.out.metas // channel: [ meta ] + + ch_inputs = PREPARE_INPUT.out.metas // channel: [ meta ] // OncoAnalyser inputs ch_amber = PREPARE_INPUT.out.amber // channel: [ meta, amber_dir ] ch_cobalt = PREPARE_INPUT.out.cobalt // channel: [ meta, cobalt_dir ] ch_sage_somatic = PREPARE_INPUT.out.sage_somatic // channel: [ meta, sage_somatic_vcf, sage_somatic_tbi ] ch_virusbreakend = PREPARE_INPUT.out.virusbreakend // channel: [ meta, virusbreakend_dir ] - ch_call_inputs = PREPARE_INPUT.out.call_inputs // channel: [ meta, esvee_prep_dir, esvee_ref_depth_vcf ] + ch_call_inputs = PREPARE_INPUT.out.call_inputs // channel: [ meta_esvee, esvee_ref_depth_vcf, esvee_prep_dir ] // DRAGEN inputs ch_input_hrd = PREPARE_INPUT.out.hrd // channel: [ meta, hrdscore_csv ] From ac5241f3bca0f4c2c0b892e01813fe6c9429dbc6 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Tue, 3 Jun 2025 09:26:11 +1000 Subject: [PATCH 77/88] remove deprecated module --- modules/local/gripss/Dockerfile | 15 ------ modules/local/gripss/germline/main.nf | 59 --------------------- modules/local/gripss/germline/meta.yml | 68 ------------------------ modules/local/gripss/somatic/main.nf | 64 ----------------------- modules/local/gripss/somatic/meta.yml | 71 -------------------------- 5 files changed, 277 deletions(-) delete mode 100644 modules/local/gripss/Dockerfile delete mode 100644 modules/local/gripss/germline/main.nf delete mode 100644 modules/local/gripss/germline/meta.yml delete mode 100644 modules/local/gripss/somatic/main.nf delete mode 100644 modules/local/gripss/somatic/meta.yml diff --git a/modules/local/gripss/Dockerfile b/modules/local/gripss/Dockerfile deleted file mode 100644 index 075afea8..00000000 --- a/modules/local/gripss/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM mambaorg/micromamba:1.5.7 as build - -USER root - -RUN \ - micromamba create -y -p /env/ -c bioconda -c conda-forge \ - zlib && \ - micromamba clean --all --yes - -FROM quay.io/biocontainers/hmftools-gripss:2.4--hdfd78af_0 - -COPY --from=build /env/ /env/ - -ENV PATH="/env/bin:${PATH}" -ENV LD_LIBRARY_PATH="/env/lib/:${LD_LIBRARY_PATH}" diff --git a/modules/local/gripss/germline/main.nf b/modules/local/gripss/germline/main.nf deleted file mode 100644 index 444ee912..00000000 --- a/modules/local/gripss/germline/main.nf +++ /dev/null @@ -1,59 +0,0 @@ -process GRIPSS_GERMLINE { - tag "${meta.id}" - label 'process_low' - - container 'docker.io/scwatts/hmftools-gripss:2.4--0' - - input: - tuple val(meta), path(gridss_vcf) - path genome_fasta - val genome_ver - path genome_fai - path pon_breakends - path pon_breakpoints - path known_fusions - path repeatmasker_annotations - - output: - tuple val(meta), path('*.filtered.germline.vcf.gz'), path('*.filtered.germline.vcf.gz.tbi'), emit: vcf - tuple val(meta), path('*gripss.germline.vcf.gz'), path('*gripss.germline.vcf.gz.tbi') , emit: vcf_unfiltered - path 'versions.yml' , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - - """ - gripss \\ - -Xmx${Math.round(task.memory.bytes * 0.95)} \\ - ${args} \\ - -sample ${meta.normal_id} \\ - -reference ${meta.tumor_id} \\ - -vcf ${gridss_vcf} \\ - -germline \\ - -ref_genome ${genome_fasta} \\ - -ref_genome_version ${genome_ver} \\ - -pon_sgl_file ${pon_breakends} \\ - -pon_sv_file ${pon_breakpoints} \\ - -known_hotspot_file ${known_fusions} \\ - -repeat_mask_file ${repeatmasker_annotations} \\ - -output_id germline \\ - -output_dir ./ - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - gripss: \$(gripss -version | sed 's/^.* //') - END_VERSIONS - """ - - stub: - """ - touch ${meta.normal_id}.gripss.filtered.germline.vcf.gz - touch ${meta.normal_id}.gripss.filtered.germline.vcf.gz.tbi - touch ${meta.normal_id}.gripss.germline.vcf.gz - touch ${meta.normal_id}.gripss.germline.vcf.gz.tbi - echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml - """ -} diff --git a/modules/local/gripss/germline/meta.yml b/modules/local/gripss/germline/meta.yml deleted file mode 100644 index 03ae9328..00000000 --- a/modules/local/gripss/germline/meta.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: gripss_germline -description: Filter and process GRIDSS germline structural variants -keywords: - - sv - - filtering - - germline -tools: - - gripss: - description: Apply filtering and post-processing to GRIDSS structural variants. - homepage: https://github.com/hartwigmedical/hmftools/tree/master/gripss - documentation: https://github.com/hartwigmedical/hmftools/tree/master/gripss - licence: ["GPL v3"] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [id: 'sample_id', normal_id: 'normal_name'] - - gridss_vcf: - type: file - description: VCF file - pattern: "*.{vcf.gz}" - - genome_fasta: - type: file - description: Reference genome assembly FASTA file - pattern: "*.{fa,fasta}" - - genome_ver: - type: string - description: Reference genome version - - genome_fai: - type: file - description: Reference genome assembly fai file - pattern: "*.{fai}" - - pon_breakends: - type: file - description: GRIDSS breakend PON file - pattern: "*.{bed.gz}" - - pon_breakpoints: - type: file - description: GRIDSS breakpoint PON file - pattern: "*.{bedpe.gz}" - - known_fusions: - type: file - description: HMF Known Fusions file - pattern: "*.{bedpe}" - - repeatmasker_annotations: - type: file - description: RepeatMasker annotations file -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [id: 'sample_id', normal_id: 'normal_name'] - - vcf: - type: list - description: Filtered VCF file and index file - pattern: "*.{vcf.gz,vcf.gz.tbi}" - - vcf_unfiltered: - type: list - description: Unfiltered VCF file and index file - pattern: "*.{vcf.gz,vcf.gz.tbi}" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@scwatts" diff --git a/modules/local/gripss/somatic/main.nf b/modules/local/gripss/somatic/main.nf deleted file mode 100644 index 47bd1c41..00000000 --- a/modules/local/gripss/somatic/main.nf +++ /dev/null @@ -1,64 +0,0 @@ -process GRIPSS_SOMATIC { - tag "${meta.id}" - label 'process_low' - - container 'docker.io/scwatts/hmftools-gripss:2.4--0' - - input: - tuple val(meta), path(gridss_vcf) - path genome_fasta - val genome_ver - path genome_fai - path pon_breakends - path pon_breakpoints - path known_fusions - path repeatmasker_annotations - path target_region_bed - - output: - tuple val(meta), path('*.gripss.filtered{,.somatic}.vcf.gz'), path('*.gripss.filtered{,.somatic}.vcf.gz.tbi'), emit: vcf - tuple val(meta), path('*.gripss{,.somatic}.vcf.gz'), path('*.gripss{,.somatic}.vcf.gz.tbi') , emit: vcf_unfiltered - path 'versions.yml' , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - - def reference_arg = meta.containsKey('normal_id') ? "-reference ${meta.normal_id}" : '' - def target_regions_bed_arg = target_region_bed ? "-target_regions_bed ${target_region_bed}" : '' - def output_id_arg = meta.containsKey('normal_id') ? '-output_id somatic' : '' - - """ - gripss \\ - -Xmx${Math.round(task.memory.bytes * 0.95)} \\ - ${args} \\ - -sample ${meta.tumor_id} \\ - ${reference_arg} \\ - -vcf ${gridss_vcf} \\ - -ref_genome ${genome_fasta} \\ - -ref_genome_version ${genome_ver} \\ - -pon_sgl_file ${pon_breakends} \\ - -pon_sv_file ${pon_breakpoints} \\ - -known_hotspot_file ${known_fusions} \\ - -repeat_mask_file ${repeatmasker_annotations} \\ - ${target_regions_bed_arg} \\ - ${output_id_arg} \\ - -output_dir ./ - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - gripss: \$(gripss -version | sed 's/^.* //') - END_VERSIONS - """ - - stub: - """ - touch ${meta.tumor_id}.gripss.filtered.somatic.vcf.gz - touch ${meta.tumor_id}.gripss.filtered.somatic.vcf.gz.tbi - touch ${meta.tumor_id}.gripss.somatic.vcf.gz - touch ${meta.tumor_id}.gripss.somatic.vcf.gz.tbi - echo -e '${task.process}:\\n stub: noversions\\n' > versions.yml - """ -} diff --git a/modules/local/gripss/somatic/meta.yml b/modules/local/gripss/somatic/meta.yml deleted file mode 100644 index 060ed5cc..00000000 --- a/modules/local/gripss/somatic/meta.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: gripss_somatic -description: Filter and process GRIDSS somatic structural variants -keywords: - - sv - - filtering - - somatic -tools: - - gripss: - description: Apply filtering and post-processing to GRIDSS structural variants. - homepage: https://github.com/hartwigmedical/hmftools/tree/master/gripss - documentation: https://github.com/hartwigmedical/hmftools/tree/master/gripss - licence: ["GPL v3"] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [id: 'sample_id', tumor_id: 'tumor_name', normal_id: 'normal_name'] - - gridss_vcf: - type: file - description: VCF file - pattern: "*.{vcf.gz}" - - genome_fasta: - type: file - description: Reference genome assembly FASTA file - pattern: "*.{fa,fasta}" - - genome_ver: - type: string - description: Reference genome version - - genome_fai: - type: file - description: Reference genome assembly fai file - pattern: "*.{fai}" - - pon_breakend: - type: file - description: GRIDSS breakend PON file - pattern: "*.{bed.gz}" - - pon_breakpoints: - type: file - description: GRIDSS breakpoint PON file - pattern: "*.{bedpe.gz}" - - known_fusions: - type: file - description: HMF Known Fusions file - pattern: "*.{bedpe}" - - repeatmasker_annotations: - type: file - description: RepeatMasker annotations file - - target_region_bed: - type: file - description: Target region BED file (optional) -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [id: 'sample_id', tumor_id: 'tumor_name', normal_id: 'normal_name'] - - vcf: - type: list - description: Hard filtered VCF file and index file - pattern: "*.{vcf.gz,vcf.gz.tbi}" - - vcf_unfiltered: - type: list - description: Unfiltered VCF file and index file - pattern: "*.{vcf.gz,vcf.gz.tbi}" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@scwatts" From 993bcb9175a3cfb12d4cbcbf137fa426cc101978 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 4 Jun 2025 15:51:19 +1000 Subject: [PATCH 78/88] update changelog --- CHANGELOG.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbfbe71a..2873957c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,19 +15,32 @@ Initial release of umccr/sash, created with the [nf-core](https://nf-co.re/) tem ### `Deprecated` -## [0.6.0] – 2025-05-19 +## [0.6.0] - 2025-06-04 ### Added -- _none_ +- Software versions are now in `pipeline_info/software_versions.yml` ### Changed -- SV caller switched GRIPSS → eSVe -- Cancer-report Structural Variants plot: `SR`→`SF`, `PR`→`DF` -- Linx upgraded 1.25 → 2.0 -- Purple upgraded 4.0.1 → 4.1.0 -- Bolt & GPGR updated for the adpat to above change +- **SV caller:** GRIPSS → eSVee + - SV counts (unmelted & melted) + - CNV counts + - TMB-SV counts + - CopyNumberSegment counts + - SV in Circos plots + - Breakpoints & Breakends tables + - Copy-number variants tables + - Genome-wide somatic CNV segment tracks + - SV Map visualisation +- Cancer-report Structural Variants summary plot: + - SR (Split Read) → SF (Split Fragments) + - PR (Paired-Read) → DF (Discordant Fragments) +- Linx v1.25 → v2.0 (affects all Linx reports/files) +- Purple v4.0.1 → v4.1.0 + - MSI calculation relay on SAGE-specific tags #7 + - (reverted) Circos have link sizes dependent on the size of SV #6 +- Filter PoN SV in cancer report tables #8 ### Removed @@ -38,10 +51,11 @@ Initial release of umccr/sash, created with the [nf-core](https://nf-co.re/) tem ### Deprecated - Metric aliases `SR`, `PR` +- GRIDSS/GRIPSS modules ### Fixed -- _none_ +- Nextflow `Stub` run ### Security From 968f7a4905c3332e7bebe99340339b99f22fddff Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 6 Jun 2025 11:03:59 +1000 Subject: [PATCH 79/88] upgrade bolt container to ghcr.io/umccr/bolt:0.2.14-dev --- modules/local/bolt/other/multiqc_report/main.nf | 2 +- modules/local/bolt/other/purple_baf_plot/main.nf | 2 +- modules/local/bolt/smlv_germline/prepare/main.nf | 2 +- modules/local/bolt/smlv_germline/report/main.nf | 2 +- modules/local/bolt/smlv_somatic/annotate/main.nf | 2 +- modules/local/bolt/smlv_somatic/filter/main.nf | 2 +- modules/local/bolt/smlv_somatic/report/main.nf | 2 +- modules/local/bolt/smlv_somatic/rescue/main.nf | 2 +- modules/local/bolt/sv_somatic/annotate/main.nf | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/local/bolt/other/multiqc_report/main.nf b/modules/local/bolt/other/multiqc_report/main.nf index d35be097..b42a1453 100644 --- a/modules/local/bolt/other/multiqc_report/main.nf +++ b/modules/local/bolt/other/multiqc_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_MULTIQC_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-multiqc' + container 'ghcr.io/umccr/bolt:0.2.14-dev-multiqc' input: tuple val(meta), path(input_files) diff --git a/modules/local/bolt/other/purple_baf_plot/main.nf b/modules/local/bolt/other/purple_baf_plot/main.nf index 1e24d501..cb70c8e6 100644 --- a/modules/local/bolt/other/purple_baf_plot/main.nf +++ b/modules/local/bolt/other/purple_baf_plot/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_PURPLE_BAF_PLOT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-circos' + container 'ghcr.io/umccr/bolt:0.2.14-dev-circos' input: tuple val(meta), path(purple_dir) diff --git a/modules/local/bolt/smlv_germline/prepare/main.nf b/modules/local/bolt/smlv_germline/prepare/main.nf index be419948..8a0bae23 100644 --- a/modules/local/bolt/smlv_germline/prepare/main.nf +++ b/modules/local/bolt/smlv_germline/prepare/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_GERMLINE_PREPARE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13' + container 'ghcr.io/umccr/bolt:0.2.14-dev' input: tuple val(meta), path(smlv_vcf) diff --git a/modules/local/bolt/smlv_germline/report/main.nf b/modules/local/bolt/smlv_germline/report/main.nf index b454e5ae..e5706842 100644 --- a/modules/local/bolt/smlv_germline/report/main.nf +++ b/modules/local/bolt/smlv_germline/report/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_GERMLINE_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-pcgr' + container 'ghcr.io/umccr/bolt:0.2.14-dev-pcgr' input: tuple val(meta), path(smlv_vcf), path(smlv_unfiltered_vcf) diff --git a/modules/local/bolt/smlv_somatic/annotate/main.nf b/modules/local/bolt/smlv_somatic/annotate/main.nf index 5374b5ca..659a50fc 100644 --- a/modules/local/bolt/smlv_somatic/annotate/main.nf +++ b/modules/local/bolt/smlv_somatic/annotate/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_ANNOTATE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-pcgr' + container 'ghcr.io/umccr/bolt:0.2.14-dev-pcgr' input: tuple val(meta), path(smlv_vcf) diff --git a/modules/local/bolt/smlv_somatic/filter/main.nf b/modules/local/bolt/smlv_somatic/filter/main.nf index 310dc4bf..0208eabe 100644 --- a/modules/local/bolt/smlv_somatic/filter/main.nf +++ b/modules/local/bolt/smlv_somatic/filter/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_FILTER { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13' + container 'ghcr.io/umccr/bolt:0.2.14-dev' input: tuple val(meta), path(smlv_vcf) diff --git a/modules/local/bolt/smlv_somatic/report/main.nf b/modules/local/bolt/smlv_somatic/report/main.nf index c5bbf6b9..13fe7196 100644 --- a/modules/local/bolt/smlv_somatic/report/main.nf +++ b/modules/local/bolt/smlv_somatic/report/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-pcgr' + container 'ghcr.io/umccr/bolt:0.2.14-dev-pcgr' input: tuple val(meta), path(smlv_vcf), path(smlv_filters_vcf), path(smlv_dragen_vcf), path(purple_purity) diff --git a/modules/local/bolt/smlv_somatic/rescue/main.nf b/modules/local/bolt/smlv_somatic/rescue/main.nf index 57fea4a2..e8802fbe 100644 --- a/modules/local/bolt/smlv_somatic/rescue/main.nf +++ b/modules/local/bolt/smlv_somatic/rescue/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_RESCUE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13' + container 'ghcr.io/umccr/bolt:0.2.14-dev' input: tuple val(meta), path(smlv_vcf), path(smlv_tbi), path(sage_smlv_vcf), path(sage_smlv_tbi) diff --git a/modules/local/bolt/sv_somatic/annotate/main.nf b/modules/local/bolt/sv_somatic/annotate/main.nf index 6437a4c1..1819ab16 100644 --- a/modules/local/bolt/sv_somatic/annotate/main.nf +++ b/modules/local/bolt/sv_somatic/annotate/main.nf @@ -2,7 +2,7 @@ process BOLT_SV_SOMATIC_ANNOTATE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/scwatts/bolt:0.2.13-snpeff' + container 'ghcr.io/umccr/bolt:0.2.14-dev-snpeff' input: tuple val(meta), path(sv_vcf), path(cnv_tsv) From de018ece94fd798691dc1940020230ab1047b3b1 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 6 Jun 2025 11:58:58 +1000 Subject: [PATCH 80/88] update containers --- modules/local/bolt/other/cancer_report/main.nf | 2 +- modules/local/bolt/sv_somatic/prioritise/main.nf | 2 +- modules/local/esvee/call/main.nf | 2 +- modules/local/linx/germline/main.nf | 2 +- modules/local/linx/somatic/main.nf | 2 +- modules/local/linx/visualiser/main.nf | 2 +- modules/local/linxreport/main.nf | 2 +- modules/local/pave/somatic/main.nf | 2 +- modules/local/purple/main.nf | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index cb02bbc7..67da0c6c 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:0.2.13-gpgr-esvee-5' + container 'ghcr.io/umccr/bolt:0.2.14-dev-gpgr' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) diff --git a/modules/local/bolt/sv_somatic/prioritise/main.nf b/modules/local/bolt/sv_somatic/prioritise/main.nf index 0a59abe6..616a5043 100644 --- a/modules/local/bolt/sv_somatic/prioritise/main.nf +++ b/modules/local/bolt/sv_somatic/prioritise/main.nf @@ -2,7 +2,7 @@ process BOLT_SV_SOMATIC_PRIORITISE { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/bolt:oncoanalyser-v2-2' + container 'ghcr.io/umccr/bolt:0.2.14-dev' input: tuple val(meta), path(sv_vcf) diff --git a/modules/local/esvee/call/main.nf b/modules/local/esvee/call/main.nf index 1447e79c..17c276c6 100644 --- a/modules/local/esvee/call/main.nf +++ b/modules/local/esvee/call/main.nf @@ -2,7 +2,7 @@ process ESVEE_CALL { tag "${meta.id}" label 'process_high' - container 'docker.io/qclayssen/esvee:1.0.3' + container 'ghcr.io/umccr/esvee:1.0.3' input: diff --git a/modules/local/linx/germline/main.nf b/modules/local/linx/germline/main.nf index 88c8c56e..cd03217c 100644 --- a/modules/local/linx/germline/main.nf +++ b/modules/local/linx/germline/main.nf @@ -2,7 +2,7 @@ process LINX_GERMLINE { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/hmftools-linx:2.0--1' + container 'ghcr.io/umccr/linx:2.0' input: tuple val(meta), path(sv_vcf) diff --git a/modules/local/linx/somatic/main.nf b/modules/local/linx/somatic/main.nf index e96e5d8e..268c782a 100644 --- a/modules/local/linx/somatic/main.nf +++ b/modules/local/linx/somatic/main.nf @@ -2,7 +2,7 @@ process LINX_SOMATIC { tag "${meta.id}" label 'process_low' - container 'docker.io/qclayssen/hmftools-linx:2.0--1' + container 'ghcr.io/umccr/linx:2.0' input: tuple val(meta), path(purple_dir) diff --git a/modules/local/linx/visualiser/main.nf b/modules/local/linx/visualiser/main.nf index 8eb1ae96..93eea38d 100644 --- a/modules/local/linx/visualiser/main.nf +++ b/modules/local/linx/visualiser/main.nf @@ -2,7 +2,7 @@ process LINX_VISUALISER { tag "${meta.id}" label 'process_medium' - container 'docker.io/qclayssen/hmftools-linx:2.0--1' + container 'ghcr.io/umccr/linx:2.0' input: tuple val(meta), path(linx_annotation_dir) diff --git a/modules/local/linxreport/main.nf b/modules/local/linxreport/main.nf index 957407c1..c6b84eff 100644 --- a/modules/local/linxreport/main.nf +++ b/modules/local/linxreport/main.nf @@ -2,7 +2,7 @@ process LINXREPORT { tag "${meta.id}" label 'process_single' - container 'docker.io/qclayssen/r-linxreport:1.1.0--0' + container 'ghcr.io/umccr/linxreport:1.1.0' input: tuple val(meta), path(linx_annotation_dir), path(linx_visualiser_dir) diff --git a/modules/local/pave/somatic/main.nf b/modules/local/pave/somatic/main.nf index 0a7c7954..ba048ad2 100644 --- a/modules/local/pave/somatic/main.nf +++ b/modules/local/pave/somatic/main.nf @@ -2,7 +2,7 @@ process PAVE_SOMATIC { tag "${meta.id}" label 'process_medium' - container 'docker.io/qclayssen/hmftools-pave:1.7--hdfd78af_0' + container 'ghcr.io/umccr/pave:1.6' input: tuple val(meta), path(vcf), path(tbi) diff --git a/modules/local/purple/main.nf b/modules/local/purple/main.nf index 29512207..9670aad7 100644 --- a/modules/local/purple/main.nf +++ b/modules/local/purple/main.nf @@ -2,7 +2,7 @@ process PURPLE { tag "${meta.id}" label 'process_medium' - container 'docker.io/qclayssen/purple:4.1.0--1' + container 'ghcr.io/umccr/purple:4.1.0' input: tuple val(meta), path(amber), path(cobalt), path(sv_tumor_vcf), path(sv_tumor_tbi), path(sv_normal_vcf), path(sv_normal_tbi), path(smlv_tumor_vcf), path(smlv_normal_vcf) From e2623860d11e3e5f6d364e3e7b1cae9d87658b53 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Fri, 6 Jun 2025 14:15:49 +1000 Subject: [PATCH 81/88] fix container name --- modules/local/pave/somatic/main.nf | 2 +- modules/local/purple/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/pave/somatic/main.nf b/modules/local/pave/somatic/main.nf index ba048ad2..a1581081 100644 --- a/modules/local/pave/somatic/main.nf +++ b/modules/local/pave/somatic/main.nf @@ -2,7 +2,7 @@ process PAVE_SOMATIC { tag "${meta.id}" label 'process_medium' - container 'ghcr.io/umccr/pave:1.6' + container 'ghcr.io/umccr/pave:1.7' input: tuple val(meta), path(vcf), path(tbi) diff --git a/modules/local/purple/main.nf b/modules/local/purple/main.nf index 9670aad7..d7dde6ba 100644 --- a/modules/local/purple/main.nf +++ b/modules/local/purple/main.nf @@ -2,7 +2,7 @@ process PURPLE { tag "${meta.id}" label 'process_medium' - container 'ghcr.io/umccr/purple:4.1.0' + container 'ghcr.io/umccr/purple:4.1' input: tuple val(meta), path(amber), path(cobalt), path(sv_tumor_vcf), path(sv_tumor_tbi), path(sv_normal_vcf), path(sv_normal_tbi), path(smlv_tumor_vcf), path(smlv_normal_vcf) From 43b988a1946bfd0dc8b0531ae038bf4dac02e4d9 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 12 Jun 2025 11:41:46 +1000 Subject: [PATCH 82/88] bump version to v0.6.0dev in nextflow.config --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 7a5d9757..79c1f4fb 100644 --- a/nextflow.config +++ b/nextflow.config @@ -173,7 +173,7 @@ manifest { description = """Cancer WGTS post-processing pipeline""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.6' - version = '0.5.0' + version = 'v0.6.0dev' doi = '' } From be3c8e60fd671367be2a147dd39167bb68547a01 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 18 Jun 2025 13:37:04 +1000 Subject: [PATCH 83/88] Change explicit linx process name as in OA --- subworkflows/local/linx_plotting.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subworkflows/local/linx_plotting.nf b/subworkflows/local/linx_plotting.nf index 66cd4eaa..180082b8 100644 --- a/subworkflows/local/linx_plotting.nf +++ b/subworkflows/local/linx_plotting.nf @@ -2,8 +2,8 @@ // LINX plotting visualises clusters structural variants // -include { LINXREPORT as REPORT } from '../../modules/local/linxreport/main' -include { LINX_VISUALISER as VISUALISER } from '../../modules/local/linx/visualiser/main' +include { LINXREPORT } from '../../modules/local/linxreport/main' +include { LINX_VISUALISER } from '../../modules/local/linx/visualiser/main' workflow LINX_PLOTTING { take: @@ -32,7 +32,7 @@ workflow LINX_PLOTTING { return [meta_linx, anno_dir] } - VISUALISER( + LINX_VISUALISER( ch_linx_visualiser_inputs, genome_version, ensembl_data_resources, @@ -57,7 +57,7 @@ workflow LINX_PLOTTING { return [meta_linxreport, anno_dir, plot_dir] } - REPORT( + LINXREPORT( ch_linxreport_inputs, ) From 50af35a8e675dd0de11bc2c00d9992f26555ccef Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 18 Jun 2025 14:54:54 +1000 Subject: [PATCH 84/88] change linx channel --- subworkflows/local/linx_plotting.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/linx_plotting.nf b/subworkflows/local/linx_plotting.nf index 180082b8..1cc9dc01 100644 --- a/subworkflows/local/linx_plotting.nf +++ b/subworkflows/local/linx_plotting.nf @@ -38,9 +38,9 @@ workflow LINX_PLOTTING { ensembl_data_resources, ) - ch_versions = ch_versions.mix(VISUALISER.out.versions) + ch_versions = ch_versions.mix(LINX_VISUALISER.out.versions) - ch_visualiser_out = WorkflowSash.restoreMeta(VISUALISER.out.plots, ch_inputs) + ch_visualiser_out = WorkflowSash.restoreMeta(LINX_VISUALISER.out.plots, ch_inputs) // Create inputs and create process-specific meta // channel: [ meta_linxreport, linx_annotation_dir, linx_plot_dir ] @@ -61,7 +61,7 @@ workflow LINX_PLOTTING { ch_linxreport_inputs, ) - ch_versions = ch_versions.mix(REPORT.out.versions) + ch_versions = ch_versions.mix(LINXREPORT.out.versions) emit: plot_dir = ch_visualiser_out // channel: [ meta, plot_dir ] From e82075f837450406be195e9b1033d3f26391a521 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Wed, 18 Jun 2025 16:51:11 +1000 Subject: [PATCH 85/88] update linx config --- conf/modules.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index b729e3af..0dc483aa 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -175,7 +175,7 @@ process { ] } - withName: '.*:LINX_ANNOTATION:(?:GERMLINE|SOMATIC)|.*:LINX_PLOTTING:VISUALISER' { + withName: '.*:LINX_ANNOTATION:(?:GERMLINE|SOMATIC)|.*:LINX_PLOTTING:LINX_VISUALISER' { ext.jarPath = '/opt/linx/linx.jar' } @@ -195,7 +195,7 @@ process { ] } - withName: '.*:LINX_PLOTTING:VISUALISER' { + withName: '.*:LINX_PLOTTING:LINX_VISUALISER' { ext.circosPath = '/opt/conda/bin/circos' publishDir = [ path: { "${params.outdir}" }, @@ -204,7 +204,7 @@ process { ] } - withName: '.*:LINX_PLOTTING:REPORT' { + withName: '.*:LINX_PLOTTING:LINXREPORT' { publishDir = [ path: { "${params.outdir}" }, mode: params.publish_dir_mode, From 3c742b2765e22d3b46504ab1bc116f223652b14c Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 19 Jun 2025 16:17:06 +1000 Subject: [PATCH 86/88] Remove dev tag for full release --- modules/local/bolt/other/cancer_report/main.nf | 2 +- modules/local/bolt/other/multiqc_report/main.nf | 2 +- modules/local/bolt/other/purple_baf_plot/main.nf | 2 +- modules/local/bolt/smlv_germline/prepare/main.nf | 2 +- modules/local/bolt/smlv_germline/report/main.nf | 2 +- modules/local/bolt/smlv_somatic/annotate/main.nf | 2 +- modules/local/bolt/smlv_somatic/filter/main.nf | 2 +- modules/local/bolt/smlv_somatic/report/main.nf | 2 +- modules/local/bolt/smlv_somatic/rescue/main.nf | 2 +- modules/local/bolt/sv_somatic/annotate/main.nf | 2 +- modules/local/bolt/sv_somatic/prioritise/main.nf | 2 +- nextflow.config | 4 ++-- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/local/bolt/other/cancer_report/main.nf b/modules/local/bolt/other/cancer_report/main.nf index 67da0c6c..e54bc8f9 100644 --- a/modules/local/bolt/other/cancer_report/main.nf +++ b/modules/local/bolt/other/cancer_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_CANCER_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-gpgr' + container 'ghcr.io/umccr/bolt:0.2.14-gpgr' input: tuple val(meta), path(smlv_somatic_vcf), path(smlv_somatic_bcftools_stats), path(smlv_somatic_counts_process), path(sv_somatic_tsv), path(sv_somatic_vcf), path(cnv_somatic_tsv), path(af_global), path(af_keygenes), path(purple_baf_plot), path(purple_dir), path(virusbreakend_dir), path(dragen_hrd) diff --git a/modules/local/bolt/other/multiqc_report/main.nf b/modules/local/bolt/other/multiqc_report/main.nf index b42a1453..099ffed6 100644 --- a/modules/local/bolt/other/multiqc_report/main.nf +++ b/modules/local/bolt/other/multiqc_report/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_MULTIQC_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-multiqc' + container 'ghcr.io/umccr/bolt:0.2.14-multiqc' input: tuple val(meta), path(input_files) diff --git a/modules/local/bolt/other/purple_baf_plot/main.nf b/modules/local/bolt/other/purple_baf_plot/main.nf index cb70c8e6..03a676d1 100644 --- a/modules/local/bolt/other/purple_baf_plot/main.nf +++ b/modules/local/bolt/other/purple_baf_plot/main.nf @@ -2,7 +2,7 @@ process BOLT_OTHER_PURPLE_BAF_PLOT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-circos' + container 'ghcr.io/umccr/bolt:0.2.14-circos' input: tuple val(meta), path(purple_dir) diff --git a/modules/local/bolt/smlv_germline/prepare/main.nf b/modules/local/bolt/smlv_germline/prepare/main.nf index 8a0bae23..e3292f23 100644 --- a/modules/local/bolt/smlv_germline/prepare/main.nf +++ b/modules/local/bolt/smlv_germline/prepare/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_GERMLINE_PREPARE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev' + container 'ghcr.io/umccr/bolt:0.2.14' input: tuple val(meta), path(smlv_vcf) diff --git a/modules/local/bolt/smlv_germline/report/main.nf b/modules/local/bolt/smlv_germline/report/main.nf index e5706842..181ea027 100644 --- a/modules/local/bolt/smlv_germline/report/main.nf +++ b/modules/local/bolt/smlv_germline/report/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_GERMLINE_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-pcgr' + container 'ghcr.io/umccr/bolt:0.2.14-pcgr' input: tuple val(meta), path(smlv_vcf), path(smlv_unfiltered_vcf) diff --git a/modules/local/bolt/smlv_somatic/annotate/main.nf b/modules/local/bolt/smlv_somatic/annotate/main.nf index 659a50fc..db49ffc7 100644 --- a/modules/local/bolt/smlv_somatic/annotate/main.nf +++ b/modules/local/bolt/smlv_somatic/annotate/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_ANNOTATE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-pcgr' + container 'ghcr.io/umccr/bolt:0.2.14-pcgr' input: tuple val(meta), path(smlv_vcf) diff --git a/modules/local/bolt/smlv_somatic/filter/main.nf b/modules/local/bolt/smlv_somatic/filter/main.nf index 0208eabe..b31055b5 100644 --- a/modules/local/bolt/smlv_somatic/filter/main.nf +++ b/modules/local/bolt/smlv_somatic/filter/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_FILTER { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev' + container 'ghcr.io/umccr/bolt:0.2.14' input: tuple val(meta), path(smlv_vcf) diff --git a/modules/local/bolt/smlv_somatic/report/main.nf b/modules/local/bolt/smlv_somatic/report/main.nf index 13fe7196..77ea9043 100644 --- a/modules/local/bolt/smlv_somatic/report/main.nf +++ b/modules/local/bolt/smlv_somatic/report/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_REPORT { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-pcgr' + container 'ghcr.io/umccr/bolt:0.2.14-pcgr' input: tuple val(meta), path(smlv_vcf), path(smlv_filters_vcf), path(smlv_dragen_vcf), path(purple_purity) diff --git a/modules/local/bolt/smlv_somatic/rescue/main.nf b/modules/local/bolt/smlv_somatic/rescue/main.nf index e8802fbe..68c39ada 100644 --- a/modules/local/bolt/smlv_somatic/rescue/main.nf +++ b/modules/local/bolt/smlv_somatic/rescue/main.nf @@ -2,7 +2,7 @@ process BOLT_SMLV_SOMATIC_RESCUE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev' + container 'ghcr.io/umccr/bolt:0.2.14' input: tuple val(meta), path(smlv_vcf), path(smlv_tbi), path(sage_smlv_vcf), path(sage_smlv_tbi) diff --git a/modules/local/bolt/sv_somatic/annotate/main.nf b/modules/local/bolt/sv_somatic/annotate/main.nf index 1819ab16..67b31c1d 100644 --- a/modules/local/bolt/sv_somatic/annotate/main.nf +++ b/modules/local/bolt/sv_somatic/annotate/main.nf @@ -2,7 +2,7 @@ process BOLT_SV_SOMATIC_ANNOTATE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev-snpeff' + container 'ghcr.io/umccr/bolt:0.2.14-snpeff' input: tuple val(meta), path(sv_vcf), path(cnv_tsv) diff --git a/modules/local/bolt/sv_somatic/prioritise/main.nf b/modules/local/bolt/sv_somatic/prioritise/main.nf index 616a5043..d820477a 100644 --- a/modules/local/bolt/sv_somatic/prioritise/main.nf +++ b/modules/local/bolt/sv_somatic/prioritise/main.nf @@ -2,7 +2,7 @@ process BOLT_SV_SOMATIC_PRIORITISE { tag "${meta.id}" label 'process_low' - container 'ghcr.io/umccr/bolt:0.2.14-dev' + container 'ghcr.io/umccr/bolt:0.2.14' input: tuple val(meta), path(sv_vcf) diff --git a/nextflow.config b/nextflow.config index 79c1f4fb..b3e7e165 100644 --- a/nextflow.config +++ b/nextflow.config @@ -168,12 +168,12 @@ dag { manifest { name = 'umccr/sash' - author = """Stephen Watts""" + author = """Stephen Watts, Quentin Clayssen""" homePage = 'https://github.com/umccr/sash' description = """Cancer WGTS post-processing pipeline""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.6' - version = 'v0.6.0dev' + version = 'v0.6.0' doi = '' } From 3a28165078b2702de249d3da1683a002fc55fb46 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 19 Jun 2025 16:53:03 +1000 Subject: [PATCH 87/88] Update version format for bumpversion compatibility --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index b3e7e165..fd3cb879 100644 --- a/nextflow.config +++ b/nextflow.config @@ -173,7 +173,7 @@ manifest { description = """Cancer WGTS post-processing pipeline""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.6' - version = 'v0.6.0' + version = '0.5.0' doi = '' } From a4615a6d954588038143fd2eb329b45528ffeb93 Mon Sep 17 00:00:00 2001 From: qclayssen Date: Thu, 19 Jun 2025 17:10:01 +1000 Subject: [PATCH 88/88] =?UTF-8?q?Bump=20version:=200.5.0=20=E2=86=92=200.6?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8a061254..4313c56b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.5.0 +current_version = 0.6.0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P[a-z0-9+]+) diff --git a/nextflow.config b/nextflow.config index fd3cb879..9a5a41e6 100644 --- a/nextflow.config +++ b/nextflow.config @@ -173,7 +173,7 @@ manifest { description = """Cancer WGTS post-processing pipeline""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.6' - version = '0.5.0' + version = '0.6.0' doi = '' }