diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000..934dfabc
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,57 @@
+name: CI
+
+on:
+ push:
+ branches: [ main, develop ]
+ pull_request:
+ types: [ opened, synchronize, reopened ]
+ workflow_dispatch:
+
+jobs:
+ test:
+ runs-on: ubuntu-24.04
+ env:
+ NXF_VER: "25.04.7"
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Clean up Disk space
+ uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
+
+ - name: Set up Java (OpenJDK 17)
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: '17'
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.9"
+
+ - name: Ensure pip is up-to-date
+ run: python -m pip install --upgrade pip
+
+ - name: Set up Nextflow
+ uses: nf-core/setup-nextflow@v2
+ with:
+ version: "${{ env.NXF_VER }}"
+
+ - name: Download and extract test-data
+ # If your data is public this will work. If it's private, see notes below.
+ run: |
+ wget -O test-data.tar.gz --no-check-certificate 'https://drive.usercontent.google.com/download?export=download&confirm=no_antivirus&id=13zUVw4BZ0_5QAW7zmdCsZ_CZDHbNxyMV'
+ tar -xzvf test-data.tar.gz
+
+ - name: Install Python deps (if requirements.txt exists)
+ run: |
+ if [ -f requirements.txt ]; then
+ pip install -r requirements.txt
+ fi
+
+ - name: Run tests
+ run: |
+ python3 tests/test_runner.py tests/tests.json
+
diff --git a/.gitignore b/.gitignore
index c0fc93ab..4cfb8656 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,10 @@ nf.*
*.gz
*.bam.*
Result/
+Results/
+result/
+results/
*.Rhistory
*.tsv
docs/node_modules
+*.pyc
diff --git a/.nvmrc b/.nvmrc
new file mode 100644
index 00000000..3c032078
--- /dev/null
+++ b/.nvmrc
@@ -0,0 +1 @@
+18
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 7de08969..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-sudo: required
-language: java
-jdk: openjdk8
-
-os:
- - linux
-
-services:
- - docker
-
-before_install:
- # Install Nextflow
- - export NXF_VER=19.10.0 && curl -fsSL get.nextflow.io | bash
- - chmod 777 nextflow
- # to change the test-data for travis, please download using the following command, extract, make changes, tarball again with gzip, and upload to google drive.
- # you will have to change the link below as well. Click to share the link, making it so anyone with the link can access, then extract the id in the link and put it here after "id="
- - wget -O test-data.tar.gz --no-check-certificate 'https://docs.google.com/uc?export=download&id=10qEFp0KXY25hBdzykD-wHLFTMH--QloF'
- - tar -xzvf test-data.tar.gz
-
-script:
- # Run test script here
- - python3 tests/test_runner.py tests/tests.json
diff --git a/README.md b/README.md
index f673b503..15579db8 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,75 @@
-[](https://travis-ci.com/mskcc/tempo)
-# Note: Tempo currently only supports human samples. The pipeline has only been tested for exome and genome sequencing experiments, and all reference files are in build GRCh37 of the human genome.
+
+---
+
+## Table of Contents
+
+### 1. Getting Started
+
+#### 1.1. Setup
+* [Installation](docs/installation.md)
+* [Setup on Juno](docs/juno-setup.md)
+* [Setup on AWS](docs/aws-setup.md)
+
+#### 1.2. Usage
+* [Running the Pipeline](docs/running-the-pipeline.md)
+ * [Overview](docs/running-the-pipeline.md#overview)
+ * [Input Files](docs/running-the-pipeline.md#input-files)
+ * [Execution Mode](docs/running-the-pipeline.md#execution-mode)
+ * [Modifying or Resuming Pipeline Run](docs/running-the-pipeline.md#modifying-or-resuming-pipeline-run)
+ * [After Successful Run](docs/running-the-pipeline.md#after-successful-run)
+* [Nextflow Basics](docs/nextflow-basics.md)
+* [Working With Containers](docs/working-with-containers.md)
+
+#### 1.3 Outputs
+* [BAM Files](docs/outputs.md#bam-files)
+* [QC Outputs](docs/outputs.md#qc-outputs)
+* [Somatic Data](docs/outputs.md#somatic-data)
+* [Germline Data](docs/outputs.md#germline-data)
+* [Cohort Level Outputs](docs/outputs.md#cohort-level-outputs)
+
+### 2. Pipeline contents
+
+#### 2.1. Bioinformatic Components
+* [Read Alignment](docs/bioinformatic-components.md#read-alignment)
+* [Somatic Analyses](docs/bioinformatic-components.md#somatic-analyses)
+* [Germline Analyses](docs/bioinformatic-components.md#germline-analyses)
+* [Quality Control](docs/bioinformatic-components.md#quality-control)
+
+#### 2.2. Reference Resources
+* [Genome Assembly](docs/reference-files.md#genome-assembly)
+* [Genomic Intervals](docs/reference-files.md#genomic-intervals)
+* [RepeatMasker and Mappability Blacklist](docs/reference-files.md#repeatmasker-and-mappability-blacklist)
+* [Preferred Transcript Isoforms](docs/reference-files.md#preferred-transcript-isoforms)
+* [Hotspot Annotation](docs/reference-files.md#hotspot-annotation.md)
+* [OncoKB Annotation](docs/reference-files.md#oncokb.md)
+* [gnomAD](docs/gnomad.md)
+* [Panel of Normals for Exomes](docs/wes-panel-of-normals.md)
+
+#### 2.3. Variant Annotation and Filtering
+* [Somatic SNVs and Indels](docs/variant-annotation-and-filtering.md#somatic-snvs-and-indels)
+* [Germline SNVs and Indels](docs/variant-annotation-and-filtering.md#germline-snvs-and-indels)
+* [Somatic and Germline SVs](docs/variant-annotation-and-filtering.md#somatic-and-germline-svs)
+
+### 3. Help and Other Resources
+* [Troubleshooting](docs/troubleshooting.md)
+* [AWS Glossary](docs/aws-glossary.md)
+
+### 4. Contributing
+* [Contributing to Tempo](docs/contributing-to-tempo.md)
+
+### 5. Acknowledgements
+* [Acknowledgements](docs/acknowledgements.md)
-The inspiration for this project derives from [Sarek](https://github.com/SciLifeLab/Sarek), developed at [SciLifeLab](https://github.com/SciLifeLab).
## Pipeline Flowchart
@@ -22,3 +85,4 @@ The inspiration for this project derives from [Sarek](https://github.com/SciLife
+---
diff --git a/conf/awsbatch.config.template b/conf/awsbatch.config.template
index 00d70f47..5494dd23 100644
--- a/conf/awsbatch.config.template
+++ b/conf/awsbatch.config.template
@@ -20,8 +20,8 @@ executor {
process {
queue =
scratch = true
- errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }
maxRetries = 3
+ errorStrategy = { task.attempt <= process.maxRetries ? 'retry' : 'ignore' }
}
params {
diff --git a/conf/containers.config b/conf/containers.config
index 5e2ddf4d..ea228832 100755
--- a/conf/containers.config
+++ b/conf/containers.config
@@ -24,29 +24,29 @@
//------------------- Somatic pipeline
- withName:SomaticDellyCall {
- container = "cmopipeline/delly-bcftools:0.0.1"
- }
withName:RunMutect2 {
container = "broadinstitute/gatk:4.1.0.0"
}
withName:SomaticCombineMutect2Vcf {
container = "cmopipeline/bcftools-vt:1.2.0"
}
- withName:SomaticRunManta {
- container = "cmopipeline/strelka2_manta:latest"
+ withName:SomaticMergeSVs {
+ container = "cmopipeline/bcftools-vt-mergesvvcf:0.0.1"
}
- withName:SomaticMergeDellyAndManta {
- container = "cmopipeline/bcftools-vt:1.2.0"
+ withName:SomaticSVVcf2Bedpe {
+ container = "cmopipeline/svtools:0.0.3"
+ }
+ withName:SomaticAnnotateSVBedpe {
+ container = "cmopipeline/iannotatesv:0.0.2"
}
withName:"SomaticRunStrelka2.*" {
- container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.0"
+ container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
}
withName:SomaticCombineChannel {
container = "cmopipeline/bcftools-vt:1.2.3"
}
withName:SomaticAnnotateMaf {
- container = "cmopipeline/vcf2maf:vep88_1.2.7"
+ container = "cmopipeline/vcf2maf:vep88_1.3.0"
}
withName:DoFacets {
container = "cmopipeline/facets-suite-preview-htstools:0.0.1"
@@ -70,12 +70,47 @@
container = "cmopipeline/facets-suite-preview-htstools:0.0.1"
}
withName:RunNeoantigen {
- container = "cmopipeline/neoantigen:0.3.2-hotfix"
+ container = "cmopipeline/neoantigen:0.3.3"
}
withName:MetaDataParser {
container = "cmopipeline/metadataparser:0.5.9"
}
-
+ withName:SomaticDellyCall {
+ container = "cmopipeline/delly-bcftools:0.0.1"
+ }
+ withName:'.*DellyCombine' {
+ container = "cmopipeline/delly-bcftools:0.0.1"
+ }
+ withName:SomaticRunManta {
+ container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
+ }
+ withName: '.*RunSvABA' {
+ container = "cmopipeline/svaba:0.0.1"
+ }
+ withName: 'runBRASS.*' {
+ container = "cmopipeline/brass:0.0.2"
+ }
+ withName:generateBasFile {
+ container = "quay.io/wtsicgp/pcap-core:5.5.0"
+ }
+ withLabel:ascat {
+ container = "quay.io/wtsicgp/ascatNgs:4.4.0"
+ }
+ withName:HRDetect {
+ container = "cmopipeline/signaturetoolslib:0.0.1"
+ }
+ withName:RunSVSignatures {
+ container = "cmopipeline/signaturetoolslib:0.0.1"
+ }
+ withName:SomaticRunSVclone {
+ container = "cmopipeline/svclone:0.0.1"
+ }
+ withName:SomaticRunClusterSV {
+ container = "cmopipeline/clustersv:0.0.1"
+ }
+ withName:SomaticRunSVCircos {
+ container = "cmopipeline/biocircos:0.0.1"
+ }
//------------------- Germline pipeline
@@ -89,10 +124,10 @@
container = "cmopipeline/bcftools-vt:1.1.1"
}
withName:GermlineRunManta {
- container = "cmopipeline/strelka2_manta:latest"
+ container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
}
withName:"GermlineRunStrelka2" {
- container = "cmopipeline/strelka2_manta:latest"
+ container = "cmopipeline/strelka2-manta-bcftools-vt:2.0.1"
}
withName:GermlineCombineChannel {
container = "cmopipeline/bcftools-vt:1.2.2"
@@ -103,10 +138,15 @@
withName:GermlineFacetsAnnotation {
container = "cmopipeline/facets-suite-preview-htstools:0.0.1"
}
- withName:GermlineMergeDellyAndManta {
- container = "cmopipeline/bcftools-vt:1.1.1"
+ withName:GermlineMergeSVs {
+ container = "cmopipeline/bcftools-vt-mergesvvcf:0.0.1"
+ }
+ withName:GermlineSVVcf2Bedpe {
+ container = "cmopipeline/svtools:0.0.3"
+ }
+ withName:GermlineAnnotateSVBedpe {
+ container = "cmopipeline/iannotatesv:0.0.2"
}
-
//------------------- Quality Control
diff --git a/conf/genome.config b/conf/genome.config
index 9712770f..ccee632d 100644
--- a/conf/genome.config
+++ b/conf/genome.config
@@ -36,4 +36,7 @@ params {
minCoverageFilter = 10
}
scatterCount = 1000
+ ascat {
+ alleleCountLimit = ["GRCh37","smallGRCh37","GRCh38"].contains(params.genome) ? 48 : 1
+ }
}
diff --git a/conf/juno.config b/conf/juno.config
index 55486450..eedc5de0 100644
--- a/conf/juno.config
+++ b/conf/juno.config
@@ -18,15 +18,14 @@ process {
clusterOptions = ""
scratch = true
beforeScript = "module load singularity/3.1.1; unset R_LIBS; catch_term () { echo 'caught USR2/TERM signal'; set +e; false; on_exit ; } ; trap catch_term USR2 TERM"
+ maxRetries = 3
+ errorStrategy = { task.attempt <= process.maxRetries ? 'retry' : 'ignore' }
}
-process.errorStrategy = { task.attempt <= 3 ? 'retry' : 'ignore' }
-process.maxRetries = 3
-
params {
max_memory = "128.GB"
mem_per_core = true
- reference_base = "/juno/work/taylorlab/cmopipeline"
+ reference_base = "/juno/work/tempo/cmopipeline"
// targets_base = "/juno/work/ccs/resources/tempo/${params.genome}"
targets_base = "${reference_base}/mskcc-igenomes/${params.genome.toLowerCase()}/tempo_targets"
genome_base = params.genome == 'GRCh37' ? "${reference_base}/mskcc-igenomes/igenomes/Homo_sapiens/GATK/GRCh37" : params.genome == 'GRCh38' ? "${reference_base}/mskcc-igenomes/igenomes/Homo_sapiens/GATK/GRCh38" : "${reference_base}/mskcc-igenomes/igenomes/smallGRCh37"
diff --git a/conf/references.config b/conf/references.config
index dd61fca3..abcfc63d 100644
--- a/conf/references.config
+++ b/conf/references.config
@@ -35,14 +35,35 @@ params {
knownIndelsIndex = "${params.genome_base}/{1000G_phase1,Mills_and_1000G_gold_standard}.indels.b37.small.vcf.idx"
msiSensorList = "${params.genome_base}/small.msi.list"
snpeffDb = "GRCh37.75"
- vepCacheVersion = "95"
+ vepCache = "${params.reference_base}/vep"
+ vepCacheVersion = "88"
+ facetsVcf = "${params.genome_base}/dbsnp_138.b37.small.vcf"
svCallingExcludeRegions = "${params.genome_base}/human.hg19.excl.tsv"
svCallingIncludeRegions = "${params.genome_base}/b37.test.bed.gz"
svCallingIncludeRegionsIndex = "${svCallingIncludeRegions}.tbi"
+ repeatMasker = "${params.reference_base}/annotation/rmsk_mod.bed.gz"
+ repeatMaskerIndex = "${repeatMasker}.tbi"
+ mapabilityBlacklist = "${params.reference_base}/annotation/wgEncodeDacMapabilityConsensusExcludable.bed.gz"
+ mapabilityBlacklistIndex = "${mapabilityBlacklist}.tbi"
+ isoforms = "${params.reference_base}/annotation/isoforms"
exomePoN = "${params.genome_base}/pon_test.vcf.gz"
exomePoNIndex = "${exomePoN}.tbi"
wgsPoN = "${params.genome_base}/pon_test.1.vcf.gz"
wgsPoNIndex = "${wgsPoN}.tbi"
+ snpGcCorrections = "${params.genome_base}/SnpGcCorrections.small.tsv"
+ spliceSites = "${params.reference_base}/annotation/splice_sites.small.bed"
+ gnomadWesVcf = "${params.reference_base}/gnomad/gnomad.exomes.r2.1.1.sites.non_cancer.vcf.gz"
+ gnomadWesVcfIndex = "${gnomadWesVcf}.tbi"
+ gnomadWgsVcf = "${params.reference_base}/gnomad/gnomad.genome.r2.1.1.sites.minimal.vcf.gz"
+ gnomadWgsVcfIndex = "${gnomadWgsVcf}.tbi"
+ hlaFasta = "${params.reference_base}/hla/abc_complete.fasta"
+ hlaDat = "${params.reference_base}/hla/hla.dat"
+ neoantigenCDNA = "${params.reference_base}/neoantigen/Homo_sapiens.GRCh37.75.cdna.all.fa.gz"
+ neoantigenCDS = "${params.reference_base}/neoantigen/Homo_sapiens.GRCh37.75.cds.all.fa.gz"
+ svBlacklistBed = "${params.genome_base}/sv_calling/pcawg6_blacklist.slop.bed.gz"
+ svBlacklistBedpe = "${params.genome_base}/sv_calling/pcawg6_blacklist.slop.trunc.bedpe.gz"
+ svBlacklistFoldbackBedpe = "${params.genome_base}/sv_calling/pcawg6_blacklist_foldback_artefacts.slop.trunc.bedpe.gz"
+ svBlacklistTEBedpe = "${params.genome_base}/sv_calling/pcawg6_blacklist_TE_pseudogene.bedpe.gz"
}
'GRCh37' {
acLoci = "${params.genome_base}/Annotation/ASCAT/1000G_phase3_20130502_SNP_maf0.3.loci"
@@ -80,6 +101,15 @@ params {
hlaDat = "${params.reference_base}/mskcc-igenomes/grch37/hla/hla.dat"
neoantigenCDNA = "${params.reference_base}/mskcc-igenomes/grch37/neoantigen/Homo_sapiens.GRCh37.75.cdna.all.fa.gz"
neoantigenCDS = "${params.reference_base}/mskcc-igenomes/grch37/neoantigen/Homo_sapiens.GRCh37.75.cds.all.fa.gz"
+ snpGcCorrections = "${params.reference_base}/mskcc-igenomes/grch37/ascat/SnpGcCorrections.tsv"
+ spliceSites = "${params.reference_base}/mskcc-igenomes/grch37/splice_sites/splice_sites.bed"
+ brassRefDir = "${params.reference_base}/mskcc-igenomes/grch37/brass"
+ vagrentRefDir = "${params.reference_base}/mskcc-igenomes/grch37/vagrent"
+ // svBlacklist* source: https://bitbucket.org/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/
+ svBlacklistBed = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist.slop.bed.gz"
+ svBlacklistBedpe = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist.slop.bedpe.gz"
+ svBlacklistFoldbackBedpe = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist_foldback_artefacts.slop.bedpe.gz"
+ svBlacklistTEBedpe = "${params.reference_base}/mskcc-igenomes/grch37/sv_calling/pcawg6_blacklist_TE_pseudogene.bedpe.gz"
}
'GRCh38' {
acLoci = "${params.genome_base}/Annotation/ASCAT/1000G_phase3_GRCh38_maf0.3.loci"
@@ -101,6 +131,7 @@ params {
//AF_indexes = "${params.genome_base}/{00-All.dbsnp_151.hg38.CAF.TOPMED.alternate.allele.freq,hapmap_3.3_grch38_pop_stratified_af.HMAF,SweGen_hg38_stratified.SWAF}.vcf.idx"
hlaFasta = "${params.reference_base}/mskcc-igenomes/grch38/hla/abc_complete.fasta"
hlaDat = "${params.reference_base}/mskcc-igenomes/grch38/hla/hla.dat"
- }
+ snpGcCorrections = "${params.reference_base}/mskcc-igenomes/grch38/ascat/SnpGcCorrections.tsv"
+ }
}
}
diff --git a/conf/resources.config b/conf/resources.config
index 7e75d380..0ceeade0 100755
--- a/conf/resources.config
+++ b/conf/resources.config
@@ -7,8 +7,13 @@
*/
//------------- Read alignment
-
process {
+ withName:CrossValidateSamples {
+ cpus = { 1 }
+ memory = { 1.GB }
+ executor = 'local'
+ }
+
withName:SplitLanesR1 {
cpus = { 1 }
memory = { 1.GB }
@@ -64,7 +69,7 @@
cpus = { 2 }
memory = { 6.GB }
}
- withName:SomaticMergeDellyAndManta {
+ withName:SomaticMergeSVs {
cpus = { 2 }
memory = { 6.GB }
}
@@ -89,8 +94,8 @@
memory = { 6.GB }
}
withName:RunLOHHLA {
- cpus = { 4 }
- memory = { 8.GB }
+ cpus = { 2 }
+ memory = { 6.GB }
}
withName:RunMutationSignatures {
cpus = { 2 }
@@ -102,7 +107,7 @@
}
withName:MetaDataParser {
cpus = { 1 }
- memory = { 8.GB }
+ memory = { 6.GB }
}
//------------- Germline pipeline
@@ -139,7 +144,7 @@
cpus = { 1 }
memory = { 1.GB }
}
- withName:GermlineMergeDellyAndManta {
+ withName:GermlineMergeSVs {
cpus = { 1 }
memory = { 1.GB }
}
diff --git a/conf/resources_aws.config b/conf/resources_aws.config
index 58648c18..024c0b20 100755
--- a/conf/resources_aws.config
+++ b/conf/resources_aws.config
@@ -68,7 +68,7 @@
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
- withName:SomaticMergeDellyAndManta {
+ withName:SomaticMergeSVs {
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
@@ -139,7 +139,7 @@
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
- withName:GermlineMergeDellyAndManta {
+ withName:GermlineMergeSVs {
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
diff --git a/conf/resources_aws_genome.config b/conf/resources_aws_genome.config
index c5bd2107..02855ce3 100644
--- a/conf/resources_aws_genome.config
+++ b/conf/resources_aws_genome.config
@@ -68,7 +68,7 @@
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
- withName:SomaticMergeDellyAndManta {
+ withName:SomaticMergeSVs {
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
@@ -80,6 +80,10 @@
cpus = { 2 }
memory = { 4.GB * task.attempt }
}
+ withName:HRDetect {
+ cpus = { 2 }
+ memory = { 4.GB * task.attempt }
+ }
withName:RunMsiSensor {
cpus = { 1 }
memory = { task.attempt < 3 ? 3.GB * task.attempt : 6.GB * task.attempt}
@@ -139,7 +143,7 @@
cpus = { 1 }
memory = { 4.GB * task.attempt }
}
- withName:GermlineMergeDellyAndManta {
+ withName:GermlineMergeSVs {
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
diff --git a/conf/resources_juno.config b/conf/resources_juno.config
index da5d7d8b..556b0070 100755
--- a/conf/resources_juno.config
+++ b/conf/resources_juno.config
@@ -9,6 +9,11 @@
//------------- Read alignment
process {
+ withName:CrossValidateSamples {
+ cpus = { 1 }
+ memory = { 1.GB }
+ executor = 'local'
+ }
withName:SplitLanesR1 {
cpus = { 1 }
memory = { 1.GB }
@@ -52,6 +57,11 @@
cpus = { 8 }
memory = { 1.GB * task.attempt }
}
+ withName: '.*RunSvABA' {
+ cpus = { 8 }
+ memory = { 2.GB * task.attempt }
+ time = { task.attempt < 3 ? 30.h * task.attempt : 500.h }
+ }
withName:SomaticRunStrelka2 {
cpus = { 8 }
memory = { 1.GB * task.attempt }
@@ -64,9 +74,9 @@
cpus = { 1 }
memory = { 8.GB * task.attempt }
}
- withName:SomaticMergeDellyAndManta {
- cpus = { 1 }
- memory = { 1.GB * task.attempt }
+ withName:SomaticMergeSVs {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { task.attempt < 3 ? 1.GB * task.attempt : 5.Gb * task.attempt }
}
withName:DoFacets {
cpus = { 1 }
@@ -97,13 +107,21 @@
memory = { 1.GB * task.attempt }
}
withName:RunNeoantigen {
- cpus = { 1 }
- memory = { 8.GB * task.attempt * 2 }
+ cpus = { 4 * task.attempt }
+ memory = { 4.GB * task.attempt * 2 }
}
withName:MetaDataParser {
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
+ withName:SomaticSVVcf2Bedpe {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { task.attempt < 3 ? 1.GB * task.attempt : 5.Gb * task.attempt }
+ }
+ withName:SomaticAnnotateSVBedpe {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { 8.GB * task.attempt }
+ }
//------------- Germline pipeline
@@ -139,9 +157,17 @@
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
- withName:GermlineMergeDellyAndManta {
- cpus = { 1 }
- memory = { 1.GB * task.attempt }
+ withName:GermlineMergeSVs {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { task.attempt < 3 ? 1.GB * task.attempt : 5.Gb * task.attempt }
+ }
+ withName:GermlineSVVcf2Bedpe {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { task.attempt < 3 ? 1.GB * task.attempt : 5.Gb * task.attempt }
+ }
+ withName:GermlineAnnotateSVBedpe {
+ cpus = { task.attempt < 4 ? 2 : 3 }
+ memory = { 8.GB * task.attempt }
}
//------------- Quality Control
diff --git a/conf/resources_juno_genome.config b/conf/resources_juno_genome.config
index c32ac12d..5eda48c7 100644
--- a/conf/resources_juno_genome.config
+++ b/conf/resources_juno_genome.config
@@ -9,6 +9,11 @@
//------------- Read alignment
process {
+ withName:CrossValidateSamples {
+ cpus = { 1 }
+ memory = { 1.GB }
+ }
+
withName:SplitLanesR1 {
cpus = { 1 }
memory = { 1.GB }
@@ -37,9 +42,28 @@
//------------- Somatic pipeline
withName:SomaticDellyCall {
+ cpus = { 1 + (task.attempt * 1) }
+ memory = { 10.GB }
+ time = { task.attempt < 2 ? 100.h : 500.h }
+ }
+ withName: '.*RunSvABA' {
+ cpus = { 8 }
+ memory = { 4.GB * task.attempt }
+ time = { task.attempt < 3 ? 30.h * task.attempt : 500.h }
+ }
+ withName:'runBRASS.+' {
cpus = { 1 }
- memory = { 16.GB * task.attempt }
- time = { 500.h }
+ memory = { 3.GB * task.attempt }
+ time = { task.attempt < 3 ? 8.h * task.attempt : 500.h }
+ }
+ withName:runBRASS {
+ cpus = { 2 }
+ memory = { 8.GB * task.attempt }
+ time = { task.attempt < 3 ? 10.h * task.attempt : 500.h }
+ }
+ withName:generateBasFile {
+ cpus = { 2 }
+ memory = { 4.GB * task.attempt }
}
withName:RunMutect2 {
cpus = { 1 }
@@ -71,9 +95,9 @@
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
- withName:SomaticMergeDellyAndManta {
- cpus = { 1 }
- memory = { 1.GB * task.attempt }
+ withName:SomaticMergeSVs {
+ cpus = { 2 }
+ memory = { task.attempt < 3 ? 3.GB * task.attempt : 5.Gb * task.attempt }
}
withName:DoFacets {
cpus = { 1 }
@@ -85,6 +109,14 @@
memory = { 4.GB * task.attempt }
time = { 1.h * task.attempt }
}
+ withName:runAscat {
+ cpus = { 2 }
+ memory = { 4.GB * task.attempt }
+ }
+ withName:runAscatAlleleCount{
+ cpus = { 1 }
+ memory = { 4.GB * task.attempt }
+ }
withName:RunMsiSensor {
cpus = { 1 }
memory = { task.attempt < 3 ? 3.GB * task.attempt : 6.GB * task.attempt }
@@ -104,14 +136,30 @@
memory = { 1.GB * task.attempt }
}
withName:RunNeoantigen {
- cpus = { 1 }
- memory = { 8.GB * task.attempt * 2 }
+ cpus = { 8 * task.attempt }
+ memory = { 4.GB * task.attempt * 2 }
time = { task.attempt < 2 ? 6.h : 500.h }
}
withName:MetaDataParser {
cpus = { 1 }
memory = { 1.GB * task.attempt }
}
+ withLabel:ascat {
+ cpus = { 2 }
+ memory = { 4.GB * task.attempt }
+ }
+ withName:SomaticSVVcf2Bedpe {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { 4.GB * task.attempt }
+ }
+ withName:SomaticAnnotateSVBedpe {
+ cpus = { task.attempt < 3 ? 2 : 4 }
+ memory = { 8.GB * task.attempt }
+ }
+ withName:HRDetect {
+ cpus = { 2 }
+ memory = { 4.GB * task.attempt }
+ }
//------------- Germline pipeline
@@ -151,9 +199,17 @@
cpus = { 1 }
memory = { 4.GB * task.attempt }
}
- withName:GermlineMergeDellyAndManta {
- cpus = { 1 }
- memory = { 1.GB * task.attempt }
+ withName:GermlineMergeSVs {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { task.attempt < 3 ? 1.GB * task.attempt : 5.Gb * task.attempt }
+ }
+ withName:GermlineSVVcf2Bedpe {
+ cpus = { task.attempt < 4 ? 1 : 2 }
+ memory = { task.attempt < 3 ? 1.GB * task.attempt : 5.Gb * task.attempt }
+ }
+ withName:GermlineAnnotateSVBedpe {
+ cpus = { task.attempt < 3 ? 2 : 4 }
+ memory = { 8.GB * task.attempt }
}
//------------- Quality Control
@@ -205,6 +261,10 @@
cpus = { 1 }
memory = { 1.GB * task.attempt}
}
+ withName:SomaticAggregateHRDetect {
+ cpus = { 1 }
+ memory = { 1.GB * task.attempt}
+ }
withName:GermlineAggregateMaf {
cpus = { 1 }
memory = { 1.GB * task.attempt }
diff --git a/conf/singularity.config b/conf/singularity.config
index 469d9c56..2b0ffa0a 100644
--- a/conf/singularity.config
+++ b/conf/singularity.config
@@ -13,5 +13,5 @@ singularity {
}
process {
- beforeScript = "unset R_LIBS"
+ beforeScript = "module load singularity/3.1.1; unset R_LIBS"
}
diff --git a/conf/test.config b/conf/test.config
index 33fedf07..c35f19da 100644
--- a/conf/test.config
+++ b/conf/test.config
@@ -13,7 +13,10 @@ params {
genome = "smallGRCh37"
mem_per_core = false
reference_base = "test-data/reference"
- targets_base = "test-data/targets/GRCh37"
- genome_base = "${reference_base}"
+ targets_base = "test-data/targets"
+ genome_base = "test-data/genome"
splitLanes = false
}
+
+process.maxRetries = 3
+process.errorStrategy = { task.attempt <= process.maxRetries ? 'retry' : 'ignore' }
diff --git a/containers/bcftools-vt-mergesvvcf/Dockerfile b/containers/bcftools-vt-mergesvvcf/Dockerfile
new file mode 100644
index 00000000..d8871a0e
--- /dev/null
+++ b/containers/bcftools-vt-mergesvvcf/Dockerfile
@@ -0,0 +1,73 @@
+FROM halllab/bcftools:v1.9
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ contributor="C. Allan Bolipata (bolipatc@mskcc.org); Philip Jonsson (jonssonp@mskcc.org)" \
+ version.image="0.0.1" \
+ version.vt="0.57721" \
+ version.filter-vcf="0.2.2" \
+ version.pysam="0.15.2" \
+ source.getBaseCountsMultiSample="https://github.com/zengzheng123/GetBaseCountsMultiSample/releases/tag/v1.2.2" \
+ version.getBaseCountsMultiSample="1.2.2"
+
+ENV GBCMS_VERSION 1.2.2
+ENV mergeSVvcf_version "1.0.2"
+
+RUN apt-get update && \
+ apt-get install --yes \
+ procps \
+ gcc \
+ make \
+ cmake \
+ zlib1g-dev \
+ libbz2-dev \
+ liblzma-dev \
+ libssl-dev \
+ libcurl4-openssl-dev \
+ g++ \
+ git \
+ wget \
+ zip \
+ python-pip && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+# Install pysam
+RUN pip install pysam==0.15.2
+
+# Install vt
+RUN git clone https://github.com/atks/vt.git && \
+ cd vt && \
+ git submodule update --init --recursive && \
+ make && \
+ cp /vt/vt /usr/bin
+
+# Install GetBaseCountsMultiSample, copied from https://github.com/mskcc/roslin-variant/blob/2.5.x/build/containers/getbasecountsmultisample/1.2.2/Dockerfile
+RUN cd /tmp && \
+ wget https://github.com/zengzheng123/GetBaseCountsMultiSample/archive/v${GBCMS_VERSION}.zip && \
+ unzip v${GBCMS_VERSION}.zip && \
+ # Install bamtools
+ cd /tmp/GetBaseCountsMultiSample-${GBCMS_VERSION}/bamtools-master && \
+ rm -r build/ && \
+ mkdir build && \
+ cd build/ && \
+ cmake -DCMAKE_CXX_FLAGS=-std=c++03 .. && \
+ make && \
+ make install && \
+ cp ../lib/libbamtools.so.2.3.0 /usr/lib/ && \
+ # Install GetBaseCountsMultiSample itself
+ cd /tmp/GetBaseCountsMultiSample-${GBCMS_VERSION} && \
+ make && \
+ cp GetBaseCountsMultiSample /usr/bin/
+
+# Add filter script
+COPY filter-vcf.py /usr/bin
+RUN chmod +x /usr/bin/filter-vcf.py
+
+# Add mergesvvcf package
+RUN mkdir -p /tmp && cd /tmp \
+ && wget https://github.com/papaemmelab/mergeSVvcf/archive/v${mergeSVvcf_version}.tar.gz \
+ && tar xvzf v${mergeSVvcf_version}.tar.gz
+RUN cd /tmp/mergeSVvcf-${mergeSVvcf_version} && sed -i "s/(not svtype or svtype == \"BND\") and / /g" mergesvvcf/vcftobreakpoints.pyx && pip install .
+RUN chmod +x /tmp/mergeSVvcf-${mergeSVvcf_version}/tests/test_pymergevcfs.py && python /tmp/mergeSVvcf-${mergeSVvcf_version}/tests/test_pymergevcfs.py
+RUN rm -rf /tmp/
+
diff --git a/containers/bcftools-vt-mergesvvcf/filter-sv-vcf.py b/containers/bcftools-vt-mergesvvcf/filter-sv-vcf.py
new file mode 100644
index 00000000..cc6847d4
--- /dev/null
+++ b/containers/bcftools-vt-mergesvvcf/filter-sv-vcf.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+import argparse
+
+"""
+Perform custom filtering/annotation of variants in
+VCF file, after merging SVs with mergesvvcf. Variants
+with a minimum of PASSing callers should be filtered
+as PASS with all non-PASS filters recorded in INFO.
+Additionally, TRA is converted to BND because svtools
+will not correctly handle TRA.
+Usage: filter-sv-vcf.py -h
+"""
+
+__author__ = "Anne Marie Noronha"
+__email__ = "noronhaa@mskcc.org"
+__version__ = "0.0.1"
+__status__ = "Dev"
+
+import sys, os
+from pysam import VariantFile # version >= 0.15.2
+from itertools import groupby
+
+
+def get_flag_sources(filters, callers):
+ parsed_filters = [(i,j[len(i + "_"):]) for i in callers for j in filters if j.startswith(i + "_")]
+ dict_1=dict()
+ for caller,val in parsed_filters:
+ dict_1.setdefault(caller, []).append(val)
+ return dict_1
+
+def append_file_to_file(_from, _to):
+ block_size = 1024*1024
+ with open(_from, "rb") as infile, open(_to,"ab") as outfile:
+ while True:
+ input_block = infile.read(block_size)
+ if not input_block:
+ break
+ outfile.write(input_block)
+ infile.close()
+ outfile.close()
+
+def usage():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--input', help = 'input file', required = True)
+ parser.add_argument('--output', help = 'output file', required = True)
+ parser.add_argument('--min',type=int, default = 1, help = 'minimum number of PASS callers needed to PASS final variant' , required = True)
+ return parser.parse_args()
+
+def main():
+ args = usage()
+ filter_by_pass_callers(args.input, args.output, args.min)
+
+def filter_by_pass_callers(input,output,min_pass):
+ vcf_in = VariantFile(input, "r")
+ vcf_in.header.info.add(
+ "NumCallersPass", 1, "Integer", "Number of callers that made this call without filters"
+ )
+ vcf_in.header.filters.add(
+ "minPassFilter", None,None, "Number of callers insufficient"
+ )
+
+ all_callers = set()
+
+ out_vcf_recs = open("vcf.records.tmp", 'w')
+ counter = 0
+ for vcf_rec in vcf_in.fetch():
+ counter +=1
+ ## Variant info
+ info = vcf_rec.info.keys()
+ filter = vcf_rec.filter.keys()
+ print(filter)
+ new_flags = []
+ callers = list(vcf_rec.info["Callers"])
+ num_callers = vcf_rec.info['NumCallers']
+ if num_callers < min_pass:
+ continue
+ for i in set(callers) - all_callers:
+ vcf_in.header.info.add(
+ "{}_filters".format(i),
+ 1,
+ 'String',
+ 'Filter values from {} caller'.format(i)
+ )
+ all_callers.update(set(callers))
+
+ #parse filters
+ flag_sources = get_flag_sources(filter,callers)
+ num_callers_pass = num_callers - len(flag_sources)
+ #annotate with number of passing callers and filter values
+ vcf_rec.info.__setitem__('NumCallersPass',num_callers_pass)
+ for i in flag_sources:
+ vcf_rec.info.__setitem__(i + "_filters",",".join(flag_sources[i]))
+
+ #adjust filter based on passing callers
+ if num_callers_pass >= min_pass:
+ vcf_rec.filter.add("PASS")
+ else:
+ vcf_rec.filter.add("minPassFilter")
+
+ # change TRA to BND for svtools
+ if vcf_rec.info["SVTYPE"] == "TRA":
+ vcf_rec.info.__setitem__("SVTYPE","BND")
+
+ # The following lines are intended to stop the loss of END= in the output.
+ # Later versions of pysam will have fixed this, this code should fixed when pysam upgraded.
+ if not vcf_rec.stop or vcf_rec.stop < 1:
+ #vcf_rec.stop = vcf_rec.pos + 1
+ continue
+ if vcf_rec.chrom == vcf_rec.info["CHR2"] and abs(vcf_rec.start - vcf_rec.stop) <= 1 :
+ #vcf_rec.stop = vcf_rec.pos + 1
+ continue
+
+
+ # print result to tmp file
+ print(vcf_rec, end="", file=out_vcf_recs)
+ out_vcf_recs.close()
+
+ #write header
+ out_vcf_header = open("vcf.header.tmp", 'w')
+ print(vcf_in.header, end="", file=out_vcf_header)
+ out_vcf_header.close()
+
+ #paste together header and records in order
+ if os.path.exists(output):
+ os.remove(output)
+ for infile in ["vcf.header.tmp","vcf.records.tmp"]:
+ append_file_to_file(infile, output)
+
+if __name__ == "__main__":
+ main()
diff --git a/containers/bcftools-vt-mergesvvcf/filter-vcf.py b/containers/bcftools-vt-mergesvvcf/filter-vcf.py
new file mode 100755
index 00000000..45e380a9
--- /dev/null
+++ b/containers/bcftools-vt-mergesvvcf/filter-vcf.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Perform custom annotation of variants in VCF file, based on MuTect2 and Strelka2 variant calls and custom pre-processing.
+Usage: filter-vcf.py input_filename.vcf
+Output: 'input_filename.filter.vcf'
+"""
+
+__author__ = "Philip Jonsson"
+__contributor__ = "Yixiao Gong"
+__email__ = "jonssonp@mskcc.org; gongy@mskcc.org"
+__version__ = "0.2.2"
+__status__ = "Dev"
+
+import sys, os
+from pysam import VariantFile # version >= 0.15.2
+from itertools import groupby
+
+vcf_in = VariantFile(sys.argv[1], "r")
+normal = vcf_in.header.samples[0]
+tumor = vcf_in.header.samples[1]
+
+## Add new headers
+# vcf_in.header.filters.add('multiallelic2', None, None, 'Multiple alleles at same locus') # Note that MuTect2 already has a FILTER tag for this, which requires a new name for this one
+# vcf_in.header.filters.add('part_of_mnv', None, None, 'Variant is part of previous variant')
+# vcf_in.header.filters.add('strand_bias', None, None, 'Variant suffering from strand bias') # Note that MuTect2 already has a FILTER tag for this, but this works
+vcf_in.header.info.add('Ref_Tri', 1, 'String', 'Normalize trinucleotide context of SNVs')
+vcf_in.header.info.add('Custom_filters', '.', 'String', 'Custom filters, semi-colon separated')
+
+## Filter tags not used
+# vcf_in.header.filters.add('short_repeat', None, None, 'Variant part of a short repeat')
+# vcf_in.header.filters.add('caller_conflict', None, None, 'MuTect2 and Strelka2 provides conflicting FILTER flags for this variant')
+
+## Output by default `input_filename.filter.vcf`
+outfile = os.path.splitext(sys.argv[1])[0] + '.filter.vcf'
+vcf_out = VariantFile(outfile, "w", header = vcf_in.header)
+prev_var = None
+
+for var in vcf_in.fetch():
+
+ ## Variant info
+ info = var.info.keys()
+ pos = var.pos
+ ref = var.ref
+ alt = var.alts[0]
+ filter = var.filter.keys()
+ new_flags = []
+
+ ## Add normalized reference trinucleotide context
+ complement = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C'}
+
+ if len(ref) == len(alt) == 1:
+ ref_tri = var.info['FLANKSEQ'][9:14].replace('[', '').replace(']', '')
+ if 'N' not in ref_tri:
+ if ref not in ['C', 'T']:
+ ref_tri = ''.join([complement[nt] for nt in ref_tri])[::-1]
+ var.info.__setitem__('Ref_Tri', ref_tri)
+
+ ## Check for variant substitutions that are part of the previous variant
+ ## These are most likely (but not always?) Strelka2 calls that MuTect2 called as MNVs
+ if prev_var is not None:
+ prev_end = prev_var.pos + len(prev_var.alts[0]) - 1
+ if pos == prev_end:
+ ref_match = ref[0] == prev_var.ref[-1]
+ alt_match = alt[0] == prev_var.alts[0][-1]
+
+ if ref_match and alt_match:
+ new_flags.append('part_of_mnv')
+
+ ## Check for multiallelic2 Strelka2 calls
+ ## Strelka2 does not produce multiallelic calls by itself but provides info on all alternate bases observed
+ if "Strelka2" in info and len(ref) == len(alt) == 1:
+ alleles = { # what about indels from Strelka2?
+ 'A': var.samples[tumor].get('AU', [None]),
+ 'C': var.samples[tumor].get('CU', [None]),
+ 'G': var.samples[tumor].get('GU', [None]),
+ 'T': var.samples[tumor].get('TU', [None])
+ }
+ valid_alleles = [key for key in alleles.keys() if all(isinstance(item,int) for item in alleles[key])]
+ tier1_other = sum([alleles[a][0] for a in valid_alleles if a not in [alt, ref]])
+ tier2_other = sum([alleles[a][1] for a in valid_alleles if a not in [alt, ref]])
+ if alt in valid_alleles:
+ alt_reads = alleles[alt][0]
+ if (tier1_other + tier2_other) >= alt_reads * 0.5:
+ new_flags.append('multiallelic2')
+
+ ## Add an additional strand-bias filer
+ ## Only MuTect2 provides sufficient variant information for this
+ if "MuTect2" in info:
+ t_fw = var.samples[tumor]['F1R2']
+ t_rev = var.samples[tumor]['F2R1']
+ n_fw = var.samples[tumor]['F1R2']
+ n_rev = var.samples[tumor]['F2R1']
+
+ if t_fw[1] == 0 or t_rev[1] == 0: # if all support reads come from one read-pair orientation
+ if t_fw[0] > 10 and t_rev[0] > 10 or n_fw[0] > 10 and n_rev[0] > 10:
+ new_flags.append('strand_bias')
+
+ ## Filters not in use:
+ ## Strand bias
+ # left_flank = var.info['FLANKSEQ'].split('[')[0][::-1]
+ # right_flank = var.info['FLANKSEQ'].split(']')[1]
+ # if len(alt) > len(ref): # insertion
+ # alt_repeat = alt[1:]
+ # elif len(alt) < len(ref): # deletion
+ # alt_repeat = ref[1:]
+ # else:
+ # alt_repeat = alt
+
+ # right_flank = [right_flank[i:i+len(alt_repeat)] for i in range(0, len(right_flank), len(alt_repeat))]
+ # left_flank = [left_flank[i:i+len(alt_repeat)] for i in range(0, len(left_flank), len(alt_repeat))]
+ # right_flank = [(key, len(list(group))) for key, group in groupby(right_flank)]
+ # left_flank = [(key, len(list(group))) for key, group in groupby(left_flank)]
+ # rep_length = len(alt_repeat) * sum([bps[1] for bps in [left_flank[0], right_flank[0]] if bps[0] == alt_repeat])
+
+ # if rep_length > 5:
+ # new_flags.append('short_repeat')
+
+ ## Conflicting MuTect2 and Strelka2 filters
+ if "PASS" in filter and "Strelka2FAIL" in info:
+ new_flags.append('caller_conflict')
+
+ # Add new FILTER tags
+ if len(new_flags) > 0:
+ var.info.__setitem__('Custom_filters', ','.join(new_flags))
+ # if "PASS" in var.filter.keys():
+ # var.filter.clear()
+ # for flag in new_flags:
+ # var.filter.add(flag)
+
+ prev_var = var
+
+ # Write to output
+ vcf_out.write(var)
+
+vcf_out.close()
diff --git a/containers/biocircos/Dockerfile b/containers/biocircos/Dockerfile
new file mode 100644
index 00000000..f20ced0c
--- /dev/null
+++ b/containers/biocircos/Dockerfile
@@ -0,0 +1,21 @@
+FROM rocker/tidyverse:4.2.2
+
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.base="4.2.2" \
+ version.image="0.0.1"
+
+RUN apt-get update && apt-get install -y \
+ zlib1g-dev \
+ libbz2-dev \
+ liblzma-dev \
+ libc6-dev \
+ build-essential \
+ git \
+ ghostscript \
+ libgmp3-dev
+
+RUN R -e "install.packages(c('BioCircos','data.table'))"
+RUN R -e "install.packages('getopt')"
+RUN R -e "install.packages('htmlwidgets')"
+
diff --git a/containers/biocircos/biocircos.Rmd b/containers/biocircos/biocircos.Rmd
new file mode 100644
index 00000000..e727dc6a
--- /dev/null
+++ b/containers/biocircos/biocircos.Rmd
@@ -0,0 +1,14 @@
+---
+title: "SV and CNV Circos Plot"
+author: "TEMPO Pipeline"
+output: html_document
+---
+
+The following interactive plot was generated using the [BioCircos package](https://github.com/lvulliard/BioCircos.R).
+
+```{r circos, echo = FALSE }
+plot.biocircos
+```
+
+This report was generated using `knitr` and `rmarkdown`.
+For any questions please reach out to zzPDL_CMO_TEMPO_Support@mskcc.org
diff --git a/containers/biocircos/generate_biocircos.R b/containers/biocircos/generate_biocircos.R
new file mode 100644
index 00000000..004a0b88
--- /dev/null
+++ b/containers/biocircos/generate_biocircos.R
@@ -0,0 +1,149 @@
+#!/usr/bin/env Rscript
+
+# __author__ = "Sam Tischfield"
+# __email__ = "tischfis@mskcc.org"
+# __contributor__ = "Anne Marie Noronha (noronhaa@mskcc.org)"
+# __version__ = "0.0.1"
+# __status__ = "Dev"
+
+suppressPackageStartupMessages({
+ library(BioCircos)
+ library(data.table)
+ library(tidyverse)
+ library(getopt)
+})
+
+how_to <- function(){
+ message(" ")
+ message("This script runs the signature calling function for SVs from signature.tools.lib")
+ message("Run this script as follows:")
+ message(" ")
+ message("generate_biocircos.R -i BEDPE -g GENOME -s NAME -o OUTDIR -n NPARALLEL")
+ message(" ")
+ message(" -b BEDPE Input bedpe with candidate SV calls")
+ message(" -c CNCF Input CNCF table from Facets")
+ message(" -g GENOME hg19/hg38 (default hg19)")
+ message(" -s NAME Sample name")
+ message(" -h Show this explanation")
+}
+
+spec = matrix(c(
+ 'bedpe.path', 'b', 1, 'character',
+ 'cncf.path', 'c', 1, 'character',
+ 'genome', 'g', 2, 'character',
+ 'sampleName', 's', 1, 'character',
+ 'help', 'h', 0, 'logical'
+ ), byrow=TRUE, ncol=4)
+opt = getopt(spec)
+if ( !is.null(opt$help) ) {
+ how_to()
+ q(status=1,save = "no")
+}
+
+if ( !is.null(opt$genome) ) {
+ genome_v <- 'hg19'
+} else if (opt$genome %in% c('hg19','hg38')){
+ genome_v <- opt$genome
+} else {
+ how_to()
+ q(status=1,save = "no")
+}
+
+length.connection = pipe(paste("cat ", opt$bedpe.path, " | grep \"^##\" | wc -l", sep = ""))
+header.length = as.numeric(trimws(readLines(con = length.connection, n = 1)))
+close(length.connection)
+bedpe=fread(opt$bedpe.path,
+ skip=header.length)
+cnv=fread(opt$cncf.path)
+if (!"tcn.em" %in% names(cnv)){
+ # assume ascat format
+ names(cnv) <- unlist(str_split("index,chrom,loc.start,loc.end,tcn.em,lcn.em,normal_tcn,normal_lcn", pattern=","))
+}
+cnv %>%
+ select(c(chrom,loc.start,loc.end,tcn.em,lcn.em)) %>%
+ dplyr::rename(tcn = 'tcn.em') %>%
+ dplyr::rename(lcn = 'lcn.em') -> cnv
+
+
+
+gen.plot.biocircos <- function(bedpe,cnv,sample_name,genome_v){
+ # initiate variable to store tracks
+ tracklist <- list()
+
+ # add bedpe data
+ bedpe %>%
+ dplyr::rename(CHROM_A=`#CHROM_A`) -> bedpe
+ gen.BioCircosLinkTrack <- function(trackname,bedpe,color){
+ b <- BioCircosLinkTrack(trackname,
+ #labels=paste(trackname, bedpe$START_A, bedpe$END_A, sep=":"),
+ labels=bedpe$ID,
+ gene1Starts = bedpe$START_A,
+ gene1Ends = bedpe$END_A,
+ gene1Chromosomes = bedpe$CHROM_A,
+ gene2Chromosomes = bedpe$CHROM_B,
+ gene2Starts=bedpe$START_B,
+ gene2Ends = bedpe$END_B,
+ maxRadius = .5,
+ color = color,
+ displayLabel = F
+ )
+ }
+ for (i in list(c("BND","black"),
+ c("DEL","red"),
+ c("DUP","green"),
+ c("INV","blue"),
+ c("INS","yellow")
+ )
+ ){
+ bedpe.slice <- bedpe %>% filter(TYPE==i[1])
+ if (dim(bedpe.slice)[[1]] > 0){
+ tracklist[[i[1]]] = gen.BioCircosLinkTrack(i[1], bedpe.slice, i[2])
+ }
+ }
+
+ # add cnv data
+ # convert 23 to X and 24 to Y
+ print(head(cnv))
+ cnv %>%
+ mutate(chrom=gsub(pattern = 23,replacement = "X",x = chrom)) %>%
+ mutate(chrom=gsub(pattern = 24,replacement = "Y",x = chrom)) %>%
+ mutate(tcn.em=ifelse(tcn>=10,10,tcn)) ->cnv_test
+ cnv.range=c(0,max(cnv_test$tcn.em,na.rm = T))
+ cnv.range
+ tracklist$tcn.em =
+ BioCircosCNVTrack('cnv_track',
+ chromosomes = cnv_test$chrom,
+ starts = cnv_test$loc.start,
+ ends = cnv_test$loc.end,
+ values = cnv_test$tcn,
+ color = "black",
+ range = cnv.range)
+ tracklist$lcn.em =
+ BioCircosCNVTrack('cnv_track',
+ chromosomes = cnv_test$chrom,
+ starts = cnv_test$loc.start,
+ ends = cnv_test$loc.end,
+ values = cnv_test$lcn,
+ color = "red",
+ range = cnv.range)
+
+ tracklist$background = BioCircosBackgroundTrack("arcs_background", colors = "#2222EE")
+
+ # Add together all elements of the tracklist
+ tracklist <- Reduce('+', tracklist)
+ # Plot tracklist
+ plot=BioCircos(tracklist,
+ genomeFillColor = "PuOr",
+ genome = genome_v,
+ chrPad = 0.02,
+ displayGenomeBorder = T,
+ yChr = FALSE,
+ genomeTicksDisplay = FALSE,
+ genomeLabelTextSize = "8pt",
+ genomeLabelDy = 0)
+ return(plot)
+}
+
+plot.biocircos <- gen.plot.biocircos(bedpe=bedpe,cnv=cnv,sample_name = opt$sampleName, genome_v=genome_v )
+rmarkdown::render("biocircos.Rmd", "all", paste0(opt$sampleName,".circos.html"), output_dir='.',intermediates_dir='./tmp')
+unlink('./tmp',recursive=TRUE)
diff --git a/containers/brass/Dockerfile b/containers/brass/Dockerfile
new file mode 100644
index 00000000..b871e4db
--- /dev/null
+++ b/containers/brass/Dockerfile
@@ -0,0 +1,37 @@
+FROM quay.io/wtsicgp/brass:v6.3.4
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.image="0.0.2"
+
+USER root
+
+RUN mkdir -p /tmp \
+ && cd /tmp \
+ && apt-get update \
+ && apt-get install -y \
+ procps \
+ libz-dev \
+ libbz2-dev liblzma-dev \
+ make \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV HTSLIB_VERSION 1.9
+ENV BCFTOOLS_VERSION 1.9
+
+RUN cd /tmp && curl -L -o htslib-${HTSLIB_VERSION}.tar.bz2 https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2 \
+ && tar xvjf htslib-${HTSLIB_VERSION}.tar.bz2 \
+ && cd /tmp/htslib-${HTSLIB_VERSION} \
+ && ./configure \
+ && make && make install \
+ && rm -rf /tmp/*
+
+RUN cd /tmp && curl -L -o tmp2.tar.gz https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
+ mkdir bcftools && \
+ tar -C bcftools --strip-components 1 -jxf tmp2.tar.gz && \
+ cd bcftools && \
+ make && \
+ make install && \
+ cd .. && \
+ rm -rf /tmp/*
+
diff --git a/containers/clustersv/Dockerfile b/containers/clustersv/Dockerfile
new file mode 100644
index 00000000..233742a3
--- /dev/null
+++ b/containers/clustersv/Dockerfile
@@ -0,0 +1,21 @@
+FROM continuumio/miniconda3:4.10.3
+
+LABEL author="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ maintainer="Yixiao Gong (gongy@mskcc.org)" \
+ version.base="4.8.2" \
+ version.image="0.0.1"
+
+RUN apt-get update -y && apt-get install -y procps
+
+ENV envName "clusterSV"
+
+COPY environment.yml /
+RUN conda update -n base -c defaults conda
+RUN conda env create --name ${envName} -f /environment.yml && conda clean -a
+RUN apt-get install -y git
+RUN cd /opt \
+ && git clone https://github.com/cancerit/ClusterSV.git \
+ && cd ClusterSV \
+ && git reset --hard 1d7eeea8f22133d811bed6a4cbbaf1ad2122e2c4
+ENV PATH /opt/conda/envs/${envName}/bin:$PATH
+
diff --git a/containers/clustersv/environment.yml b/containers/clustersv/environment.yml
new file mode 100644
index 00000000..dccb49c7
--- /dev/null
+++ b/containers/clustersv/environment.yml
@@ -0,0 +1,8 @@
+channels:
+ - r
+ - defaults
+ - conda-forge
+dependencies:
+ - r-gtools=3.8.1
+ - r-doparallel=1.0.14
+ - r-r.utils
diff --git a/containers/facets-suite-preview-htstools/generate_samplestatistics.R b/containers/facets-suite-preview-htstools/generate_samplestatistics.R
new file mode 100644
index 00000000..e6ea577a
--- /dev/null
+++ b/containers/facets-suite-preview-htstools/generate_samplestatistics.R
@@ -0,0 +1,59 @@
+#!/usr/bin/env Rscript
+
+# Prepare sample statistics for BRASS SV caller
+# Outputs .samplestatistics.txt, .facets.copynumber.csv, .facets.filtered.copynumber.csv
+# Usage: Rscript generate_samplestatistics.R
+args <- commandArgs(trailingOnly=TRUE)
+rdatafile <- args[1]
+tag <- args[2]
+
+ssfile <- paste0(tag,".samplestatistics.txt")
+cnvfile <- paste0(tag,".facets.copynumber.csv")
+cnvfile.filter <- paste0(tag,".facets.filtered.copynumber.csv")
+
+load(rdatafile)
+slice <- out$out[out$out$chrom==23,]
+prop.het <- sum(slice$nhet)/sum(slice$num.mark)
+prop.het <- ifelse(is.na(prop.het),1,prop.het)
+if (prop.het < .01 ){GenderChrFound <- 'Y'} else {GenderChrFound <- 'N' }
+cat(paste('Ploidy',fit$ploidy,"\n"), file=ssfile)
+cat(paste('rho',ifelse(is.na(fit$purity),".3",fit$purity),"\n"), file=ssfile,append = T) # rho = purity
+cat(paste('GenderChr','Y',"\n"), file=ssfile,append = T)
+cat(paste('GenderChrFound',GenderChrFound,"\n"), file=ssfile,append = T)
+
+cncf <- fit$cncf
+
+cncf$n.tcn <- ifelse(
+ cncf$chrom == 24,
+ ifelse(GenderChrFound == 'Y', 1, 0),
+ ifelse(cncf$chrom < 23, 2, ifelse(GenderChrFound == 'Y', 1, 2))
+)
+cncf$n.lcn <- ifelse(
+ cncf$chrom == 24,
+ 0,
+ ifelse(cncf$chrom < 23, 1, ifelse(GenderChrFound == 'Y', 0, 1))
+)
+
+cncf.reorder <- cncf[,c(unlist(strsplit("seg,chrom,start,end,n.tcn,n.lcn,tcn,lcn",",")))]
+
+filter.na.vector <- function(x){
+ any(is.na(x))
+}
+cncf.reorder.filter <- cncf.reorder[!apply(cncf.reorder,1,filter.na.vector),]
+
+write.table(
+ cncf.reorder,
+ file=cnvfile,
+ sep=",",
+ row.names=F,
+ col.names=F,
+ quote=F
+)
+write.table(
+ cncf.reorder.filter,
+ file=cnvfile.filter,
+ sep=",",
+ row.names=F,
+ col.names=F,
+ quote=F
+)
diff --git a/containers/facets-suite-preview/Dockerfile b/containers/facets-suite-preview/Dockerfile
new file mode 100644
index 00000000..ab2bec36
--- /dev/null
+++ b/containers/facets-suite-preview/Dockerfile
@@ -0,0 +1,73 @@
+FROM rocker/tidyverse:3.6.1
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ contributor="Yixiao Gong (gongy@mskcc.org)" \
+ contributor="Nikhil Kumar (kumarn1@mskcc.org)" \
+ contributor="Philip Jonsson (jonssonp@mskcc.org)" \
+ version.image="0.0.1-rebuild" \
+ version.facets_suite="2.0.8" \
+ version.facets="0.5.14" \
+ version.alpine="3.8" \
+ version.pctGCdata="0.2.0" \
+ source.facets="https://github.com/mskcc/facets/archive/v0.5.14.tar.gz"
+
+ENV FACETS_SUITE_VERSION 2.0.8
+ENV FACETS_VERSION 0.5.14
+ENV FACETS_PREVIEW_VERSION 2.1.4
+ENV PCTGCDATA 0.2.0
+
+# Requirements
+RUN apt-get update \
+ && apt-get install -y \
+ g++ \
+ tar \
+ bzip2 \
+ libbz2-dev \
+ libc6-dev \
+ libxt-dev \
+ liblzma-dev \
+ libgtk2.0-dev \
+ libcairo2-dev \
+ xvfb \
+ xauth \
+ xfonts-base
+RUN apt-get install -y xdg-utils --fix-missing
+RUN apt-get install -y inotify-tools=3.14-2
+
+RUN R -e "install.packages(c('Cairo','argparse','gridExtra', 'binom', 'BiocManager', 'diptest', 'egg','shinyWidgets', 'shinyjs', 'rhandsontable', 'doParallel', 'configr', 'R.utils'), repos='http://cran.us.r-project.org')" \
+ && R -e "BiocManager::install('rtracklayer')"
+
+# Install FACETS, pctGCdata and facets-suite
+RUN cd /tmp \
+ && wget https://github.com/mskcc/facets-suite/archive/${FACETS_SUITE_VERSION}.tar.gz -O facets-suite-${FACETS_SUITE_VERSION}.tar.gz \
+ && wget https://github.com/mskcc/facets/archive/v${FACETS_VERSION}.tar.gz -O facets-v${FACETS_VERSION}.tar.gz \
+ && wget https://github.com/mskcc/pctGCdata/archive/v${PCTGCDATA}.tar.gz \
+ && git clone --single-branch --branch master https://github.com/taylor-lab/facets-preview.git facets-preview-master \
+ && tar xvzf facets-v${FACETS_VERSION}.tar.gz \
+ && tar xvzf v${PCTGCDATA}.tar.gz \
+ && tar xvzf facets-suite-${FACETS_SUITE_VERSION}.tar.gz \
+ && cd /tmp/pctGCdata-${PCTGCDATA} \
+ && R CMD INSTALL . \
+ && cd /tmp/facets-${FACETS_VERSION} \
+ && R CMD INSTALL . \
+ && cd /tmp/facets-suite-${FACETS_SUITE_VERSION} \
+ && R CMD INSTALL . \
+ # correct shebang line
+ && sed -i "s/opt\/common\/CentOS_6-dev\/R\/R-3.2.2\//usr\//g" *.R \
+ # copy execs to /usr/bin/facets-suite
+ && mkdir -p /usr/bin/facets-suite/ \
+ && cp -r /tmp/facets-suite-${FACETS_SUITE_VERSION}/* /usr/bin/facets-suite/ \
+ && cd /tmp/facets-preview-master \
+ && R CMD INSTALL . \
+ && mkdir -p /usr/bin/facets-preview/ \
+ && cp -r /tmp/facets-preview-master/* /usr/bin/facets-preview/ \
+ # clean up
+ && rm -rf /var/cache/apk/* /tmp/* # update to using release version when 2.1.5+ is released.
+
+ENV PYTHONNOUSERSITE set
+ENV FACETS_OVERRIDE_EXITCODE set
+
+COPY impact_juno_config.json /usr/bin/facets-preview/
+RUN chmod 744 /usr/bin/facets-preview/impact_juno_config.json
+EXPOSE 3838
+
diff --git a/containers/facets-suite-preview/impact_juno_config.json b/containers/facets-suite-preview/impact_juno_config.json
new file mode 100644
index 00000000..0ad7455b
--- /dev/null
+++ b/containers/facets-suite-preview/impact_juno_config.json
@@ -0,0 +1,23 @@
+{
+ "repo": [
+ {
+ "name": "Clinical series IMPACT",
+ "manifest_file": "/juno/work/ccs/shared/resources/impact/facets/manifests/impact_facets_manifest_latest.txt.gz",
+ "tumor_id_format": "^P-\\d{7}-T\\d+-IM\\d(,P-\\d{7}-T\\d+-IM\\d)*$",
+ "counts_file_format": "countsMerged____{sample_id}.dat.gz"
+ }
+ ],
+ "watcher_dir": "/juno/work/ccs/shared/software/refit_watcher/",
+ "facets_lib": [
+ {
+ "version": "0.5.14",
+ "lib_path": "/usr/local/lib/R/site-library/facets/"
+ }
+ ],
+ "verify_sshfs_mount" : "",
+ "r_script_path" : "/usr/local/bin/Rscript",
+ "facets_suite_lib": "/usr/local/lib/R/site-library/",
+ "facets_suite_run_wrapper": "/usr/bin/facets-suite/run-facets-wrapper.R",
+ "facets_qc_script": "/usr/bin/facets-preview/facets_qc/v1.0/facets_fit_qc.R"
+}
+
diff --git a/containers/iannotatesv/Dockerfile b/containers/iannotatesv/Dockerfile
new file mode 100644
index 00000000..7d25925d
--- /dev/null
+++ b/containers/iannotatesv/Dockerfile
@@ -0,0 +1,26 @@
+FROM python:2.7.15
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.image="0.0.2"
+
+RUN python -m pip install \
+ pandas==0.24.2 \
+ biopython==1.76 \
+ Pillow==6.2.1 \
+ openpyxl==2.6.4 \
+ reportlab==3.5.2 \
+ coloredlogs==14.0
+
+RUN apt-get install -y git
+
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && apt-get install git-lfs
+
+RUN mkdir -p /usr/bin/ && cd /usr/bin/ && git clone https://github.com/rhshah/iAnnotateSV.git \
+ && cd iAnnotateSV \
+ && git reset --hard a2f86543925169219c91fe4e3de5412a69f735a4
+
+RUN mkdir -p /tmp \
+ && apt-get update \
+ && apt-get install -y \
+ bedtools
+RUN python -m pip install pybedtools==0.8.2
diff --git a/containers/iannotatesv/detect_cdna.py b/containers/iannotatesv/detect_cdna.py
new file mode 100755
index 00000000..73af4507
--- /dev/null
+++ b/containers/iannotatesv/detect_cdna.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+"""Identify deletion rearrangements that may pertain to cDNA contaminants"""
+
+__author__ = "Anne Marie Noronha"
+__email__ = "noronhaa@mskcc.org"
+__version__ = "0.0.1"
+__status__ = "Dev"
+
+import argparse, sys, os
+import numpy as np
+import pandas as pd
+import pybedtools
+from utils import *
+
+def usage():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--exon-junct', dest="regions", help = 'regions with exon junctions', required = True)
+ parser.add_argument('--bedpe', help = 'bedpe file', required = True)
+ parser.add_argument('--out',dest="outfile",help = 'output file' , required = True)
+ parser.add_argument('--out-bedpe',dest="outbedpe",help = 'output bedpe' , required = True)
+ parser.add_argument('--intermediate', action="store_true", help = 'print intermediate file for advanced analysis')
+ return parser.parse_args()
+
+def get_artefact_calls(df):
+ """
+ Find genes with at least 2 exon-exon junctions
+ remove any variant that doesn't involve two exons
+ remove any variant that doesn't have "donor" and "acceptor" in the correct orientation
+ Only genes with at least two splice variants will be considered.
+ """
+ try:
+ df = df.groupby(["ID","gene"]).filter(lambda x: len(set(x['exon_number'])) == 2 and len(set(x['splice_orientation'])) == 2)
+ df = df.groupby(["ID","gene"]).filter(lambda x: sorted(zip(x["exon_number"],x["splice_orientation"]), key=lambda t: t[0] )[0][1] == "donor")
+ df = df.groupby(["gene"]).filter(lambda x: len(set(x['ID'])) > 1)
+ df = df[["ID","gene"]].drop_duplicates().rename(columns={"gene":"POTENTIAL_CDNA_CONTAMINATION"})
+ return df.groupby(['ID'], as_index = False).agg({'POTENTIAL_CDNA_CONTAMINATION': ';'.join})
+ except:
+ return pd.DataFrame(columns=["ID","POTENTIAL_CDNA_CONTAMINATION"])
+ #df = df.drop_duplicates().groupby(["var_id","gene"], as_index = False).agg({'exon': lambda x: len(set(x)) })
+ #artefact_genes = df[df.exon > 3].gene.tolist()
+ #return df[df.gene.isin(artefact_genes)]["var_id"].tolist()
+
+def prep_intersection(intersect=pd.DataFrame(),bedpe_header_list=[]):
+ attr_list = sorted("exon_number|splice_orientation|gene|ccds_id".split("|"))
+ try:
+ attr_list = sorted("exon_number|splice_orientation|gene|ccds_id".split("|"))
+ intersect["attr"] = intersect.apply(lambda x: {j.split(":")[0]:j.split(":")[1] for j in x["attr"].split("|")}, axis=1)
+ intersect["attr"] = intersect.apply(lambda x: [ x["attr"].get(i,None) for i in attr_list ],axis=1)
+ intersect = intersect[~intersect.apply(lambda x: None in x["attr"], axis=1)]
+ intersect[attr_list] = intersect.attr.apply(lambda x: pd.Series(x))
+ intersect = intersect[intersect["TYPE"]=="DEL"]
+ intersect = intersect[["ID"] + attr_list].drop_duplicates(keep="first")
+ return intersect
+ except:
+ return pd.DataFrame(columns=["ID"] +attr_list)
+
+
+
+def main():
+ args = usage()
+
+ print("Detecting possible cdna contamination in {}".format(os.path.basename(args.bedpe)))
+
+ [meta_header, bedpe_header_list, bedpe_df] = parse_svtools_bedpe_file(args.bedpe)
+
+ bedpe_bt = pybedtools.BedTool.from_dataframe(bedpe_df)
+ regions_bt = pybedtools.BedTool(args.regions)
+ intersect = run_pair_to_bed(bedpe_bt,regions_bt,match_type="both")
+ intersect_df = bedtool_to_df(intersect,bedpe_header_list + "#chrom|start|end|attr|score|cds_strand".split("|"))
+ intersect_df = prep_intersection(intersect_df,bedpe_header_list)
+ if args.intermediate:
+ intersect_df.to_csv("intermediate.tsv",sep="\t",header=True, index=False)
+
+ artefact_calls = get_artefact_calls(intersect_df)
+
+ cdna_filter = bedpe_df.merge(artefact_calls, on="ID", how="left")
+ cdna_filter["POTENTIAL_CDNA_CONTAMINATION"] = cdna_filter["POTENTIAL_CDNA_CONTAMINATION"].replace(np.nan, ".")
+
+ with open(args.outbedpe, "w") as fw:
+ fw.write("".join(meta_header))
+
+ cdna_filter.to_csv(args.outbedpe, header=True, index=False, sep="\t", mode="a")
+ artefact_calls.to_csv(args.outfile, sep="\t", header=False, index=False)
+
+if __name__ == "__main__":
+ main()
diff --git a/containers/iannotatesv/filter_regions_bedpe.py b/containers/iannotatesv/filter_regions_bedpe.py
new file mode 100644
index 00000000..48ba4f82
--- /dev/null
+++ b/containers/iannotatesv/filter_regions_bedpe.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+"""Filter rearrangements that overlap with a set of regions"""
+
+__author__ = "Anne Marie Noronha"
+__email__ = "noronhaa@mskcc.org"
+__version__ = "0.0.1"
+__status__ = "Dev"
+
+import argparse, sys, os
+import numpy as np
+import pandas as pd
+import pybedtools
+from utils import *
+
+def usage():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--blacklist-regions', dest="regions", help = 'regions bed/bedpe file', required = True)
+ parser.add_argument('--bedpe', help = 'bedpe file', required = True)
+ parser.add_argument('--tag', help = 'string to update FILTER column', required = True)
+ parser.add_argument('--output',dest="outfile",help = 'output file' , required = True)
+ parser.add_argument('--match-type',dest="type",help = 'both/either/notboth/neither' , required = True)
+ parser.add_argument('--ignore-strand',dest="ignore_strand", default=False, action="store_true", help = 'Use flag to ignore strand when running bedtools. Default False' )
+ return parser.parse_args()
+
+def determine_regions_filetype(filepath):
+ """
+ Determine if the regions file is bed or bedpe, and raise error if neither
+ """
+ if not any([filepath.endswith(i) for i in "bed|bedpe|bed.gz|bedpe.gz".split("|")]):
+ raise ValueError("--bedpe requires bed, bedpe, bed.gz or bedpe.gz file.")
+ elif any([filepath.endswith(i) for i in "bed|bed.gz".split("|")]):
+ return True
+ else: return False
+
+def validate_bedpe_input(filepath):
+ """
+ Raise error if filetype is not bedpe
+ """
+ if not any([filepath.endswith(i) for i in "bedpe|bedpe.gz".split("|")]):
+ raise ValueError("--bedpe requires bedpe or bedpe.gz file.")
+
+def validate_match_type(type):
+ """
+ Raise error if type is not acceptable for use in pybedtools pair_to_pair or pair_to_bed
+ """
+ if type not in overlap_type.keys():
+ raise ValueError( "--match-type must be {}.".format(" or ".join(overlap_type.keys())) )
+
+def main():
+ """
+ 1. validate inputs
+ 2. read input beds/bedpes as pybedtools.BedTool objects
+ 3. extract variant IDs from overlapping regions
+ 4. update the FILTER tag for identified variants
+ """
+ args = usage()
+
+ print("Filtering {} with {} regions".format(os.path.basename(args.bedpe), args.tag))
+
+ # parse inputs
+ try:
+ validate_bedpe_input(args.bedpe)
+ [meta_header, bedpe_header_list, bedpe_df] = parse_svtools_bedpe_file(args.bedpe)
+ except Exception as e:
+ print(e)
+ sys.exit("Unable to parse --bedpe")
+
+ try:
+ is_bed = determine_regions_filetype(args.regions)
+ regions_bt = pybedtools.BedTool(args.regions)
+ except Exception as e:
+ print(e)
+ sys.exit("Unable to parse --blacklist-regions")
+
+ # annotate bedpe
+ try:
+ bedpe_bt = pybedtools.BedTool.from_dataframe(bedpe_df)
+ ids_df = find_overlapped_ids(bedpe_bt, regions_bt, args.type, args.ignore_strand, is_bed)
+ bedpe_df = add_filter_by_id(bedpe_df,ids_df,args.tag)
+ except Exception as e:
+ print(e)
+ sys.exit("Filtering failed with inputs {} and {}".format(args.bedpe, args.regions))
+
+ # write result
+ with open(args.outfile, "w") as fw:
+ fw.write("".join(meta_header))
+ bedpe_df.to_csv(args.outfile, header=True, index=False, sep="\t", mode="a")
+
+def find_overlapped_ids(bedpe_bt,regions_bt,match_type,ignore_strand,is_bed):
+ # determine validity of match_type
+ validate_match_type(match_type)
+
+ # run pair_to_bed if regions file is bed, pair_to_pair if regions file is bedpe
+ if is_bed:
+ intersect = run_pair_to_bed(bedpe_bt,regions_bt,match_type)
+ else:
+ intersect = run_pair_to_pair(bedpe_bt,regions_bt,match_type, ignore_strand=ignore_strand)
+
+ # extract ids
+ try:
+ ids_df = pd.DataFrame({"ID":list( intersect.to_dataframe(header=None)[6].drop_duplicates() )})
+ except:
+ ids_df = pd.DataFrame(columns=["ID"])
+ return ids_df
+
+def add_filter_by_id(bedpe_df,ids_df,tag):
+ # annotate bedpe based on matching IDs
+ ids_df.columns = ["ID"]
+ ids_df["FILTER_NEW"] = tag
+ filtered_bedpe_df = bedpe_df.merge(ids_df, on="ID", how="left")
+ filtered_bedpe_df = filtered_bedpe_df.apply(lambda x: update_filter(x,x["FILTER_NEW"]) if not pd.isna(x["FILTER_NEW"]) else x, axis=1)
+ filtered_bedpe_df = filtered_bedpe_df.drop(['FILTER_NEW'], axis=1)
+
+ return filtered_bedpe_df
+
+if __name__ == "__main__":
+ main()
diff --git a/containers/iannotatesv/run_iannotatesv.py b/containers/iannotatesv/run_iannotatesv.py
new file mode 100644
index 00000000..add115a8
--- /dev/null
+++ b/containers/iannotatesv/run_iannotatesv.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+"""Annotate SV Bedpe file with iAnnotateSV"""
+__author__ = "Anne Marie Noronha"
+__email__ = "noronhaa@mskcc.org"
+__version__ = "0.0.1"
+__status__ = "Dev"
+
+import argparse, sys, os, subprocess
+from collections import OrderedDict
+from datetime import datetime
+import multiprocessing as mp
+import numpy as np
+import pandas as pd
+from utils import *
+
+def usage():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--bedpe', help = 'bedpe file', required = True)
+ parser.add_argument('--genome', default = "hg19", help = 'hg19/hg38/hg18')
+ parser.add_argument('--threads', type=int, default = 4, help = 'number of threads for parallelization')
+
+ return parser.parse_args()
+
+def run_iannotate_cmd(input_file,output_dir,output_pre,genome="hg19"):
+ print("Spawning in parallel: annotation of " + input_file)
+ run_args = "python /usr/bin/iAnnotateSV/iAnnotateSV/iAnnotateSV.py -i {} -o {} -ofp {} -r {} -d 3000""".format(input_file, output_dir, output_pre, genome).split(" ")
+ p = subprocess.Popen(run_args)
+ p.communicate()
+
+def read_iannotatesv_result(path):
+ return pd.read_csv(path, sep="\t",header=0)
+
+def main():
+ args = usage()
+
+ dt = datetime.now()
+ print("[{}] Running iAnnotateSV on {}".format(dt,os.path.basename(args.bedpe)))
+
+ meta, header_list, data = parse_svtools_bedpe_file(args.bedpe)
+ iAnnotate_input = data["#CHROM_A|START_A|STRAND_A|CHROM_B|START_B|STRAND_B|ID".split("|")]
+ iAnnotate_input = iAnnotate_input.replace("-", 1).replace("+", 0)
+ key_col = OrderedDict(zip("chr1|pos1|str1|chr2|pos2|str2".split("|"), [str,int,int,str,int,int]))
+ iAnnotate_input.columns = key_col.keys() + ["ID"]
+ for k,v in key_col.items():
+ iAnnotate_input[k] = iAnnotate_input[k].astype(v)
+
+ k = int(500)
+ n_chunks = int(((k-1)+iAnnotate_input.shape[0])/k)
+ iAnnotate_output = pd.DataFrame()
+ if not os.path.isdir("inputs"): os.mkdir("inputs")
+ if not os.path.isdir("outputs"): os.mkdir("outputs")
+
+ pool = mp.Pool(args.threads)
+ for i in range(n_chunks):
+ iAnnotate_input_chunk = iAnnotate_input.iloc[i*k:(i+1)*k]
+ input_file = os.path.join("inputs","input_" + str(i) + ".txt")
+ iAnnotate_input_chunk.to_csv(input_file,sep="\t",header=True,index=False)
+ pool.apply_async(run_iannotate_cmd, args=(input_file, "outputs", "output_" + str(i), args.genome ))
+
+ pool.close()
+ pool.join()
+
+ try:
+ iAnnotate_output = pd.concat(map(read_iannotatesv_result, [os.path.join("outputs","output_" + str(i) + "_Annotated.txt") for i in range(n_chunks)]))
+ except ValueError as ve:
+ iAnnotate_output = pd.DataFrame(columns=key_col.keys())
+ for key,val in key_col.items():
+ iAnnotate_output[key] = iAnnotate_output[key].astype(val)
+
+ iAnnotate_output = pd.merge(iAnnotate_input,iAnnotate_output, on=key_col.keys(), how="inner")
+
+ print("[{}] Merging annotation with bedpe".format(datetime.now()))
+ annot_data = iAnnotate_output.drop(key_col.keys(), axis=1).replace(np.nan, ".")
+ final_data = pd.merge(data, annot_data, on="ID",how="left")
+
+ outputbed = os.path.basename(args.bedpe)[:-6] if args.bedpe.endswith(".bedpe") else os.path.basename(args.bedpe)
+ outputbed = os.path.join(os.path.dirname(args.bedpe),outputbed + ".iannotate.bedpe")
+ with open(outputbed, "w") as fw:
+ fw.write(meta)
+ final_data.to_csv(outputbed,sep="\t",header=True,index=False, mode='a')
+
+ print("[{}] iAnnotateSV complete".format(datetime.now()))
+
+if __name__ == "__main__":
+ main()
diff --git a/containers/iannotatesv/utils.py b/containers/iannotatesv/utils.py
new file mode 100644
index 00000000..db39d38b
--- /dev/null
+++ b/containers/iannotatesv/utils.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+
+__author__ = "Anne Marie Noronha"
+__email__ = "noronhaa@mskcc.org"
+__version__ = "0.0.1"
+__status__ = "Dev"
+
+import argparse
+import numpy as np
+import pandas as pd
+import pybedtools
+
+
+overlap_type = {"both":"notboth",
+ "notboth":"both",
+ "either":"neither",
+ "neither":"either"
+ }
+
+def add_tag(val,tag ):
+ """
+ update FILTER value
+ """
+ newTag = tag
+ if val not in [".",None,"PASS",np.nan]:
+ newTag = "{};{}".format(val,tag)
+ return newTag
+
+def update_filter(row, tag):
+ """
+ return row with updated FILTER value
+ """
+ row["FILTER"] = add_tag(row["FILTER"], tag)
+ return row
+
+def run_pair_to_bed(bedpe,bed,match_type):
+ """
+ use pybedtools to run bedtools pairtobed
+ a length of 1 base must be artificially added to each end with a length of 0, otherwise no overlap
+ """
+ result = bedpe.pair_to_bed(bed, **{'type': match_type})
+ return result
+
+def run_pair_to_pair(bedpe,filter_bedpe,match_type,ignore_strand=False):
+ """
+ use pybedtools to run bedtools pairtopair
+ a length of 1 base must be artificially added to each end with a length of 0, otherwise no overlap
+ """
+ result = bedpe.pair_to_pair(filter_bedpe, **{'type': match_type,'is':ignore_strand})
+ return result
+
+
+def bedtool_to_df(bt,header_list):
+ try:
+ df = bt.to_dataframe(header=None)
+ df = df[df.columns[:len(header_list)]]
+ df.columns = header_list
+ except Exception as e:
+ print(e)
+ print("unable to convert bedtool to pandas dataframe, continuing with empty dataframe")
+ df = pd.DataFrame(columns = header_list)
+ try:
+ df = df.astype({i:int for i in "START_A|END_A|START_B|END_B".split("|")})
+ for j in ["#CHROM_A","CHROM_B"]:
+ if df[j].dtype == 'float64':
+ df = df.astype({j:int}).astype({j:str})
+ except Exception as e:
+ print(e)
+ print("Unable to apply formatting to bedpe columns using pandas, when converting pybedtool.Bedtool to pandas.DataFrame. Skipping...")
+ return df.drop_duplicates()
+
+def parse_svtools_bedpe_file(bedpe, offset = True):
+ """
+ Read bedpe file
+ 1. Separate components meta-data, header line, and records (main data)
+ 2. Coerce chromosome values to string, and position coordinates to integer
+ 3. if offset set to True, add +1 to END_* coordinates. This may be necessary for certain pybedtools operations.
+ """
+ meta_header=""
+ with open(bedpe, 'r') as f:
+ main_data = False
+ while main_data == False:
+ x = f.readline()
+ if x.startswith("##"):
+ meta_header += x
+ else:
+ header = x
+ header_list = header.strip().split("\t")
+ main_data = True
+ try:
+ records_df = pd.read_csv(f, header=None, sep="\t" )
+ records_df.columns = header_list
+ except:
+ records_df = pd.DataFrame(columns = header_list)
+
+ try:
+ records_df = records_df.astype({i:int for i in "START_A|END_A|START_B|END_B".split("|")})
+ for j in ["#CHROM_A","CHROM_B"]:
+ if records_df[j].dtype == 'float64':
+ records_df = records_df.astype({j:int}).astype({j:str})
+ except Exception as e:
+ print(e)
+ print("Unable to apply formatting to bedpe columns in pandas. Skipping...")
+
+ try:
+ if offset:
+ if records_df.shape[0] > 0:
+ records_df["END_A"] = records_df.apply(lambda row: row["END_A"] + 1 if row["START_A"] == row["END_A"] else row["END_A"],axis=1)
+ records_df["END_B"] = records_df.apply(lambda row: row["END_B"] + 1 if row["START_B"] == row["END_B"] else row["END_B"],axis=1)
+ except Exception as e:
+ print(e)
+ print("Unable to add +1 offset to END_A and END_B columns in pandas. There may be issues with pybedtools. Skipping...")
+ return [meta_header, header_list, records_df]
diff --git a/containers/neoantigen/Dockerfile b/containers/neoantigen/Dockerfile
index cc50b5bb..b4f2743f 100644
--- a/containers/neoantigen/Dockerfile
+++ b/containers/neoantigen/Dockerfile
@@ -1,14 +1,14 @@
FROM ubuntu:18.04
LABEL maintainer="Allan Bolipata , Evan Biederstedt , Yixiao Gong " \
- version.image="0.3.2-hotfix" \
- version.neoantigen-dev="0.3.2-hotfix" \
+ version.image="0.3.3" \
+ version.neoantigen-dev="0.3.3" \
version.netMHC="4.0a" \
version.netMHCpan="4.0a" \
version.python="2.7.15rc1"
ENV TMPDIR="/tmp"
-ENV NEOANTIGEN_VERSION 0.3.2-hotfix
+ENV NEOANTIGEN_VERSION="0.3.3"
RUN apt-get update && apt-get install -y \
tcsh \
@@ -20,6 +20,8 @@ RUN apt-get update && apt-get install -y \
&& apt-get clean && apt-get purge \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install --upgrade pip setuptools
+
# For copying netMHC and netMHCpan gz files for installation. This Dockerfile expects them to be in the same directory
# NOTE: You will have to acquire them through their website, as they are not readily available for download from:
#https://services.healthtech.dtu.dk/software.php
@@ -31,7 +33,7 @@ RUN tar -xvf /tmp/netMHC/netMHC-4.0a.Linux.tar.gz -C /usr/local/bin \
# Install netMHC data
RUN cd /usr/local/bin/netMHC-4.0/ \
- && wget http://www.cbs.dtu.dk/services/NetMHC-4.0/data.tar.gz \
+ && wget https://services.healthtech.dtu.dk/services/NetMHC-4.0/data.tar.gz \
&& gunzip -c data.tar.gz | tar xvf - \
&& rm data.tar.gz
@@ -42,7 +44,7 @@ RUN tar -xvf /tmp/netMHC/netMHCpan-4.0a.Linux.tar.gz -C /usr/local/bin \
# Install netMHCpan data; directory needed permission changes, drwxr-xr-x
RUN cd /usr/local/bin/netMHCpan-4.0/ \
- && wget http://www.cbs.dtu.dk/services/NetMHCpan-4.0/data.Linux.tar.gz \
+ && wget https://services.healthtech.dtu.dk/services/NetMHCpan-4.0/data.Linux.tar.gz \
&& tar -zxvf data.Linux.tar.gz \
&& chmod -R o+rx data/ \
&& rm -rf data.Linux.tar.gz
diff --git a/containers/signaturetoolslib/Dockerfile b/containers/signaturetoolslib/Dockerfile
new file mode 100644
index 00000000..930e38d7
--- /dev/null
+++ b/containers/signaturetoolslib/Dockerfile
@@ -0,0 +1,25 @@
+FROM rocker/tidyverse:4.1.3
+
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.base="4.1.3" \
+ version.image="0.0.1"
+
+RUN apt-get update && apt-get install -y \
+ zlib1g-dev \
+ libbz2-dev \
+ liblzma-dev \
+ libc6-dev \
+ build-essential \
+ git \
+ ghostscript \
+ libgmp3-dev
+
+RUN R -e "install.packages(c('roxygen2','getopt'))"
+RUN R -e "install.packages('devtools')"
+RUN R -e "devtools::install_github('r-lib/xml2@v1.3.3')"
+RUN R -e "install.packages(c('XML','rversions','data.table'))"
+RUN R -e "devtools::install_github('linxihui/NNLM@0.4.4') "
+RUN R -e "install.packages('BiocManager') ; BiocManager::install(c('BSgenome.Mmusculus.UCSC.mm10','BSgenome.Hsapiens.1000genomes.hs37d5','BSgenome.Hsapiens.UCSC.hg38'))"
+RUN R -e "options(download.file.method = 'wget') ;devtools::install_github('Nik-Zainal-Group/signature.tools.lib@v2.1.2')"
+
diff --git a/containers/signaturetoolslib/HRDetect_wrapper.R b/containers/signaturetoolslib/HRDetect_wrapper.R
new file mode 100644
index 00000000..b956b029
--- /dev/null
+++ b/containers/signaturetoolslib/HRDetect_wrapper.R
@@ -0,0 +1,209 @@
+#!/usr/bin/env Rscript
+
+
+# __author__ = "Vasilisa Rudneva"
+# __email__ = "rudnevav@mskcc.org"
+# __contributor__ = "Yixiao Gong (gongy@mskcc.org); Anne Marie Noronha (noronhaa@mskcc.org)"
+# __version__ = "1.0"
+
+
+# ## Installation:
+# install.packages("devtools")
+# BiocManager::install("BSgenome")
+# BiocManager::install("RcppProgress")
+# BiocManager::install("BSgenome.Mmusculus.UCSC.mm10")
+# BiocManager::install("BSgenome.Hsapiens.1000genomes.hs37d5")
+# BiocManager::install("BSgenome.Hsapiens.UCSC.hg38")
+# install.packages("https://cran.r-project.org/src/contrib/Archive/NNLM/NNLM_0.4.3.tar.gz", repo=NULL)
+# #devtools::install_github('linxihui/NNLM')
+# devtools::install_github('Nik-Zainal-Group/signature.tools.lib')
+
+suppressPackageStartupMessages({
+library(signature.tools.lib)
+library(data.table)
+library(tools)
+library(stringr)
+library(BSgenome.Hsapiens.1000genomes.hs37d5)
+library(BSgenome.Hsapiens.UCSC.hg38)
+library(plyr)
+library(dplyr)
+})
+
+args = commandArgs(trailingOnly=TRUE)
+
+
+inputTSV = args[1]
+genome_version = args[2]
+if (genome_version == 'hg19'){
+ ref.genome = BSgenome.Hsapiens.1000genomes.hs37d5
+} else if (genome_version == 'hg38'){
+ ref.genome = BSgenome.Hsapiens.NCBI.GRCh38
+} else {
+ stop("Input genome must be hg19/hg38")
+}
+
+n_parallel = as.integer(args[3])
+if(is.na(n_parallel)){n_parallel=1}
+
+message(paste("========= Genome Version ", genome_version, "=========", sep=" "))
+input.files<-fread(inputTSV, header = T, data.table = T)
+
+sample_names<-input.files$sample
+
+dir.create("tmp")
+
+# 1. SV_bedpe_files
+#The files should contain a header in the first line with the following columns: "chrom1", "start1", "end1", "chrom2", "start2", "end2" and "sample" (sample name).
+#In addition, either two columns indicating the strands of the mates, "strand1" (+ or -) and "strand2" (+ or -),
+#or one column indicating the structural variant class, "svclass": translocation, inversion, deletion, tandem-duplication.
+#The column "svclass" should correspond to (Sanger BRASS convention): inversion (strands +/- or -/+ and mates on the same chromosome),
+#deletion (strands +/+ and mates on the same chromosome), tandem-duplication (strands -/- and mates on the same chromosome),
+#translocation (mates are on different chromosomes)..
+
+correctSV <- function(this_sample){
+ cat(this_sample)
+ this_sv<-fread(cmd = paste("grep -v '^##'", input.files[input.files$sample==this_sample,]$sv), header = T, data.table = F)
+ this_sv<-this_sv[this_sv$FILTER=="PASS",]
+ setnames(this_sv, "#CHROM_A", "chrom1")
+ setnames(this_sv, "START_A", "start1")
+ setnames(this_sv, "END_A", "end1")
+ setnames(this_sv, "CHROM_B", "chrom2")
+ setnames(this_sv, "START_B", "start2")
+ setnames(this_sv, "END_B", "end2")
+ setnames(this_sv, "STRAND_A", "strand1")
+ setnames(this_sv, "STRAND_B", "strand2")
+ svclass_dict <- c("BND"="translocation","INV"="inversion","DEL"="deletion","DUP"="tandem-duplication")
+ this_sv$svclass <- stringr::str_replace_all(string = this_sv$TYPE,
+ pattern= svclass_dict)
+ print(table(this_sv$svclass))
+ this_sv$sample=this_sample
+ this_sv <- this_sv[,unlist(strsplit("chrom1,start1,end1,chrom2,start2,end2,strand1,strand2,sample,svclass",split=","))]
+ filename<-paste0("tmp/",this_sample, ".sv")
+ write.table(file = filename, x = this_sv, quote = F, row.names = F, col.names = T, sep = "\t")
+}
+message("")
+message("========= Preprocessing SV inputs =========")
+invisible(lapply(sample_names,correctSV))
+SV_bedpe_files <- paste0("tmp/",sample_names,".sv")
+names(SV_bedpe_files) <- sample_names
+
+
+# 2 Indels_tab_files and SNV_tab_files
+#list of file names corresponding to Indels/SNV TAB files to be used to classify Indels and compute the proportion of indels at micro-homology or 96-channel substitution catalogues.
+#This should be a named vector, where the names indicate the sample name,
+#so that each file can be matched to the corresponding row in the data_matrix input.
+#The files should only contain indels (or SNV) and should already be filtered according to the user preference,
+#as all indels in the file will be used and no filter will be applied.
+#Each File contains indels from a single sample and the following minimal columns: chr, position, REF, ALT.
+correctMutations <- function(this_sample){
+ cat(this_sample)
+ this_mutations<-fread(input.files[input.files$sample==this_sample,]$mutations, header = T, data.table = F)
+ setnames(this_mutations, "Chromosome", "chr")
+ setnames(this_mutations, "vcf_pos", "position")
+ setnames(this_mutations, "vcf_id", "ID")
+ setnames(this_mutations, "Reference_Allele", "REF")
+ setnames(this_mutations, "Allele", "ALT")
+ setnames(this_mutations, "vcf_qual", "QUAL")
+ this_indels<-this_mutations[this_mutations$Variant_Type %in% c("DEL", "DNP", "INS", "TNP"),]
+ this_indels <- this_indels %>%
+ dplyr::rowwise() %>%
+ mutate(left_b = as.character(ref.genome[[chr]][position])) %>%
+ mutate(REF = ifelse(ALT == "-",paste0(left_b,REF),REF),
+ ALT = ifelse(ALT == "-",left_b,ALT)) %>%
+ mutate(ALT = ifelse(REF == "-",paste0(left_b,ALT),ALT),
+ REF = ifelse(REF == "-",left_b,REF)) %>%
+ select(-c(left_b))
+
+ this_snv<-this_mutations[!this_mutations$Variant_Type %in% c("DEL", "DNP", "INS", "TNP"),]
+ print(table(rbind(this_indels, this_snv)$Variant_Type))
+ write.table(file = paste0("tmp/", this_sample, ".indels"), x = this_indels, quote = F, row.names = F, col.names = T, sep = "\t")
+ write.table(file = paste0("tmp/", this_sample, ".snv"), x = this_snv, quote = F, row.names = F, col.names = T, sep = "\t")
+}
+message("")
+message("========= Preprocessing Indels and SNV inputs =========")
+invisible(lapply(sample_names,correctMutations))
+Indels_tab_files <- paste0("tmp/",sample_names,".indels")
+SNV_tab_files <- paste0("tmp/",sample_names,".snv")
+names(Indels_tab_files) <- sample_names
+names(SNV_tab_files) <- sample_names
+
+# 3. CNV_tab_files
+# list of file names corresponding to CNV TAB files (similar to ASCAT format)
+#to be used to compute the HRD-LOH index. This should be a named vector,
+#where the names indicate the sample name,
+#so that each file can be matched to the corresponding row in the data_matrix input.
+#The files should contain a header in the first line with the following columns:
+#'seg_no', 'Chromosome', 'chromStart', 'chromEnd',
+#'total.copy.number.inNormal', 'minor.copy.number.inNormal', 'total.copy.number.inTumour', 'minor.copy.number.inTumour'
+correctCNV <- function(this_sample){
+ cat(this_sample)
+ this_cnv<-fread(input.files[input.files$sample==this_sample,]$cnv, header = F, data.table = F)
+ # Input file does not have header. Be alert. https://github.com/cancerit/ascatNgs/wiki/Protocol-Correction---format-of-copynumber.caveman.csv
+ colnames(this_cnv)<-c("seg_no", "Chromosome", "chromStart", "chromEnd", "total.copy.number.inNormal", "minor.copy.number.inNormal", "total.copy.number.inTumour", "minor.copy.number.inTumour")
+ print(table(this_cnv$Chromosome))
+ filename<-paste0("tmp/", this_sample, ".cnv")
+ write.table(file = filename, x = this_cnv, quote = F, row.names = F, col.names = T, sep = "\t")
+}
+message("")
+message("========= Preprocessing CNV inputs =========")
+invisible(lapply(sample_names,correctCNV))
+CNV_tab_files <- paste0("tmp/",sample_names,".cnv")
+names(CNV_tab_files) <- sample_names
+
+
+#load SNV data and convert to SNV mutational catalogues
+SNVcat_list <- list()
+message("")
+message("========= Converting SNV mutational catalogues =========")
+for (i in 1:length(SNV_tab_files)){
+ message(sample_names[i])
+ tmpSNVtab <- read.table(SNV_tab_files[i],sep = "\t", fill = T, quote="", header = TRUE,check.names = FALSE, stringsAsFactors = FALSE)
+ res <- tabToSNVcatalogue(subs = tmpSNVtab,genome.v = genome_version)
+ colnames(res$catalogue) <- sample_names[i]
+ SNVcat_list[[i]] <- res$catalogue
+}
+SNV_catalogues <- do.call(cbind,SNVcat_list)
+
+#Initialize feature matrix
+col_hrdetect <- c("del.mh.prop", "SNV3", "SV3", "SV5", "hrd", "SNV8")
+input_matrix <- matrix(NA,nrow = length(sample_names), ncol = length(col_hrdetect), dimnames = list(sample_names,col_hrdetect))
+
+# Compute the proportion of indels at micro-homology
+Indel.del.mh.prop_list <- list()
+message("")
+message("========= Computing the proportion of indels at micro-homology =========")
+for (i in 1:length(Indels_tab_files)){
+ message(sample_names[i])
+ tmpIndeltab <- read.table(Indels_tab_files[i],sep = "\t", fill = T, quote="", header = TRUE, check.names = FALSE, stringsAsFactors = FALSE)
+ res<-tabToIndelsClassification(tmpIndeltab,sample_names[i], genome_version)
+ Indel.del.mh.prop_list[[i]] <- res$count_proportion
+}
+names(Indel.del.mh.prop_list)<-sample_names
+Indel.del.mh.prop <- do.call(rbind,Indel.del.mh.prop_list)
+
+input_matrix[rownames(Indel.del.mh.prop),"del.mh.prop"] <- Indel.del.mh.prop[,"del.mh.prop"]
+
+# Run the pipeline
+message("")
+message("========= Running HRDetect Pipeline =========")
+hrd.arglist = list()
+hrd.arglist[["data_matrix"]] = input_matrix
+hrd.arglist[["genome.v"]] = genome_version
+hrd.arglist[["SV_bedpe_files"]] = SV_bedpe_files
+hrd.arglist[["Indels_tab_files"]] = Indels_tab_files
+hrd.arglist[["CNV_tab_files"]] = CNV_tab_files
+hrd.arglist[["SNV_catalogues"]] = SNV_catalogues
+hrd.arglist[["nparallel"]] = n_parallel
+if (strsplit(as.character(packageVersion("signature.tools.lib")),"\\.")[[1]][1] %in% c("0","1")){
+ hrd.arglist[["signature_type"]] = "COSMIC"
+} else {
+ hrd.arglist[["SNV_signature_version"]] = "COSMICv2"
+}
+res <- do.call(HRDetect_pipeline, hrd.arglist)
+
+#save HRDetect scores
+hrdetect_output = as.data.table(res$hrdetect_output, keep.rownames="sample")
+write.table(hrdetect_output,file = paste0(basename(file_path_sans_ext(inputTSV)),".hrdetect.tsv"), row.names=F, quote=F, sep = "\t")
+
+# Cleanup
+unlink("tmp", recursive = T)
diff --git a/containers/signaturetoolslib/sv_signatures_wrapper.R b/containers/signaturetoolslib/sv_signatures_wrapper.R
new file mode 100755
index 00000000..44b16e90
--- /dev/null
+++ b/containers/signaturetoolslib/sv_signatures_wrapper.R
@@ -0,0 +1,133 @@
+#!/usr/bin/env Rscript
+
+suppressPackageStartupMessages({
+library(signature.tools.lib)
+library(data.table)
+library(tools)
+library(stringr)
+library(BSgenome.Hsapiens.1000genomes.hs37d5)
+library(BSgenome.Hsapiens.UCSC.hg38)
+library(plyr)
+library(dplyr)
+library(getopt)
+})
+
+how_to <- function(){
+ message(" ")
+ message("This script runs the signature calling function for SVs from signature.tools.lib")
+ message("Run this script as follows:")
+ message(" ")
+ message("run_sv_signatures.R -i BEDPE -g GENOME -s NAME -o OUTDIR -n NPARALLEL")
+ message(" ")
+ message(" -i BEDPE Input bedpe with candidate SV calls")
+ message(" -g GENOME hg19/hg38")
+ message(" -s NAME Sample name")
+ message(" -o OUTDIR Output directory")
+ message(" -n NPARALLEL Number of threads")
+ message(" -h Show this explanation")
+}
+
+spec = matrix(c(
+ 'input', 'i', 1, 'character',
+ 'genome', 'g', 1, 'character',
+ 'sampleName', 's', 1, 'character',
+ 'outDir', 'o', 2, 'character',
+ 'nparallel', 'n', 2, 'integer',
+ 'help', 'h', 0, 'logical'
+ ), byrow=TRUE, ncol=4)
+opt = getopt(spec)
+
+if ( !is.null(opt$help) ) {
+ how_to()
+ q(status=1,save = "no")
+}
+
+input.file <- opt$input
+genome.v = opt$genome
+if (genome.v == 'hg19'){
+ ref.genome = BSgenome.Hsapiens.1000genomes.hs37d5
+} else if (genome.v == 'hg38'){
+ ref.genome = BSgenome.Hsapiens.NCBI.GRCh38
+} else {
+ stop("Input genome must be hg19/hg38")
+}
+
+sampleName <- opt$sampleName
+outDir <- opt$outDir
+nparallel <- ifelse( is.null(opt$nparallel),1,opt$nparallel)
+message("## creating output directory if it does not exist ##")
+if ( is.null(opt$outDir)){ outDir <- "./" } else {outDir <- opt$outDir}
+if ( ! dir.exists(outDir)){ dir.create(outDir) }
+
+message("## read and pre-process input ##")
+this_sv <- fread(cmd = paste("grep -v '^##'", input.file), header = T, data.table = F)
+if ("FILTER" %in% names(this_sv)){ this_sv <- this_sv[this_sv$FILTER=="PASS",] }
+this_sv$sample <- rep(sampleName,dim(this_sv)[1])
+setnames(this_sv, "#CHROM_A", "chrom1")
+setnames(this_sv, "START_A", "start1")
+setnames(this_sv, "END_A", "end1")
+setnames(this_sv, "CHROM_B", "chrom2")
+setnames(this_sv, "START_B", "start2")
+setnames(this_sv, "END_B", "end2")
+setnames(this_sv, "STRAND_A", "strand1")
+setnames(this_sv, "STRAND_B", "strand2")
+svclass_dict <- c("BND"="translocation","INV"="inversion","DEL"="deletion","DUP"="tandem-duplication")
+this_sv$svclass <- stringr::str_replace_all(string = this_sv$TYPE,
+ pattern = svclass_dict)
+this_sv <- this_sv[,unlist(strsplit("chrom1,start1,end1,chrom2,start2,end2,strand1,strand2,sample,svclass",split=","))]
+
+SV_bedpe_file <- paste(outDir,paste0(sampleName,".reformat.bedpe"),sep="/")
+writeTable(this_sv, SV_bedpe_file)
+names(SV_bedpe_file) <- sampleName
+
+message("## use signature.tools.lib functions ##")
+message(paste0("# Version: ",as.character(packageVersion("signature.tools.lib"))))
+cat_sv <- bedpeToRearrCatalogue(this_sv)
+randomSeed <- NULL
+set.seed(randomSeed)
+
+sig.arglist = list()
+sig.arglist[["genome.v"]] = genome.v
+sig.arglist[["nparallel"]] = nparallel
+if (strsplit(as.character(packageVersion("signature.tools.lib")),"\\.")[[1]][1] %in% c("0","1")){
+ sig.arglist[["cat"]] = cat_sv
+ sig.arglist[["signature_data_matrix"]] = signature.tools.lib:::RS.Breast560
+ res <- do.call(SignatureFit_withBootstrap, sig.arglist)
+ plotRearrSignatures(res$cat, output_file = paste(outDir,paste0(sampleName,"_catalogues.pdf"),sep="/"))
+ writeTable(t(res$E_median_filtered),paste(outDir,paste0(sampleName,"_exposures.tsv"),sep="/"))
+} else {
+
+ sig.arglist[["useBootstrap"]] = TRUE
+ sig.arglist[["fit_method"]] = "Fit"
+ sig.arglist[["SV_bedpe_files"]] = SV_bedpe_file
+ sig.arglist[["signature_version"]] = "RefSigv2"
+ res <- do.call(signatureFit_pipeline, sig.arglist)
+ plotSignatures(res$catalogues, output_file = paste(outDir,paste0(sampleName,"_catalogues.pdf"),sep="/"),ncolumns=1)
+
+ exp <- res$fitResults$exposures
+ perc <- res$fitResults$exposures %>%
+ as.data.frame %>%
+ mutate(across()/rowSums(across())) %>%
+ rename_with(~paste0(., "_perc"))
+ pvals <- res$fitResults$bootstrap_exposures_pvalues %>%
+ t %>%
+ as.data.frame %>%
+ rename_with(~paste0(., "_pval"))
+
+ exp_extended <- as.data.frame(do.call(cbind, list(exp, perc, pvals)))
+
+ ordered.colnames <- c(paste("RefSigR",rep(c(c(1:5),"6a","6b",c(7:20)), each = 3 ), c("","_perc","_pval"), sep = ""), "unassigned","unassigned_perc")
+ exp_extended <- exp_extended %>%
+ select(ordered.colnames) %>%
+ tibble::rownames_to_column(var="SampleID") %>%
+ relocate(SampleID)
+ write.table(exp_extended,
+ file = paste(outDir,paste0(sampleName,"_exposures.tsv"),sep="/"),
+ row.names = F,
+ quote=F,
+ sep="\t"
+ )
+
+}
+
+saveRDS(res,file="result.rds")
diff --git a/containers/strelka2-manta-bcftools-vt/Dockerfile b/containers/strelka2-manta-bcftools-vt/Dockerfile
old mode 100644
new mode 100755
index cbd6cecc..a8989af1
--- a/containers/strelka2-manta-bcftools-vt/Dockerfile
+++ b/containers/strelka2-manta-bcftools-vt/Dockerfile
@@ -1,8 +1,9 @@
FROM nfcore/base:latest
LABEL \
- authors="Yixiao Gong (gongy@mskcc.org)"
- version.image="2.0.0"
+ authors="Yixiao Gong (gongy@mskcc.org)" \
+ contributors="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.image="2.0.1"
COPY environment.yml /
RUN conda env create -f /environment.yml && conda clean -a
diff --git a/containers/strelka2-manta-bcftools-vt/environment.yml b/containers/strelka2-manta-bcftools-vt/environment.yml
old mode 100644
new mode 100755
index 02434a64..53d6f41c
--- a/containers/strelka2-manta-bcftools-vt/environment.yml
+++ b/containers/strelka2-manta-bcftools-vt/environment.yml
@@ -12,3 +12,4 @@ dependencies:
- pysam=0.15.2
- strelka=2.9.10
- manta=1.5.0
+ - bioconda::vcftools=0.1.16
diff --git a/containers/strelka2_manta/Dockerfile b/containers/strelka2_manta/Dockerfile
deleted file mode 100644
index 307ea790..00000000
--- a/containers/strelka2_manta/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-FROM centos:6
-
-MAINTAINER Jason Hwee
-
-RUN yum -y update && \
- yum -y install wget && \
- yum -y install tar.x86_64 && \
- yum clean all
-
-RUN wget https://github.com/Illumina/strelka/releases/download/v2.9.10/strelka-2.9.10.centos6_x86_64.tar.bz2 \
- && tar xvjf strelka-2.9.10.centos6_x86_64.tar.bz2 \
- && wget --no-check-certificate https://github.com/Illumina/manta/releases/download/v1.5.0/manta-1.5.0.centos6_x86_64.tar.bz2 \
- && tar xvjf manta-1.5.0.centos6_x86_64.tar.bz2
-
-ENV PATH="/strelka-2.9.10.centos6_x86_64/bin:/manta-1.5.0.centos6_x86_64/bin:${PATH}"
-ENV STRELKA_INSTALL_PATH="/strelka-2.9.10.centos6_x86_64"
diff --git a/containers/svaba/Dockerfile b/containers/svaba/Dockerfile
new file mode 100644
index 00000000..16172fe5
--- /dev/null
+++ b/containers/svaba/Dockerfile
@@ -0,0 +1,38 @@
+# adapted from https://bitbucket.org/weischenfeldt/svaba/src/master/Dockerfile
+# they use SvABA version 134 (4a0606eba1bfd43c3d38433b27ac7f5e4887bc1e) but that is outdated at this point so start with latest commit
+FROM centos:7.3.1611
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.image="0.0.1"
+
+RUN yum install -y epel-release
+RUN yum install -y \
+ git \
+ zlib-devel \
+ gcc gcc-c++ \
+ gcc-gfortran \
+ make \
+ wget \
+ bzip2-devel.x86_64 \
+ xz-devel \
+ bcftools
+
+# original commit for pcawg
+ENV SVABA_COMMIT 4a0606eba1bfd43c3d38433b27ac7f5e4887bc1e
+# latest release commit
+#ENV SVABA_COMMIT d12cf224f7a488b913eabbcf54a215e17238032c
+
+# SvABA Version: 1.1.3 (4d7b57);
+# Submodule path 'SeqLib': checked out 'f7a89a127409a3f52fdf725fa74e5438c68e48fb'
+# Submodule path 'SeqLib/bwa': checked out 'c02766e3c34ac3f4af9842e20a54b7f9f4b36d0b'
+# Submodule path 'SeqLib/fermi-lite': checked out '5bc90f8d70e2b66184eccbd223a3be714c914365'
+# Submodule path 'SeqLib/htslib': checked out '49fdfbda20acbd73303df3c7fef84f2d972c5f8d'
+RUN git clone https://github.com/walaj/svaba && \
+cd svaba && \
+git checkout ${SVABA_COMMIT} && \
+git submodule update --init --recursive && \
+./configure && \
+make && \
+make install
+
+ENV PATH=/svaba/bin/:$PATH
diff --git a/containers/svclone/Dockerfile b/containers/svclone/Dockerfile
new file mode 100644
index 00000000..76468e62
--- /dev/null
+++ b/containers/svclone/Dockerfile
@@ -0,0 +1,23 @@
+FROM continuumio/miniconda3:4.12.0
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.base="4.10.3" \
+ version.image="0.0.1"
+
+ENV envName "svclone"
+
+RUN apt-get update && apt-get install -y procps && apt-get clean -y
+
+RUN conda update -n base -c defaults conda
+RUN conda create --name ${envName} -c bioconda -c conda-forge svclone=1.1.1-0 r-base
+ENV PATH /opt/conda/envs/${envName}/bin:$PATH
+RUN echo "export PATH=$PATH" > /etc/environment
+
+RUN mkdir -p /config && \
+ cd /config && \
+ wget https://github.com/mcmero/SVclone/raw/4a02d5e2d8f548b2f38e9f7f77ea432bf456b349/svclone_config.ini
+
+RUN sed -i "s/itrx_class: INTRX/itrx_class: INTRX,TRA,BND/g" /config/svclone_config.ini
+
+# The following solves a bug where certain files are written to the wrong location. Once svclone is upgraded from 1.1.1-0, this line should be deleted.
+RUN wget https://raw.githubusercontent.com/mcmero/SVclone/034d04996216f4a3378f23e583c984724e05ffbc/SVclone/post_assign.R -O /opt/conda/envs/${envName}/lib/python3.6/site-packages/SVclone/post_assign.R
diff --git a/containers/svclone/environment.yml b/containers/svclone/environment.yml
new file mode 100644
index 00000000..41a39db7
--- /dev/null
+++ b/containers/svclone/environment.yml
@@ -0,0 +1,8 @@
+# You can use this file to create a conda environment for this pipeline:
+# conda env create -f environment.yml
+channels:
+ - bioconda
+
+dependencies:
+ - conda-forge::r-base
+ - bioconda::svclone==1.1.1-0
diff --git a/containers/svclone/svclone_config.ini b/containers/svclone/svclone_config.ini
new file mode 100644
index 00000000..aafcf5c5
--- /dev/null
+++ b/containers/svclone/svclone_config.ini
@@ -0,0 +1,157 @@
+##################################################################################################################
+# Sample config file for SVclone
+##################################################################################################################
+
+# SV processing-related options
+
+[BamParameters]
+# read length of BAM file; -1 = infer dynamically.
+read_len: -1
+
+# Mean fragment length (also known as insert length); -1 = infer dynamically.
+insert_mean: 1000
+
+# Standard deviation of insert length; -1 = infer dynamically.
+insert_std: 400
+
+# mean coverage of the bam
+# used as parameter in cluster number initialisation
+# informs max read depth we consider when extracting reads from SV loci
+mean_cov: 50
+
+# maximum considered copy-number
+# informs max read depth we consider when extracting reads from SV loci
+max_cn: 10
+
+[SVannotateParameters]
+# Whether to use breakpoint direction in the input file (must be specified in input).
+use_dir: True
+
+# if SV classes exist on input, specify SV class field name.
+sv_class_field: none
+
+# Use specified breaks without checking for the soft-clip consensus position.
+# If your SV caller offsets breaks due to micro-homology, e.g. Socrates/GRIDSS,
+# using this option is not recommended. Note: cannot be skipped if use_dir is false.
+trust_sc_position: False
+
+[SVcountParameters]
+# "wobble length" tolerance threshold which we allow breaks to be inexact.
+threshold: 6
+
+# minimum basepairs a "normal" read must overlap break to be counted.
+norm_overlap: 10
+
+# minimum basepairs a supporting read must be softclipped over the break.
+sc_len: 10
+
+[SVclasses]
+# Naming conventions used to label SV types.
+inversion_class: INV
+deletion_class: DEL
+dna_gain_class: DUP,INTDUP
+dna_loss_class: DEL,INV,TRX
+itrx_class: INTRX,BND
+
+## Options if using Socrates SV caller.
+
+[SocratesOpts]
+# Column names used by Socrates output format (input must be headered).
+pos1: C1_anchor
+dir1: C1_anchor_dir
+pos2: C1_realign
+dir2: C1_realign_dir
+avg_mapq1: C1_avg_realign_mapq
+avg_mapq2: C2_avg_realign_mapq
+repeat1: repeat1
+repeat2: repeat2
+
+# categories of repeats to filter out
+filter_repeats: Satellite,Simple_repeat
+
+# Filter out SVs with lower average MAPQ than this value.
+min_mapq: 20
+
+[DebugParameters]
+# Whether to output (as a bam) and accurately recount anomalous reads.
+# Useful for diagnosing issues with read counting.
+write_anomalous: False
+
+##################################################################################################################
+# Filtering options
+##################################################################################################################
+
+[FilterParameters]
+# Keep only copy-number neutral variants if True.
+neutral: False
+
+# Filter out SVs below this size. If -1, size is insert_mean + (3 * insert_std).
+size_filter: -1
+
+# Filter any variants with total depth below this value.
+min_dep: 8
+
+# Require at least N spanning/discordant reads to keep SV break-pair.
+min_span: 1
+
+# Require at least N split reads to keep SV break-pair.
+min_split: 1
+
+# Filter out variants with depth values that are considered outliers, based on the
+# copy-number adjusted distribution of depths. Use with caution: may overfilter data.
+filter_outliers: False
+
+# Filters out variants on non-canonical chroms (i.e. mapping to contigs or non-standard chromosomes)
+filter_chroms: False
+
+# Filters out variants where variant falls in a locus with a subclonal CNV (for SVs this could be either locus)
+filter_subclonal_cnvs: False
+
+# Do not remove variants based on their CNV state if true
+# Either matches the closest proximity CNV state; in case of no data for
+# target chromsome, assumes variant is ploidy/2 for both major/minor
+# Note: does not affect SNVs as they are always strict-filtered
+strict_cnv_filt: True
+
+# bp threshold that a germline and tumour SV must match to be considered the same event.
+germline_threshold: 10
+
+# base scaling factor for supporting reads = 1 + (support_adjust_factor * purity).
+# recommended values: 0.12 for bowtie-aligned samples, 0.2 for bwa-aligned samples
+support_adjust_factor: 0
+
+# SVs are offset by this number of base-pairs when matching CNVs.
+sv_offset: 100000
+
+[ValidationParameters]
+chroms: 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,X,Y
+
+##################################################################################################################
+# Clustering options
+##################################################################################################################
+
+[ClusterParameters]
+# If n_runs > 1, runs can be multithreaded
+threads: 8
+
+# ccube numOfRepeat parameter
+repeat: 5
+
+# Max iterations for ccube clustering
+n_iter: 1000
+
+# Max number of possible clusters
+# higher values mean longer iteration time
+clus_limit: 6
+
+# Use adjusted normal reads counts rather than raw.
+adjusted: True
+
+# whether the sample is male, only affects assumption of
+# normal copy-number state for SVs on the X chromosome
+male: True
+
+# to bolster SV numbers, sets the number of SV per SV to
+# simulate when coclustering - e.g. with a data set of 50 SV,
+# setting this to 1 will simulate 50 extra SV for clustering
+sv_to_sim: 0
diff --git a/containers/svclone/svclone_wrapper.py b/containers/svclone/svclone_wrapper.py
new file mode 100644
index 00000000..364016b7
--- /dev/null
+++ b/containers/svclone/svclone_wrapper.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+
+"""
+Run SVclone with Tempo inputs. Does preprocessessing of
+Tempo inputs before executing SVclone steps.
+ * Maf file: should not contain filtered variants. The
+ columns Chromosome, vcf_pos, t_ref_count, t_alt_count are
+ retained.
+ * Bedpe file: should not contain filtered variants. The
+ columns #CHROM_A, START_A, STRAND_A, CHROM_B, START_B,
+ STRAND_B and TYPE are retained.
+ * Purity/ploidy file: can be produced either by ASCAT or by
+ generate_samplestatistics.R script in the DoFacets process.
+ * Config file: Template config file which is modified before
+ running SVclone.
+Usage: svclone_wrapper_tempo.py -h
+"""
+
+__author__ = "Anne Marie Noronha"
+__email__ = "noronhaa@mskcc.org"
+__version__ = "0.0.1"
+__status__ = "Dev"
+
+import configparser
+import argparse, os, sys
+import pysam, pandas as pd, numpy as np
+import subprocess
+
+def usage():
+ parser = argparse.ArgumentParser(description="Prepare inputs for SVClone")
+ parser.add_argument("--maf", help="Filtered maf file")
+ parser.add_argument("--bedpe", help="Filtered SV calls in bedpe format")
+ parser.add_argument("--sampleid", metavar="s_C_XXXXXX_XNNN_d__s_C_XXXXXX_XNNN_d")
+ parser.add_argument("--purity_ploidy", help="purity and ploidy file from Facets process")
+ parser.add_argument("--out_dir",default="svclone_inputs", help="Folder to write all svclone inputs")
+ parser.add_argument("--cfg_template",help="Template config for editing")
+ parser.add_argument("--bam",help="Tumor Bam file")
+ parser.add_argument("--cnv",help="CNV file")
+
+ return parser.parse_args()
+
+def main():
+ args = usage()
+
+ if os.path.isdir(args.out_dir):
+ pass
+ else:
+ os.mkdir(args.out_dir)
+ mut_out = os.path.join(args.out_dir,"callstats.txt")
+ sv_out = os.path.join(args.out_dir,"simple.sv.txt")
+ purity_ploidy_out = os.path.join(args.out_dir,"svclone_ploidy.txt")
+ cfg_out = os.path.join(args.out_dir,"svclone_config.ini")
+
+ svclone_inputs = {"snv":mut_out,"sv":sv_out, "purity_ploidy":purity_ploidy_out, "cfg":cfg_out}
+
+ ## read in maf and output reformatted SNPs
+ mut = pd.read_csv(args.maf,sep="\t",header=0)
+ mut["judgement"] = "KEEP"
+ mut = mut["Chromosome,vcf_pos,t_ref_count,t_alt_count,judgement".split(",")]
+ mut.columns = "contig,position,t_ref_sum,t_alt_sum,judgement".split(",")
+ mut.to_csv(mut_out, sep="\t", header=True, index=False)
+
+ ## read in bedpe and output reformatted SVs
+ with open(args.bedpe,"r") as f:
+ in_meta=True
+ line_cursor = 0
+ while in_meta:
+ bedpe_line = f.readline()
+ if bedpe_line.startswith("#CHROM"):
+ in_meta=False
+ f.seek(line_cursor)
+ sv = pd.read_csv(f, header=0, sep="\t")
+ line_cursor = f.tell()
+
+ sv = sv["#CHROM_A,START_A,STRAND_A,CHROM_B,START_B,STRAND_B,TYPE".split(",")]
+ sv.columns = "chr1,pos1,dir1,chr2,pos2,dir2,classification".split(",")
+ sv.to_csv(sv_out,header=True, sep="\t", index=False )
+
+ ## read in ploidy file and output reformatted information
+ purity_ploidy = pd.read_csv(args.purity_ploidy, header=None, sep=" ", index_col=0, usecols=[0, 1]).T
+ purity_ploidy["sample"] = args.sampleid
+ purity_ploidy = purity_ploidy[["sample","rho","Ploidy"]]
+ purity_ploidy.columns = "sample,purity,ploidy".split(",")
+ purity_ploidy.to_csv(purity_ploidy_out, index=False, header=True, sep="\t", )
+
+ ## read in config and adjust insert_mean, insert_std
+ ## also adjust SVClass descriptor terms.
+ cfg = configparser.ConfigParser()
+ cfg.read(args.cfg_template)
+ max_cn = max(pd.read_csv(args.cnv, sep=",",header=None).iloc[:, 6].tolist())
+ insert_mean, insert_std = estimateInsertSizeDistribution(args.bam)
+ read_len = estimateReadLen(args.bam)
+ mean_cov = estimateCoverage(args.bam)
+ cfg.set('BamParameters', 'read_len', str(int(read_len)))
+ cfg.set('BamParameters', 'insert_mean', str(int(insert_mean)))
+ cfg.set('BamParameters', 'insert_std', str(int(insert_std)))
+ cfg.set('BamParameters', 'max_cn', str(int(max_cn)))
+ cfg.set('BamParameters', 'mean_cov', str(int(mean_cov)))
+ cfg.set('SVclasses', 'itrx_class', "INTRX,TRA,BND")
+ cfg.set('SVannotateParameters','sv_class_field','classification')
+ with open(cfg_out, 'w') as configfile:
+ cfg.write(configfile)
+
+ print(svclone_inputs)
+
+ svclone_wrapper(
+ sv=svclone_inputs["sv"],
+ bam=args.bam,
+ outPrefix=args.sampleid,
+ config=svclone_inputs["cfg"],
+ cnv=args.cnv,
+ snv=svclone_inputs["snv"],
+ purity_ploidy=svclone_inputs["purity_ploidy"]
+ )
+
+def estimateInsertSizeDistribution(bamfile, alignments=10000):
+ sam = pysam.AlignmentFile(bamfile)
+ inserts = np.array([read.tlen for read in sam.head(alignments) if read.tlen > 0 and read.tlen < 500000 and read.is_paired and read.next_reference_id == read.reference_id ])
+ insert_mean, insert_std = np.mean(inserts), np.std(inserts)
+ return [ insert_mean, insert_std ]
+
+def readConfigFile(filePath):
+ config = configparser.ConfigParser(delimiters=":")
+ config.read(filePath)
+ return config
+
+def writeConfigFile(outFilePath,cfg_obj):
+ with open(outFilePath, 'w') as configfile:
+ cfg_obj.write(configfile,space_around_delimiters=True)
+
+def estimateCoverage(bamfile):
+ """
+ estimate coverage in bam file
+ assumes uniform coverage
+ """
+ sam = pysam.AlignmentFile(bamfile)
+ cov = list()
+ interval_len = 1000000
+ for i in sam.references:
+ if i.startswith("GL") or i.startswith("NC"): continue
+ if i in ["hs37d5","X","Y","chrX","chrY"]: continue
+ for j in range(1,int(sam.get_reference_length(i)/interval_len)):
+ try:
+ x = sam.count_coverage(i,start=j*interval_len, stop=(j*interval_len) + 50)
+ cov += list(np.add(np.add(x[0],x[1]),np.add(x[2],x[3])))
+ except ValueError as e:
+ print(e)
+ pass
+ return int(sum(cov)/len(cov))
+
+def estimateReadLen(bamfile,alignments=1000):
+ sam = pysam.AlignmentFile(bamfile)
+ sizes = [read.rlen for read in sam.head(alignments)]
+ sizes_dict = dict()
+ for i in sizes:
+ if not i in sizes_dict: sizes_dict[i] = 0
+ sizes_dict[i] += 1
+ read_len = max(sizes_dict, key=sizes_dict.get)
+ if sizes_dict[read_len] < alignments * .5:
+ read_len = int(sum(sizes)/len(sizes))
+ return read_len
+
+def svclone_wrapper(sv,bam,outPrefix,config,cnv,snv,purity_ploidy):
+ cmd_dict = dict()
+ cmd_dict["annotate"] = [
+ "svclone", "annotate",
+ "-cfg", config,
+ "-b", bam,
+ "-s", outPrefix,
+ "-i", sv,
+ "--sv_format","simple"
+ ]
+ cmd_dict["count"] = [
+ "svclone", "count",
+ "-cfg", config,
+ "-b", bam,
+ "-s", outPrefix,
+ "-i",os.path.join(outPrefix,outPrefix+"_svin.txt")
+ ]
+ cmd_dict["filter"] = [
+ "svclone", "filter",
+ "-cfg", config,
+ "-s", outPrefix,
+ "-i",os.path.join(outPrefix,outPrefix+"_svinfo.txt"),
+ "--snv_format","mutect_callstats",
+ "--snvs",snv,
+ "--cnvs",cnv,
+ "-p",purity_ploidy
+ ]
+ cmd_dict["cluster"] = [
+ "svclone", "cluster",
+ "-cfg", config,
+ "-s", outPrefix
+ ]
+ cmd_dict["postassign"] = [
+ "svclone", "postassign",
+ "-s", outPrefix,
+ "--joint"
+ ]
+
+ print("Running SVclone from wrapper")
+ for i in ["annotate","count","filter","cluster","postassign"]:
+ print("Running svclone {}".format(i), file=sys.stderr)
+ subprocess.run(cmd_dict[i])
+ print("Completed svclone {}".format(i), file=sys.stderr)
+
+if __name__ == "__main__":
+ main()
diff --git a/containers/svtools/Dockerfile b/containers/svtools/Dockerfile
new file mode 100644
index 00000000..d78dd0ab
--- /dev/null
+++ b/containers/svtools/Dockerfile
@@ -0,0 +1,38 @@
+FROM halllab/svtools:v0.5.1
+
+LABEL maintainer="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.image="0.0.3"
+
+RUN mkdir -p /tmp \
+ && apt-get update \
+ && apt-get install -y \
+ procps \
+ gcc \
+ libz-dev \
+ sqlite3 \
+ bedtools \
+ tcl \
+ curl \
+ wget \
+ build-essential \
+ bcftools \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --upgrade pip
+RUN pip install pybedtools
+
+ENV ANNOTSV /opt/annotsv
+ENV ANNOTSV_VERSION v3.1.1
+
+# This is an installation of AnnotSV source code, with the preparation of the reference files intentionally omitted
+# Reference files were prepared outside of docker using this container after it was built.
+# To prepare reference files, start a container and: cd /opt/annotsv/ && make PREFIX= install-human-annotation
+RUN mkdir -p unpack_annotsv \
+ && wget https://github.com/lgmgeo/AnnotSV/archive/refs/tags/${ANNOTSV_VERSION}.tar.gz \
+ && tar -C unpack_annotsv -xvzf ${ANNOTSV_VERSION}.tar.gz \
+ && mv unpack_annotsv/AnnotSV* ${ANNOTSV} \
+ && rm -rf unpack_annotsv \
+ && cd ${ANNOTSV} \
+ && make PREFIX=. install
+
diff --git a/containers/vcf2maf/Dockerfile b/containers/vcf2maf/Dockerfile
index 0b4e64fa..c36ac745 100644
--- a/containers/vcf2maf/Dockerfile
+++ b/containers/vcf2maf/Dockerfile
@@ -1,10 +1,10 @@
-FROM ubuntu:18.04
+from cmopipeline/vcf2maf:vep88_1.2.7
LABEL maintainer="Christopher Allan Bolipata (bolipatc@mskcc.org)" \
contributor="Nikhil Kumar (kumarn1@mskcc.org)" \
contributor="Philip Jonsson (jonssonp@mskcc.org)" \
contributor="Anne Marie Noronha (noronhaa@mskcc.org)" \
- version.image="1.2.7" \
+ version.image="1.3.0" \
version.vcf2maf="1.6.17" \
version.vep="88" \
version.htslib="1.9" \
@@ -13,9 +13,9 @@ LABEL maintainer="Christopher Allan Bolipata (bolipatc@mskcc.org)" \
version.perl="5.26.2-r1" \
version.alpine="3.8" \
version.oncokb_annotator="1.1.0" \
- version.filter_somatic_maf="0.6.2" \
+ version.filter_somatic_maf="0.7.0" \
version.filter_germline_maf="0.2.2" \
- version.annotateMaf="1.0.2" \
+ version.annotateMaf="1.0.3" \
source.vcf2maf="https://github.com/mskcc/vcf2maf/releases/tag/v1.6.17" \
source.vep="http://dec2016.archive.ensembl.org/info/docs/tools/vep/script/vep_download.html#versions" \
source.htslib="https://github.com/samtools/htslib/releases/tag/1.9" \
@@ -24,128 +24,14 @@ LABEL maintainer="Christopher Allan Bolipata (bolipatc@mskcc.org)" \
source.oncokb_annotator="https://github.com/oncokb/oncokb-annotator/releases/tag/v1.1.0" \
source.annotateMaf="https://github.com/taylor-lab/annotateMaf"
-ENV VCF2MAF_VERSION 1.6.17
-ENV VEP_VERSION 88
-ENV HTSLIB_VERSION 1.9
-ENV SAMTOOLS_VERSION 1.9
-ENV BCFTOOLS_VERSION 1.9
-ENV ONCOKB_VERSION 1.1.0
-ENV ANNOTATE_MAF 1.0.2
-
-RUN apt-get update --fix-missing && apt-get install -y \
- tcsh \
- libnss-sss \
- git \
- python2.7 \
- python-pip \
- build-essential \
- wget \
- && apt-get clean && apt-get purge \
- && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-RUN apt-get update && apt-get install -y \
- cpanminus \
- perl \
- curl \
- libssl-dev \
- libperlio-gzip-perl \
- libgd-perl \
- && apt-get clean && apt-get purge \
- && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-# Install python dependencies
-RUN pip install --upgrade pip
-RUN pip install matplotlib
-
-RUN apt-get update && apt-get install -y libnet-ssleay-perl libcrypt-ssleay-perl libextutils-pkgconfig-perl
-
-# install perl libraries that VEP will need
-RUN cpanm --notest LWP LWP::Simple LWP::Protocol::https
-
-RUN cpanm --notest Archive::Extract Archive::Tar Archive::Zip
-
-RUN cpanm --notest CGI DBI Encode version Time::HiRes File::Copy::Recursive Perl::OSType Module::Metadata
-
-RUN cpanm --notest Sereal JSON Bio::Root::Version Set::IntervalTree
-
-RUN apt-get update && apt-get install -y zlib1g-dev bzip2 libbz2-dev liblzma-dev unzip
-
-# install htslib (for vep)
-RUN cd /tmp && wget https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2 \
- && tar xvjf htslib-${HTSLIB_VERSION}.tar.bz2 \
- && cd /tmp/htslib-${HTSLIB_VERSION} \
- && ./configure \
- && make && make install \
- && rm -rf /tmp/*
-
-# install vep
-RUN cd /tmp && wget https://github.com/Ensembl/ensembl-tools/archive/release/${VEP_VERSION}.zip \
- && unzip ${VEP_VERSION} \
- && cd /tmp/ensembl-tools-release-${VEP_VERSION}/scripts/variant_effect_predictor \
- && perl INSTALL.pl --AUTO a 2>&1 | tee install.log \
- && cd /tmp && mv /tmp/ensembl-tools-release-${VEP_VERSION}/scripts/variant_effect_predictor /usr/bin/vep \
- && rm -rf /tmp/*
-
-# Install samtools
-RUN apt-get update && \
- apt-get install --yes \
- libncurses5-dev \
- vcftools \
- libbz2-dev \
- libxml2 \
- libxml2-dev \
- libcurl4-openssl-dev \
- liblzma-dev && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
-
-RUN cd /tmp && curl -L -o tmp.tar.gz https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2 && \
- mkdir samtools && \
- tar -C samtools --strip-components 1 -jxf tmp.tar.gz && \
- cd samtools && \
- ./configure && \
- make && \
- make install && \
- cd .. && \
- rm -rf /tmp/*
-
-RUN cd /tmp && curl -L -o tmp2.tar.gz https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
- mkdir bcftools && \
- tar -C bcftools --strip-components 1 -jxf tmp2.tar.gz && \
- cd bcftools && \
- make && \
- make install && \
- cd .. && \
- rm -rf /tmp/*
-
-# Install vcf2maf
-WORKDIR /opt/
-
-RUN cd /opt/ && curl -ksSL -o tmp.tar.gz https://github.com/mskcc/vcf2maf/archive/v${VCF2MAF_VERSION}.tar.gz && \
- tar --strip-components 1 -zxf tmp.tar.gz && \
- rm tmp.tar.gz && \
- chmod +x *.pl
-
-# Install R with depencies
-RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install --yes \
- r-base-core r-base-dev
-
-# versioning R packages for reproducibility and compatibility
-RUN R -e "install.packages('remotes')"
-COPY r_versions.txt /
-RUN R -e "require(remotes) ; x <- read.table('/r_versions.txt',sep='\t',header=T) ; for (i in 1:dim(x)[[1]]){ install_version(as.character(x\$package[i]), version = as.character(x\$version[i]), repos = 'http://cran.us.r-project.org') }"
+ENV ANNOTATE_MAF 1.0.3
RUN R -e "devtools::install_github('taylor-lab/annotateMaf', ref = '${ANNOTATE_MAF}', upgrade = 'never')"
-
-# install oncokb-annotator
-RUN cd /tmp && wget -O oncokb_annotator-v${ONCOKB_VERSION} https://github.com/oncokb/oncokb-annotator/archive/v${ONCOKB_VERSION}.zip \
- && unzip oncokb_annotator-v${ONCOKB_VERSION} \
- && mkdir -p /usr/bin/oncokb_annotator \
- && cp -r oncokb-annotator-${ONCOKB_VERSION}/* /usr/bin/oncokb_annotator/ \
- && rm -rf /var/cache/apk/* /tmp/*
# Add filter script
+RUN rm /usr/bin/filter-somatic-maf.R
+RUN rm /usr/bin/filter-germline-maf.R
+
COPY filter-somatic-maf.R /usr/bin
COPY filter-germline-maf.R /usr/bin
RUN chmod 755 /usr/bin/filter-somatic-maf.R
diff --git a/containers/vcf2maf/Dockerfile_old b/containers/vcf2maf/Dockerfile_old
new file mode 100644
index 00000000..0b4e64fa
--- /dev/null
+++ b/containers/vcf2maf/Dockerfile_old
@@ -0,0 +1,153 @@
+FROM ubuntu:18.04
+
+LABEL maintainer="Christopher Allan Bolipata (bolipatc@mskcc.org)" \
+ contributor="Nikhil Kumar (kumarn1@mskcc.org)" \
+ contributor="Philip Jonsson (jonssonp@mskcc.org)" \
+ contributor="Anne Marie Noronha (noronhaa@mskcc.org)" \
+ version.image="1.2.7" \
+ version.vcf2maf="1.6.17" \
+ version.vep="88" \
+ version.htslib="1.9" \
+ version.bcftools="1.9" \
+ version.samtools="1.9" \
+ version.perl="5.26.2-r1" \
+ version.alpine="3.8" \
+ version.oncokb_annotator="1.1.0" \
+ version.filter_somatic_maf="0.6.2" \
+ version.filter_germline_maf="0.2.2" \
+ version.annotateMaf="1.0.2" \
+ source.vcf2maf="https://github.com/mskcc/vcf2maf/releases/tag/v1.6.17" \
+ source.vep="http://dec2016.archive.ensembl.org/info/docs/tools/vep/script/vep_download.html#versions" \
+ source.htslib="https://github.com/samtools/htslib/releases/tag/1.9" \
+ source.bcftools="https://github.com/samtools/bcftools/releases/tag/1.9" \
+ source.samtools="https://github.com/samtools/samtools/releases/tag/1.9" \
+ source.oncokb_annotator="https://github.com/oncokb/oncokb-annotator/releases/tag/v1.1.0" \
+ source.annotateMaf="https://github.com/taylor-lab/annotateMaf"
+
+ENV VCF2MAF_VERSION 1.6.17
+ENV VEP_VERSION 88
+ENV HTSLIB_VERSION 1.9
+ENV SAMTOOLS_VERSION 1.9
+ENV BCFTOOLS_VERSION 1.9
+ENV ONCOKB_VERSION 1.1.0
+ENV ANNOTATE_MAF 1.0.2
+
+RUN apt-get update --fix-missing && apt-get install -y \
+ tcsh \
+ libnss-sss \
+ git \
+ python2.7 \
+ python-pip \
+ build-essential \
+ wget \
+ && apt-get clean && apt-get purge \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN apt-get update && apt-get install -y \
+ cpanminus \
+ perl \
+ curl \
+ libssl-dev \
+ libperlio-gzip-perl \
+ libgd-perl \
+ && apt-get clean && apt-get purge \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install python dependencies
+RUN pip install --upgrade pip
+RUN pip install matplotlib
+
+RUN apt-get update && apt-get install -y libnet-ssleay-perl libcrypt-ssleay-perl libextutils-pkgconfig-perl
+
+# install perl libraries that VEP will need
+RUN cpanm --notest LWP LWP::Simple LWP::Protocol::https
+
+RUN cpanm --notest Archive::Extract Archive::Tar Archive::Zip
+
+RUN cpanm --notest CGI DBI Encode version Time::HiRes File::Copy::Recursive Perl::OSType Module::Metadata
+
+RUN cpanm --notest Sereal JSON Bio::Root::Version Set::IntervalTree
+
+RUN apt-get update && apt-get install -y zlib1g-dev bzip2 libbz2-dev liblzma-dev unzip
+
+# install htslib (for vep)
+RUN cd /tmp && wget https://github.com/samtools/htslib/releases/download/${HTSLIB_VERSION}/htslib-${HTSLIB_VERSION}.tar.bz2 \
+ && tar xvjf htslib-${HTSLIB_VERSION}.tar.bz2 \
+ && cd /tmp/htslib-${HTSLIB_VERSION} \
+ && ./configure \
+ && make && make install \
+ && rm -rf /tmp/*
+
+# install vep
+RUN cd /tmp && wget https://github.com/Ensembl/ensembl-tools/archive/release/${VEP_VERSION}.zip \
+ && unzip ${VEP_VERSION} \
+ && cd /tmp/ensembl-tools-release-${VEP_VERSION}/scripts/variant_effect_predictor \
+ && perl INSTALL.pl --AUTO a 2>&1 | tee install.log \
+ && cd /tmp && mv /tmp/ensembl-tools-release-${VEP_VERSION}/scripts/variant_effect_predictor /usr/bin/vep \
+ && rm -rf /tmp/*
+
+# Install samtools
+RUN apt-get update && \
+ apt-get install --yes \
+ libncurses5-dev \
+ vcftools \
+ libbz2-dev \
+ libxml2 \
+ libxml2-dev \
+ libcurl4-openssl-dev \
+ liblzma-dev && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN cd /tmp && curl -L -o tmp.tar.gz https://github.com/samtools/samtools/releases/download/${SAMTOOLS_VERSION}/samtools-${SAMTOOLS_VERSION}.tar.bz2 && \
+ mkdir samtools && \
+ tar -C samtools --strip-components 1 -jxf tmp.tar.gz && \
+ cd samtools && \
+ ./configure && \
+ make && \
+ make install && \
+ cd .. && \
+ rm -rf /tmp/*
+
+RUN cd /tmp && curl -L -o tmp2.tar.gz https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
+ mkdir bcftools && \
+ tar -C bcftools --strip-components 1 -jxf tmp2.tar.gz && \
+ cd bcftools && \
+ make && \
+ make install && \
+ cd .. && \
+ rm -rf /tmp/*
+
+# Install vcf2maf
+WORKDIR /opt/
+
+RUN cd /opt/ && curl -ksSL -o tmp.tar.gz https://github.com/mskcc/vcf2maf/archive/v${VCF2MAF_VERSION}.tar.gz && \
+ tar --strip-components 1 -zxf tmp.tar.gz && \
+ rm tmp.tar.gz && \
+ chmod +x *.pl
+
+# Install R with depencies
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+ r-base-core r-base-dev
+
+# versioning R packages for reproducibility and compatibility
+RUN R -e "install.packages('remotes')"
+COPY r_versions.txt /
+RUN R -e "require(remotes) ; x <- read.table('/r_versions.txt',sep='\t',header=T) ; for (i in 1:dim(x)[[1]]){ install_version(as.character(x\$package[i]), version = as.character(x\$version[i]), repos = 'http://cran.us.r-project.org') }"
+
+RUN R -e "devtools::install_github('taylor-lab/annotateMaf', ref = '${ANNOTATE_MAF}', upgrade = 'never')"
+
+# install oncokb-annotator
+RUN cd /tmp && wget -O oncokb_annotator-v${ONCOKB_VERSION} https://github.com/oncokb/oncokb-annotator/archive/v${ONCOKB_VERSION}.zip \
+ && unzip oncokb_annotator-v${ONCOKB_VERSION} \
+ && mkdir -p /usr/bin/oncokb_annotator \
+ && cp -r oncokb-annotator-${ONCOKB_VERSION}/* /usr/bin/oncokb_annotator/ \
+ && rm -rf /var/cache/apk/* /tmp/*
+
+# Add filter script
+COPY filter-somatic-maf.R /usr/bin
+COPY filter-germline-maf.R /usr/bin
+RUN chmod 755 /usr/bin/filter-somatic-maf.R
+RUN chmod 755 /usr/bin/filter-germline-maf.R
+
diff --git a/containers/vcf2maf/filter-somatic-maf.R b/containers/vcf2maf/filter-somatic-maf.R
index 37af9d4c..983eafc5 100755
--- a/containers/vcf2maf/filter-somatic-maf.R
+++ b/containers/vcf2maf/filter-somatic-maf.R
@@ -3,7 +3,7 @@
# __author__ = "Philip Jonsson"
# __email__ = "jonssonp@mskcc.org"
# __contributor__ = "Anne Marie Noronha (noronhaa@mskcc.org)"
-# __version__ = "0.6.2"
+# __version__ = "0.7.0"
# __status__ = "Dev"
suppressPackageStartupMessages({
@@ -39,6 +39,8 @@ parser$add_argument('-gaf', '--gnomad-allele-frequency', type = 'double', requir
default = 0.01, help = 'gnomAD allele frequency cut-off [default %(default)s]')
parser$add_argument('-pon', '--normal-panel-count', type = 'integer', required = FALSE,
default = 10, help = 'Panel of normals count cut-off [default %(default)s]')
+parser$add_argument('-onco', '--oncokb-url', type = 'character', required = FALSE,
+ default = "https://data-legacy.oncokb.aws.mskcc.org/api/v1/genes/", help = 'Panel of normals count cut-off [default %(default)s]')
# Get inputs
args = parser$parse_args()
@@ -51,6 +53,7 @@ normal_depth_cutoff = args$normal_depth
normal_readcount_cutoff = args$normal_count
gnomad_af_cutoff = args$gnomad_allele_frequency
pon_cutoff = args$normal_panel_count
+oncokb_url = args$oncokb_url
add_tag = function(filter, tag) {
split_filter <- strsplit(filter,";")
@@ -100,7 +103,7 @@ maf[(t_alt_count_raw > 10 & alt_bias & MuTect2 == 0) |
maf[, `:=` (Custom_filters = NULL)]
# Tag and whitelist hotspots --------------------------------------------------------------------------------------
-maf = hotspot_annotate_maf(maf)
+maf = hotspot_annotate_maf(maf,oncokbbaseurl=oncokb_url)
maf = as.data.table(maf) # necessary because of the class of output from previous call
maf[Hotspot == TRUE & t_var_freq >= 0.02 & FILTER == 'low_vaf', FILTER := 'PASS'] # note: variants flagged by other filters will not be rescued by this
maf[Hotspot == TRUE & FILTER == 'low_mapping_quality', FILTER := 'PASS']
diff --git a/docs/.vuepress/config.yml b/docs/.vuepress/config.yml
index d613d476..e35e2347 100644
--- a/docs/.vuepress/config.yml
+++ b/docs/.vuepress/config.yml
@@ -3,7 +3,7 @@ description: CCS Research Pipeline for Whole-Genome and Whole-Exome Sequencing
themeConfig:
logo: /allegro.jpg
sidebar:
- - "/"
+ - ['', 'Home']
- title: Setup
collapsable: false
children:
diff --git a/docs/README.md b/docs/README.md
deleted file mode 100644
index 6a2c1e8e..00000000
--- a/docs/README.md
+++ /dev/null
@@ -1,102 +0,0 @@
----
-home: true
-heroImage: /tempoLogo.jpg
-actionText: Get Started →
-actionLink: /installation/
-features:
-- title: Reproducible Workflows
- details: Containerized workflows with Docker and Singularity
-- title: Portable
- details: Tailored for LSF and AWS
-- title: User-Friendly
- details: Written to be quickly run and used by anyone in the CMO
-footer: MIT Licensed | Copyright © 2019-present
----
-
-# Time-Efficient Mutational Profiling in Oncology (Tempo)
-
-Tempo is a computational pipeline for processing data of paired-end whole-exome (WES) and whole-genome sequencing (WGS) of human cancer samples with matched normals. Its components are containerized and the pipeline runs on the [Juno high-performance computing cluster](http://mskcchpc.org/display/CLUS/Juno+Cluster+Guide) at Memorial Sloan Kettering Cancer Center and on [Amazon Web Services (AWS)](https://aws.amazon.com). The pipeline was written by members of the [Center for Molecular Oncology](https://www.mskcc.org/research-programs/molecular-oncology).
-
-These pages contain instructions on how to run the Tempo pipeline. It also contains documentation on the bioinformatic components in the pipeline, some motivation for various parameter choices, plus an outline describing the reference resources used.
-
-If there are any questions or comments, you are welcome to [raise an issue](https://github.com/mskcc/tempo/issues/new?title=[User%20question]).
-
-Note: Tempo currently only supports human samples. The pipeline has only been tested for exome and genome sequencing experiments, and all reference files are in build GRCh37 of the human genome.
-
----
-
-## Table of Contents
-
-### 1. Getting Started
-
-#### 1.1. Setup
-* [Installation](installation.md)
-* [Setup on Juno](juno-setup.md)
-* [Setup on AWS](aws-setup.md)
-
-#### 1.2. Usage
-* [Running the Pipeline](running-the-pipeline.md)
- * [Overview](running-the-pipeline.md#overview)
- * [Input Files](running-the-pipeline.md#input-files)
- * [Execution Mode](running-the-pipeline.md#execution-mode)
- * [Modifying or Resuming Pipeline Run](running-the-pipeline.md#modifying-or-resuming-pipeline-run)
- * [After Successful Run](running-the-pipeline.md#after-successful-run)
-* [Nextflow Basics](nextflow-basics.md)
-* [Working With Containers](working-with-containers.md)
-
-#### 1.3 Outputs
-* [BAM Files](outputs.md#bam-files)
-* [QC Outputs](outputs.md#qc-outputs)
-* [Somatic Data](outputs.md#somatic-data)
-* [Germline Data](outputs.md#germline-data)
-* [Cohort Level Outputs](outputs.md#cohort-level-outputs)
-
-### 2. Pipeline contents
-
-#### 2.1. Bioinformatic Components
-* [Read Alignment](bioinformatic-components.md#read-alignment)
-* [Somatic Analyses](bioinformatic-components.md#somatic-analyses)
-* [Germline Analyses](bioinformatic-components.md#germline-analyses)
-* [Quality Control](bioinformatic-components.md#quality-control)
-
-#### 2.2. Reference Resources
-* [Genome Assembly](reference-files.md#genome-assembly)
-* [Genomic Intervals](reference-files.md#genomic-intervals)
-* [RepeatMasker and Mappability Blacklist](reference-files.md#repeatmasker-and-mappability-blacklist)
-* [Preferred Transcript Isoforms](reference-files.md#preferred-transcript-isoforms)
-* [Hotspot Annotation](reference-files.md#hotspot-annotation.md)
-* [OncoKB Annotation](reference-files.md#oncokb.md)
-* [gnomAD](gnomad.md)
-* [Panel of Normals for Exomes](wes-panel-of-normals.md)
-
-#### 2.3. Variant Annotation and Filtering
-* [Somatic SNVs and Indels](variant-annotation-and-filtering.md#somatic-snvs-and-indels)
-* [Germline SNVs and Indels](variant-annotation-and-filtering.md#germline-snvs-and-indels)
-* [Somatic and Germline SVs](variant-annotation-and-filtering.md#somatic-and-germline-svs)
-
-### 3. Help and Other Resources
-* [Troubleshooting](troubleshooting.md)
-* [AWS Glossary](aws-glossary.md)
-
-### 4. Contributing
-* [Contributing to Tempo](contributing-to-tempo.md)
-
-### 5. Acknowledgements
-* [Acknowledgements](acknowledgements.md)
-
-
-## Pipeline Flowchart
-
-
-
-## Directed Acyclic Graph
-
-
-
-##
-
-
----
diff --git a/docs/README.md b/docs/README.md
new file mode 120000
index 00000000..32d46ee8
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1 @@
+../README.md
\ No newline at end of file
diff --git a/docs/bioinformatic-components.md b/docs/bioinformatic-components.md
index a1f9d5ce..f0b36cb7 100644
--- a/docs/bioinformatic-components.md
+++ b/docs/bioinformatic-components.md
@@ -25,13 +25,16 @@ Tempo accepts as input sequencing reads from one or multiple FASTQ file pairs (c
## Somatic Analyses
* __SNVs and indels__ are called using [MuTect2](https://software.broadinstitute.org/gatk/documentation/tooldocs/4.beta.4/org_broadinstitute_hellbender_tools_walkers_mutect_Mutect2.php) and [Strelka2](https://github.com/Illumina/strelka). Subsequently, they are combined, annotated and filtered as described [in the section on variant annotation and filtering](variant-annotation-and-filtering.md#somatic-snvs-and-indels).
-* __Structural variants__ are detected by [Delly](https://github.com/dellytools/delly) and [Manta](https://github.com/Illumina/manta) then combined, filtered and annotated as described [in the section on variant annotation and filtering](variant-annotation-and-filtering.md#somatic-and-germline-svs).
+* __Structural variants__ are detected by multiple callers and then merged, filtered and annotated as described [in the section on variant annotation and filtering](variant-annotation-and-filtering.md#somatic-and-germline-svs). Somatic and germline variants are generated with [Delly](https://github.com/dellytools/delly), [Manta](https://github.com/Illumina/manta) and [SvABA](https://github.com/walaj/svaba). Somatic whole genome variants are generated with the same callers, along with [BRASS](https://github.com/cancerit/BRASS).
* __Copy-number analysis__ is performed with [FACETS](https://github.com/mskcc/facets) and processed using [facets-suite](https://github.com/mskcc/facets-suite). Locus-specific copy-number, purity and ploidy estimates are integrated with the SNV/indel calls to perform clonality and zygosity analyses.
* __Microsatellite instability__ is detected using [MSIsensor](https://github.com/ding-lab/msisensor).
* __HLA genotyping__ is performed with [POLYSOLVER](https://software.broadinstitute.org/cancer/cga/polysolver).
* __LOH at HLA loci__ is assessed with [LOHHLA](https://github.com/mskcc/lohhla).
* __Mutational signatures__ are inferred with [https://github.com/mskcc/tempoSig](https://github.com/mskcc/tempoSig).
* __Neoantigen prediction__ using estimates of class I MHC binding affinity is performed with [NetMHC 4.0](https://www.ncbi.nlm.nih.gov/pubmed/28978689) and integrated into the set of SNV/indel calls using [https://github.com/taylor-lab/neoantigen-dev](https://github.com/taylor-lab/neoantigen-dev) (_Note: this repository is currently private_).
+* __Clonality using Structural Variants__ is assessed using [SVclone](https://github.com/mcmero/SVclone). Joint calling is performed on both SNPs and structural variants, providing an alternative to the SNP clonality inference offered by Facets, while also assigning structural variants in the same clonal structure.
+* __Structural variant signatures__ are inferred with [signature.tools.lib](https://github.com/Nik-Zainal-Group/signature.tools.lib)
+* __Homologous recombination deficiency__ is assessed using [HRDetect from the signature.tools.lib package](https://github.com/Nik-Zainal-Group/signature.tools.lib)
## Germline Analyses
diff --git a/docs/dag.png b/docs/dag.png
index 2863b8c4..8fb4d365 100644
Binary files a/docs/dag.png and b/docs/dag.png differ
diff --git a/docs/juno-setup.md b/docs/juno-setup.md
index 3d3aa745..a1b07a20 100644
--- a/docs/juno-setup.md
+++ b/docs/juno-setup.md
@@ -37,26 +37,27 @@ The command `which singularity` should return `/opt/local/singularity/3.1.1/bin/
## Java Version
-Nextflow requires Java version 8 or later. On Juno, you can load it using `module`:
+Nextflow requires Java version 11 or later. On Juno, you can load it using `module`:
```shell
-module load java/jdk1.8.0_202
+module load java/jdk-11.0.11
```
or put it in your `PATH` by inserting this into your bash profile:
```shell
-export JAVA_HOME=/opt/common/CentOS_7/java/jdk1.8.0_202/
+export JAVA_HOME=/opt/common/CentOS_7/java/jdk-11.0.11/
export PATH=$JAVA_HOME/bin:$PATH
```
-The call `which java` should return `/opt/common/CentOS_7/java/jdk1.8.0_202/bin/java` if you have done this correctly.
+The call `which java` should return `/opt/common/CentOS_7/java/jdk-11.0.11/bin/java` if you have done this correctly.
## Test Your Environment
You can run a the pipeline on small test files to ensure that you are ready to run real data. If you experience any issues, something in your environment might be the reason. The following should take approximately 30 minutes and all tasks should succeed at first attempt:
```shell
-nextflow run pipeline.nf \
- --mapping test_inputs/local/full_test_mapping.tsv \
+nextflow run dsl2.nf \
+ --mapping test_inputs/local/full_test_mapping.tsv \
--pairing test_inputs/local/full_test_pairing.tsv \
-profile test_singularity \
--outDir results
- --somatic --germline --QC --aggregate
+ --aggregate
+ --workflows="SNV,qc,lohhla" \
```
diff --git a/docs/nextflow-basics.md b/docs/nextflow-basics.md
index 5dd5d8c6..0c36ad19 100644
--- a/docs/nextflow-basics.md
+++ b/docs/nextflow-basics.md
@@ -12,4 +12,4 @@
* __View intermediate output__: As the pipeline runs, everything needed to execute each process in the pipeline is located in the `work` in the run directory. Thus, you can peek at input and output files for each step of the pipeline in real time.
-* __Run or skip specific tools:__ `pipeline.nf` has the argument `--tools`, which allows users to run only certain bioinformatic tools and skip others. The `--somatic` and `--germline` flags already have a preset of tools to include during a run, but you can limit this further by providing the `--tools` flag, followed by a comma-delimited string. For example, to use only DELLY for your somatic/germline runs, do `--somatic --germline --tools delly`; to use MuTect2, Manta, and Strelka2, do `--somatic --tools mutect2,manta,strelka2`.
+* __Run or skip specific subworkflows:__ `dsl2.nf` has the argument `--workflows`, which allows users to run only certain bioinformatic subworkflows and skip others. By default, nextflow uses the following string to run all subworkflows: `'snv,sv,mutsig,lohhla,facets,qc,msisensor'`. You can limit the subworkflows by providing a comma-delimited string to `--workflows` containing only the workflows you want.
diff --git a/docs/outputs.md b/docs/outputs.md
index fe3dc1d7..949b4854 100644
--- a/docs/outputs.md
+++ b/docs/outputs.md
@@ -22,7 +22,8 @@ outDir/bams/
│  ├── collecthsmetrics
│  ├── fastp
│  ├── multiqc
-│  └── pileup
+│  ├── pileup
+│  └── qualimap
├── DU874145-T
│  ├── DU874145-T.bam
│  └── DU874145-T.bam.bai
@@ -35,6 +36,7 @@ These outputs are:
- `collectshsmetrics`: For exomes, per-sample hybridisation-selection metrics in the.
- `pileup`: Per tumor-normal-pair, the Conpair-generated SNP pileup files.
- `multiqc`: A summary report of FASTQ/BAM QC metrics from Picard, fastp and other tools.
+- `qualimap`: Per-sample BAM file alignment metrics in text and html files.
## Somatic data
@@ -43,41 +45,58 @@ The result of the somatic analyses is output in summarized forms in the `somatic
```shell
outDir/somatic
├── DU874145-T__DU874145-N
+│  ├── brass
│  ├── combined_mutations
│  ├── combined_svs
│  ├── conpair
│  ├── delly
│  ├── facets
+│  ├── hrdetect
│  ├── lohhla
│  ├── manta
│  ├── meta_data
│ ├── multiqc
│  ├── mutect2
│  ├── neoantigen
-│  └── strelka2
+│  ├── strelka2
+│  ├── svaba
+| └── svclone
└── DU874146-T__DU874146-N
+ ├── brass
├── combined_mutations
├── combined_svs
+ ├── conpair
├── delly
├── facets
+ ├── hrdetect
├── lohhla
├── manta
├── meta_data
├── multiqc
├── mutect2
├── neoantigen
- └── strelka2
+ ├── strelka2
+ ├── svaba
+ └── svclone
```
These outputs are:
+- `brass`: BRASS SV caller output. WGS only.
- `combined_mutatations`: unfiltered and final filtered maf per tumor-normal pair.
- `*.somatic.unfiltered.maf`: Unfiltered mutations `generated in the SomaticAnnotateMaf`.
- `*.somatic.final.maf`: Filtered mutations from MuTect2 and Strelka2, annotated with mutational effects, neoantigen predictions, and zygosity, as [described elsewhere](variant-annotation-and-filtering.md#somatic-snvs-and-indels).
- - `intermidiate_files/*`: 3 intermidiate vcf files contains all mutations before any filter after mutect and strelka, mutations after `filter-vcf.py`, and mutations after bcftools filter by `FILTER=PASS`.
-- `combined_svs`: Combined Delly and Manta SV calls.
+ - `intermediate_files/*`: 3 intermediate vcf files contains all mutations before any filter after mutect and strelka, mutations after `filter-vcf.py`, and mutations after bcftools filter by `FILTER=PASS`.
+- `combined_svs`: Combined BRASS (WGS only), Delly, Manta and SvABA SV calls.
+ - `*.unfiltered.bedpe`: Unfiltered combined somatic SVs.
+ - `*.final.bedpe`: Filtered combined somatic SVs.
+ - `intermediate_files/*`: 3 intermediate combined SV files:
+ - `*.merged.raw.vcf.gz`: Raw output of `mergesvvcf`.
+ - `*.merged.vcf.gz`: Reformatted output of `mergesvvcf` where events with number of PASSing callers lower than minimum are filtered out.
+ - `*.combined.bedpe`: Merged vcf calls converted to bedpe using `svtools vcftobedpe`.
- `conpair`: Per tumor-normal-pair, the Conpair-generated concordance and contamination files.
- `delly`: Delly output.
- `facets`: Individual copy-number profiles from FACETS, per tumor-normal pair.
+- `hrdetect`: Detection of BRCA1/BRCA2-deficiency. WGS only.
- `lohhla`: LOHHLA output.
- `manta`: Manta output.
- `meta_data`: Summarized meta_data file which includes the following results:
@@ -91,7 +110,9 @@ These outputs are:
- `multiqc`: A summary report of tumor/normal pair QC metrics from Conpair and Facets.
- `mutect2`: Manta output.
- `neoantigens`: Neoantigen predictions from NetMHCpan per sample.
-- `strelka2`: Manta output.
+- `strelka2`: Strelka2 output.
+- `svaba`: SvABA SV caller output.
+- `svclone`: clustering output of structural variants. WGS only.
::: warning Be aware
* LOHHLA is temporarily disabled due to a bug need future investigation. It will be enabled again in the future release.
@@ -109,25 +130,34 @@ outDir/germline/
│  ├── delly
│  ├── haplotypecaller
│  ├── manta
-│  └── strelka2
+│  ├── strelka2
+│  └── svaba
└── DU874146-N
├── combined_mutations
├── combined_svs
├── delly
├── haplotypecaller
├── manta
- └── strelka2
+ ├── strelka2
+ └── svaba
```
These outputs are:
- `combined_mutatations`: unfiltered and final filtered maf per tumor-normal pair.
- - `*.germline.unfiltered.maf`: Unfiltered mutations `generated in the GermlineAnnotateMaf`.
+ - `*.germline.unfiltered.maf`: Unfiltered mutations generated in the `GermlineAnnotateMaf`.
- `*.germline.final.maf`: Filtered mutations from HaplotypeCaller and Strelka2, annotated with mutational effects and zygosity, as [described elsewhere](variant-annotation-and-filtering.md#germline-snvs-and-indels).
- - `intermidiate_files/*`: 3 intermidiate vcf files contains all mutations before any filter after mutect and strelka, mutations after bcftools filter by `FILTER=PASS`, and gnomAD filter.
-- `combined_svs`: Combined Delly and Manta SV calls.
+ - `intermediate_files/*`: 3 intermediate vcf files contains all mutations before any filter after mutect and strelka, mutations after bcftools filter by `FILTER=PASS`, and gnomAD filter.
+- `combined_svs`: Combined Delly, Manta and SvABA SV calls per normal sample.
+ - `*.unfiltered.bedpe`: Unfiltered germline SVs from combined Delly, Manta and SvABA SV calls.
+ - `*.final.bedpe`: Filtered germline SVs from combined Delly, Manta and SvABA SV calls
+ - `intermediate_files/*`: 3 intermediate combined SV files:
+ - `*.merged.raw.vcf.gz`: Raw output of `mergesvvcf`.
+ - `*.merged.vcf.gz`: Reformatted output of `mergesvvcf` where events with number of PASSing callers lower than minimum are filtered out.
+ - `*.combined.bedpe`: Merged vcf calls converted to bedpe using `svtools vcftobedpe`.
- `delly`: Delly output.
- `manta`: Manta output.
-- `strelka2`: Manta output.
+- `strelka2`: Strelka2 output.
+- `svaba`: SvABA SV caller output.
## Cohort Level Outputs
@@ -146,16 +176,19 @@ outDir/cohort_level/
│  ├── contamination_qc.txt
│  ├── DNA.IntegerCPN_CI.txt
│  ├── HLAlossPrediction_CI.txt
+│  ├── hrdetect.tsv
│  ├── multiqc_report.html
│  ├── multiqc_data.zip
│  ├── mut_germline.maf
│  ├── mut_somatic.maf
│  ├── mut_somatic_neoantigens.txt
│  ├── sample_data.txt
-│  ├── sv_germline.vcf.gz
-│  ├── sv_germline.vcf.gz.tbi
-│  ├── sv_somatic.vcf.gz
-│  └── sv_somatic.vcf.gz.tbi
+│  ├── svclone_sv_cluster_certainty.tsv
+│  ├── svclone_snv_cluster_certainty.tsv
+│  ├── sv_catalogues.pdf
+│  ├── sv_exposures.tsv
+│  ├── sv_germline.bedpe
+│  └── sv_somatic.bedpe
├── cohort2
│  ├── alignment_qc.txt
│  ├── cna_armlevel.txt
diff --git a/docs/reference-files.md b/docs/reference-files.md
index aafa7e5f..54110d49 100644
--- a/docs/reference-files.md
+++ b/docs/reference-files.md
@@ -1,6 +1,6 @@
# Reference Files
-This and associated pages in this section provide details on the provenance and generation of all reference files used in `pipeline.nf`. Usage of these files is defined in the [references configuration file](https://github.com/mskcc/tempo/blob/master/conf/references.config).
+This and associated pages in this section provide details on the provenance and generation of all reference files used in `dsl2.nf`. Usage of these files is defined in the [references configuration file](https://github.com/mskcc/tempo/blob/master/conf/references.config).
::: tip Note
All reference files described herein are in assembly GRCh37/hg19 of the human genome.
@@ -121,6 +121,7 @@ Functional mutation effects and predicted oncogenicity of variants, as well as l
Annotation of germline variants in _BRCA1_ and _BRCA2_ is carried out with the [annotateMaf package](https://github.com/taylor-lab/annotateMaf). This includes variant-level annotation from the ENIGMA consortium and ClinVar.
## Structural Variant Calling
+
Delly provides and takes as an argument a [file of regions](https://github.com/dellytools/delly/tree/master/excludeTemplates) to _exclude_ from variant calling. This excludes telomeres and centromeres from auto- and allosomes as well as any other contig.
For Manta, subtract these regions from a bed file of the whole genome to generate a list of regions to _include_. First clean up the file provided by Delly, since it is not in `bed` format:
@@ -128,3 +129,25 @@ For Manta, subtract these regions from a bed file of the whole genome to generat
grep -Ev "chr|MT|GL00|NC|hs37d5" human.hg19.excl.tsv > human.hg19.excl.clean.bed
bedtools subtract -a b37.bed -b human.hg19.excl.clean.bed > b37.minusDellyExclude.bed
```
+
+For BRASS, Tempo is using a pre-built reference packages linked to [Sanger's dockstore registry](ftp://ftp.sanger.ac.uk/pub/cancer/dockstore/human/). To download references for GRCh37:
+``` shell
+wget ftp://ftp.sanger.ac.uk/pub/cancer/dockstore/human/VAGrENT_ref_GRCh37d5_ensembl_75.tar.gz
+tar -xzvf VAGrENT_ref_GRCh37d5_ensembl_75.tar.gz
+wget ftp://ftp.sanger.ac.uk/pub/cancer/dockstore/human/CNV_SV_ref_GRCh37d5_brass6+.tar.gz
+tar -xzvf CNV_SV_ref_GRCh37d5_brass6+.tar.gz
+# grab CNV_SV_ref/brass and VAGrENT_ref_GRCh37d5_ensembl_75/vagrent
+```
+GRCh38 is also available from the same ftp server. To build a new reference, follow the instructions on the [BRASS wiki](https://github.com/cancerit/BRASS/wiki).
+
+## Structural Variant Annotation
+
+The bed and bedpe files used for the flags `pcawg_blacklist_bed`,`pcawg_blacklist_bedpe`, `pcawg_blacklist_fb_bedpe` and `pcawg_blacklist_te_bedpe` are sourced from the [SV merging tool used in the PCAWG paper](https://bitbucket.org/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/). They can be downloaded as follows:
+``` shell
+wget https://api.bitbucket.org/2.0/repositories/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/pcawg6_blacklist.slop.bed.gz
+wget https://api.bitbucket.org/2.0/repositories/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/pcawg6_blacklist.slop.bedpe.gz
+wget https://api.bitbucket.org/2.0/repositories/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/pcawg6_blacklist_foldback_artefacts.slop.bedpe.gz
+wget https://api.bitbucket.org/2.0/repositories/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/pcawg6_blacklist_TE_pseudogene.bedpe.gz
+```
+
+Structural variants are also filtered with RepeatMasker and Mappability blacklists, whose preparation are described in [one of the above sections](#repeatMasker-and-mappability-blacklist).
diff --git a/docs/running-the-pipeline.md b/docs/running-the-pipeline.md
index e162d5c7..f2b48df7 100644
--- a/docs/running-the-pipeline.md
+++ b/docs/running-the-pipeline.md
@@ -6,16 +6,15 @@
Tempo does not support running samples from mixed sequencing platforms together. By default, the pipeline assumes the inputs are from exome sequencing.
:::
-This page provides instructions on how to run the pipeline through the `pipeline.nf` script. The basic command below shows how to run Tempo, with an explanation of flags and input arguments and files. Below is also described how to best [run the pipeline on Juno](running-the-pipeline.md#running-the-pipeline-on-juno) as well as [on AWS](running-the-pipeline.md#running-the-pipeline-on-aws).
+This page provides instructions on how to run the pipeline through the `dsl2.nf` script. The basic command below shows how to run Tempo, with an explanation of flags and input arguments and files. Below is also described how to best [run the pipeline on Juno](running-the-pipeline.md#running-the-pipeline-on-juno) as well as [on AWS](running-the-pipeline.md#running-the-pipeline-on-aws).
+
```shell
-nextflow run pipeline.nf \
- --mapping/--bamMapping \
- --outDir \
+nextflow run dsl2.nf \
+ --mapping/--bamMapping
```
@@ -24,16 +23,17 @@ _Note: [The number of dashes matters](nextflow-basics.md)._
**Required arguments:**
* `--mapping/--bamMapping ` is required except running in `--aggregate [tsv]` mode. When `--mapping [tsv]` is provided, FASTQ file paths are expected in the TSV file, and the pipeline will start from FASTQ files and go through all steps to generate BAM files. When `--bamMapping [tsv]` if provided, BAM file paths are expected in the TSV file. See [The Mapping File](running-the-pipeline.md#input-files) and [Execution Mode](running-the-pipeline.md#execution-mode) for details.
* `--pairing ` is required when `--somatic` and/or `--germline` are enabled. `--pairing ` is not needed when you are running BAM generation part only, even if you are doign it with `--QC`( or `--QC` and `--aggregate`) enabled. See [The Mapping File](running-the-pipeline.md#input-files) and [Execution Mode](running-the-pipeline.md#execution-mode) for details.
-* `--assayType` ensures appropriate resources are allocated for indicated assay type. Only `exome` or `genome` is supported. Note: Please also make sure this value matches the `TARGET` field you put in the mapping.tsv file. Available TARGET field value for `exome` are `idt` or `agilent`i (can be mixed), for `genome` is `wgs`.
-* `--outDir` is the directory where the output will end up. This directory does not need to exist. If not set, by default it will be set to run directory (i.e. the directory from which the command `nextflow run` is executed.)
* `-profile` loads the preset configuration required to run the pipeline in the supported environment. Accepted values are `juno` and `awsbatch` for execution on the [Juno cluster](juno-setup.md) or on [AWS Batch](aws-setup.md), respectively. `-profile test_singularity` is for testing on `juno`.
+* `--assayType` ensures appropriate resources are allocated for indicated assay type. Only `exome` or `genome` is supported. Default is exome. Note: Please also make sure this value matches the `TARGET` field you put in the mapping.tsv file. Available TARGET field value for `exome` are `idt` or `agilent`i (can be mixed), for `genome` is `wgs`.
+* `--workflows` inidicates which [sub-workflows](sub-workflows.md) should be executed for this run. Possible options are `snv`, `sv`, `mutsig`, `germSNV`, `germSV`, `lohhla`, `facets`,`qc`, and `msisensor`. Multiple arguments can be provided in quotation marks (i.e. `--workflows="snv,qc"`).
**Section arguments:**
-* `--somatic`, `--germline` and `--QC` flags are boolean that indicate to run the somatic, germline variant calling and QC modules, respectively. Default value are `false` for all. Note: Currently the pipeline will enable `--somatic` automatically if only `--germline` is enabled, since germline analysis needs results from somatic analysis for now. (default: `false` for all)
-* `--aggregate ` can be boolean or be given a path to a tsv file. Default value is `false`. It has to work together with `--somatic`, `--germline` and/or `--QC` to aggregate the results of these operations together as a cohort. There will be a `cohort_level/[cohort]/` directory generated under `--outDir [path]`. When boolean value `true` is given (equal to only give `--aggregate`), TEMPO will aggregate all the samples in the mapping and pairing file as one cohort named "default cohort". When `--aggregate ` file is given, the pipeline will aggregate samples and tumor/normal pairs based on the value is given in `COHORT` columns. Each sample and tumor/normal pairs can be assigned to different cohorts in different rows.
+* `--workflows` can be run independently as needed, however, when the output of a sub-workflow is required to as a dependency for an indicated workflow provided via the `--workflows` argument, the necessary dependent workflows will be automatically enabled. Note that while sub-workflows can be run independently, specific processes must be run as part of as sub-workflow. See the [sub-workflows](sub-workflows.md) section for more details.
+* `--aggregate ` can be boolean or be given a path to a tsv file. Default value is `false`. A `cohort_level/[cohort]/` directory generated under `--outDir [path]`. When boolean value `true` is given (equal to only give `--aggregate`), TEMPO will aggregate all the samples in the mapping and pairing file as one cohort named "default cohort". When `--aggregate ` file is given, the pipeline will aggregate samples and tumor/normal pairs based on the value is given in `COHORT` columns. Each sample and tumor/normal pairs can be assigned to different cohorts in different rows.
**Optional arguments:**
+* `--outDir` is the directory where the output will end up. This directory does not need to exist. If not set, by default it will be set to run directory (i.e. the directory from which the command `nextflow run` is executed.)
* `-work-dir`/`-w` is the directory where the temporary output will be cached. By default, this is set to the run directory. Please see `NXF_WORK` in [Nextflow environment variables](https://www.nextflow.io/docs/latest/config.html#environment-variables).
* `-publishAll` is a boolean, resulting in retention of intermediate output files ((default: `true`).
* `--splitLanes` indicates that the provided FASTQ files will be scanned for all unique sequencing lanes and demultiplexed accordingly. This is recommended for some steps of the alignment pipeline. See more under [The Mapping File](running-the-pipeline.md#input-files) (default: `true`).
@@ -46,12 +46,12 @@ _Note: [The number of dashes matters](nextflow-basics.md)._
Using test inputs provided in the GitHub repository, here is a concrete example:
```shell
-nextflow run pipeline.nf \
- --mapping test_inputs/local/full_test_mapping.tsv \
- --pairing test_inputs/local/full_test_pairing.tsv \
+nextflow run dsl2.nf \
-profile juno \
- --outDir results
- --somatic --germline --aggregate --QC \
+ --mapping test_inputs/local/full_test_mapping.tsv \
+ --pairing test_inputs/local/full_test_pairing.tsv
+ --workflows="SNV,qc,lohhla" \
+ --aggregate true
```
## Input Files
@@ -81,9 +81,9 @@ Tempo checks for the following aspects:
For processing paired-end FASTQ inputs, users must provide both a mapping file using `--mapping `, as described below.
-You do not need `--pairing ` when you are running BAM generation alone (even together with `--QC`).
+You do not need `--pairing ` when you are running BAM generation alone.
-You must to give `--pariring ` when `--somatic` or `--germline` is enabled.
+You must to give `--pariring ` when running any other sub-workflow.
::: warning Be aware
Tempo can deal with any number of sequencing lanes per sample, in any combination of lanes split or combined across multiple FASTQ pairs. Different FASTQ pairs for the same sample can be provided as different lines in the mapping file and give the same SAMPLE ID in the `SAMPLE` field, and repeating the `TARGET` field. By default, Tempo will look for all distinct sequencing lanes in provided FASTQ files by scanning each FASTQ read name. The pipeline uses this and the instrument, run, and flowcell IDs from the _sequence identifiers_ in the input FASTQs to generate all different read group IDs for each sample. This information is used by the base quality score recalibration steps of the GATK suite of tools. If FASTQ files name explicitly specified the lane name in the format of `_L(\d){3}_` ("\_L" + "3 integer" + "\_"), the pipeline will assume this FASTQ files contain only one lane, and it will skip scanning and splitting the FASTQ files, and give one read group ID all the reads in the FASTQ files based on the name of the first read in the FASTQ file. Please refer to [this GATK Forum Page](https://gatkforums.broadinstitute.org/gatk/discussion/6472/read-groups)for more details.
@@ -109,7 +109,7 @@ Read further details on these parameters [here](reference-files.md#genomic-inter
If the user is using pre-processed BAMs, the input TSV file is a similar format as FASTQ mapping TSV file, with slight difference showing below.
-You must to give `--pariring ` and at least one of `--somatic` `--germline` `--QC` simultaneously when you use `--bamMapping `.
+You must to give `--pariring ` and specify at least one [sub-workflow](sub-workflow.md) when beginning with BAM mapping files.
Example:
@@ -123,7 +123,7 @@ Example:
The `--pairing ` file will be exactly the same as using FASTQ mapping TSV file, describing below.
::: tip Note
-The pipeline expects BAM file indices in the same subdirectories as `TUMOR_BAM` and `NORMAL_BAM`. If the index files `*.bai` or `*.bam.bai` do not exist, `pipeline.nf` will throw an error. The BAI column in the BAM Mapping TSV file is not actually used.
+The pipeline expects BAM file indices in the same subdirectories as `TUMOR_BAM` and `NORMAL_BAM`. If the index files `*.bai` or `*.bam.bai` do not exist, `dsl2.nf` will throw an error. The BAI column in the BAM Mapping TSV file is not actually used.
Different from FASTQ mapping tsv, in this TSV file each SAMPLE id can only appear once, meaning the pipeline will not combine different BAMs for you for the same sample.
:::
@@ -132,7 +132,7 @@ Different from FASTQ mapping tsv, in this TSV file each SAMPLE id can only appea
The pipeline needs to know which tumor and normal samples are to be analyzed as matched pairs. This file provides that pairing by referring to the sample names as provided in the `SAMPLE` column in the mapping file.
-You do not need `--pairing ` when you are running BAM generation alone (even together with `--QC`).
+You do not need `--pairing ` when you are running only the alignment sub-workflow.
Example:
@@ -147,7 +147,7 @@ Example:
* When boolean value `true` is given (equal to only give `--aggregate`), TEMPO will aggregate all the samples in the mapping and pairing file as one cohort named "default cohort".
* When `--aggregate ` file is given, the pipeline will aggregate samples and tumor/normal pairs based on the value is given in `COHORT` column. Each sample and tumor/normal pairs can be assigned to different cohorts in different rows.
-* When running aggregation only mode, `PATH` column need to be provided to introduce the TEMPO result directories for each sample and tumor/normal pairs (only up to the parent folder of `qc`, `somatic` and `germline` folder).
+* When running aggregation only mode, the `PATH` column needs to be provided to introduce the TEMPO result directories for each sample and tumor/normal pairs (only up to a TEMPO produced output folder).
Example:
@@ -161,58 +161,28 @@ Example:
The `--pairing ` file will be exactly the same as using FASTQ mapping TSV file, describing below.
::: tip Note
-The pipeline expects BAM file indices in the same subdirectories as `TUMOR_BAM` and `NORMAL_BAM`. If the index files `*.bai` or `*.bam.bai` do not exist, `pipeline.nf` will throw an error. The BAI column in the BAM Mapping TSV file is not actually used.
+The pipeline expects BAM file indices in the same subdirectories as `TUMOR_BAM` and `NORMAL_BAM`. If the index files `*.bai` or `*.bam.bai` do not exist, `dsl2.nf` will throw an error. The BAI column in the BAM Mapping TSV file is not actually used.
Different from FASTQ mapping tsv, in this TSV file each SAMPLE id can only appear once, meaning the pipeline will not combine different BAMs for you for the same sample.
:::
## Execution Mode
-
-There are overall 4 execution modes that TEMPO accept, depending on the way of input arguments are given. The pipeline will throw an error when it detects incompatible _Section arguments_ (`--somatic`, `--germline`, `--QC`, `--aggregate`) combinations. Compatibility of analysis argument combinations are described below:
+There are a variety of execution modes that can be executed by TEMPO. Specific details can be found in the [sub-workflows](sub-workflows.md) section. Additionally, there are special cases under which TEMPO can run.
### `--mapping ` only
-When no additional _Section arguments_ are given, only alignment steps will be performed.
-
-***Compatible _Section Arguments_ Combinations:***
-* `--QC`: `QcAlfred` and `QcCollectHsMetrics` will be performed.
-* `--QC --aggregate true/`: `QcAlfred` and `QcCollectHsMetrics` will be performed and aggregated in `cohort_level/[cohort]` folder.
-
-***Incompatible _Section Arguments_:***
-* `--somatic`: No pairing infomation.
-* `--germline`: No pairing information.
-* `--bamMapping `: Conflicts.
+When no additional sub-workflow arguments are given, only alignment steps will be performed.
### `--mapping/--bamMapping ` and `--pairing ` (We are describing two modes in this section)
When `--mapping ` is given, the pipeline will use FASTQ input and start from alignment steps.
When `--bamMapping ` is given, the pipeline will use BAM input and skip alignment steps.
-When no additional _Sectionarguments_ are given, pipeline will throw an error indicating that `--pairing ` is not used.
-
-***Compatible _Section Arguments_ Combinations:***
-* `--somatic` with or without `--aggregate true/`
-* `--somatic --germline` with or without `--aggregate true/`
-* `--somatic --QC` with or without `--aggregate true/`
-* `--QC` with or without `--aggregate true/`: `QcConpair` will be performed together with `QcAlfred` and `QcCollectHsMetrics`.
-
-***Incompatible _Section Arguments_:***
-* `--germline` with or without `--aggregate true/` and `--QC`: The pipeline will auto-enable `--somatic` because germline analysis need the results from somatic analysis at this stage.
-* `--aggregate `: Conflicts.
+When no additional sub-workflow arguments are given, the pipeline will throw an error indicating that `--pairing ` is not used.
### `--aggregate ` only
-This mode can only be run when the TEMPO produced output structure path is provided as `PATH` column in `--aggregate `. It explicitly relies on the output structure ((only up to the parent folder of `qc`, `somatic` and `germline` folder) that are auto-generated by TEMPO to identify how and what files need to be aggregated together as a cohort level result under folder `cohort_level/[cohort]`. Please refer to [Outputs](outputs.md#outputs) for more detail.
-
+This mode can only be run when the TEMPO produced output structure path is provided as `PATH` column in `--aggregate `. It explicitly relies on the output structure (only up to the parent folder of of a TEMPO generated output directory) that are auto-generated by TEMPO to identify how and what files need to be aggregated together as a cohort level result under folder `cohort_level/[cohort]`. Please refer to [Outputs](outputs.md#outputs) for more detail.
-***Compatible _Section Arguments_ Combinations:***
-* The pipeline will auto-detect which sections need to be aggregated, so no _Section_Arguments_ need to be given.
-
-***Incompatible _Section Arguments_:***
-* `--somatic`: Not needed. Auto-detected.
-* `--germline`: Not needed. Auto-detected.
-* `--QC`: Not needed. Auto-detected.
-* `--mapping `: Conflicts.
-* `--bamMapping `: Conflicts.
-* `--pairing `: Conflicts.
+When using this mode, no sub-workflow arguments need to be given.
## Running the Pipeline on Juno
@@ -222,34 +192,40 @@ First follow the instructions to [set up your enviroment on Juno](juno-setup.md)
### Submitting the Pipeline to LSF
-We recommend submitting your `nextflow run pipeline.nf <...>` command to the cluster via `bsub`, which will launch a leader job from which individual processes are submitted as jobs to the cluster.
+We recommend submitting your `nextflow run dsl2.nf <...>` command to the cluster via `bsub`, which will launch a leader job from which individual processes are submitted as jobs to the cluster.
```
bsub -W -n 2 -R "rusage[mem=]" \
-o .out -e .err \
- nextflow run pipeline.nf -profile juno <...>
+ nextflow run dsl2.nf -profile juno <...>
```
We recommend that users check the [documentation for LSF](https://www.ibm.com/support/knowledgecenter/en/SSETD4_9.1.2/lsf_command_ref/bsub.1.html) to clarify each of the arguments above. However,
-* `-W ` sets the time allotted for `nextflow run pipeline.nf` to run to completion.
-* `-n 2` is requesting one slot. This should be sufficient for `nextflow run pipeline.nf`
+* `-W ` sets the time allotted for `nextflow run dsl2.nf` to run to completion.
+* `-n 2` is requesting one slot. This should be sufficient for `nextflow run dsl2.nf`
* ` -o .out` is the name of the STDOUT file, which is quite informative for Nextflow. We **strongly** encourage users to set this.
* ` -e .err` is the name of the STDERR file. Please set this.
-* ` -R "rusage[mem=]"` is the requested memory for `nextflow run pipeline.nf`, which will not be memory intensive at all.
+* ` -R "rusage[mem=]"` is the requested memory for `nextflow run dsl2.nf`, which will not be memory intensive at all.
Here is a concrete example of a bsub command to process 25 WES TN pairs, running somatic and germline variant calling modules:
```shell
-bsub -W 80:00 -n 2 -R "rusage[mem=8]" -o nf_output.out -e nf_output.err \
- nextflow run /pipeline.nf --somatic --germline \
- --mapping test_inputs/local/WES_25TN.tsv --pairing test_inputs/local/WES_25TN_pairing.tsv
+
+ bsub -W 80:00 -n 2 -R "rusage[mem=8]" \
+ -o nf_output.out \
+ -e nf_output.err \
+ nextflow run /dsl2.nf \
+ --mapping test_inputs/local/WES_25TN.tsv \
+ --pairing test_inputs/local/WES_25TN_pairing.tsv \
--outDir results \
- -profile juno
+ -profile juno \
+ --workflows="SNV,qc,lohhla" \
+ --aggregate true
```
::: warning Be aware
-Whereas a few exome samples finish within a few hours, larger batches and genomes will take .s. Allow for this by setting`-W` to a good amount of hours. The pipeline will die if the leader job does, but can be [resumed](running-the-pipelinf.md#modifying-or-resuming-pipeline-run) subsequently.
+Whereas a few exome samples finish within a few hours, larger batches and genomes will take .s. Allow for this by setting `-W` to a good amount of hours. The pipeline will die if the leader job does, but can be [resumed](running-the-pipelinf.md#modifying-or-resuming-pipeline-run) subsequently.
:::
### Running From a `screen` Session
@@ -274,7 +250,7 @@ Nextflow supports [modify and resume](https://www.nextflow.io/docs/latest/getsta
To resume an interrupted Nextflow pipeline run, add `-resume` (note the single dash) to your command-line call to access the cache history of Nextflow and continue a job from where it left off. This will trigger a check of which jobs already completed before starting unfinished jobs in the pipeline.
-This function also allows you to make changes to values in the `pipeline.nf` script and continue from where you left off. Nextflow will use the cached information from the unchanged sections while running only the modified processes. If you want to make changes to processes that already successfully completed, you have to manually delete the subdirectories in `work` where those processes where run.
+This function also allows you to make changes to values in the `dsl2.nf` script and continue from where you left off. Nextflow will use the cached information from the unchanged sections while running only the modified processes. If you want to make changes to processes that already successfully completed, you have to manually delete the subdirectories in `work` where those processes where run.
::: tip Note
* If you use `-resume` for the first time of a timeline run, Nextflow will recognize this as superfluous, and continue.
@@ -304,13 +280,13 @@ Users can then restart the pipeline at specific run, using either the `RUN NAME`
or equivalently
```shell
-> nextflow run naseq-nf -resume 4dc656d2-c410-44c8-bc32-7dd0ea87bebf
+> nextflow run rnaseq-nf -resume 4dc656d2-c410-44c8-bc32-7dd0ea87bebf
```
Sometimes the resume feature may not work entirely as expected, as described in troubleshooting tips [here on the Nextflow blog](https://www.nextflow.io/blog/2019/troubleshooting-nextflow-resume.html)
-## After Successful Run
+## After A Successful Run
Nextflow generate many intermediate output files. All the relevant output data should be in the directory given to the `outDir` argument. Once you have verified that the data are satisfactory, everything outside this directory can be removed. In particular, the `work` directory will contain all intermediate output files, which takes up a great deal of disk space and should be removed. The `nextflow clean -force` command does all of this. Also see `nextflow clean -help` for options.
diff --git a/docs/sub-workflows.md b/docs/sub-workflows.md
new file mode 100644
index 00000000..735d230e
--- /dev/null
+++ b/docs/sub-workflows.md
@@ -0,0 +1,26 @@
+# Sub-Workflows
+
+Control over which processes are performed in TEMPO is managed by user-defined sub-workflow specification. When executing a TEMPO run, the `--workflow` argument is provided, with any combination of sub-workflows indicated by the user.
+
+Multiple arguments can be provided in quotation marks, for example, to run the SNV and QC workflows, the following workflow argument could be specified as a comma delimited list.
+
+```
+(i.e. `--workflows="snv,qc"`)
+```
+
+Possible subworkflow arguments are: Possible options are
+* `snv`
+* `sv`
+* `mutsig`
+* `germSNV`
+* `germSV`
+* `lohhla`
+* `facets`
+* `qc`
+* `msisensor`
+
+## Sub-Workflow Processes & Dependencies
+Each sub-workflow consists of one or more DSL2 style nextflow modules. A sub-workflow acts as a wrapper that handles the input/output processing and module execution. A description of the processes executed by each sub-workflow are provided below.
+
+In some cases, a sub-workflow will require data that is generated by a different sub-workflow as a dependency. In these events, the required processes will be automatically activated by TEMPO. For example, output from the `facets` process is required for the `lohhla` sub-workflow to function. Specifying `workflow="lohhla"` will then automatically enable the facets processes to accomidate the dependency.
+
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index ef937618..fe4443ca 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -13,7 +13,7 @@ You can follow the Nextflow process by following what is printed to `stdout`. Ad
A job running a single process inside the pipeline can fail due to inadequate resources, which will trigger a re-run with increased resources. For other failures, you need to look inside the `work` directory. In the `.html` reports generated by default (or the `trace.txt` file) in the run directory you will find for each process run its status (`COMPLETED`; `CACHED` if resumed and completed in a prior run; and `FAILED` if an error occured) and a `hash`. The hash indicates the subdirectory in which the process was run (for example `a4/00365e` points to `work/a4/00365e9190eca55907746edeb58f77`). In this directory you find the following files which are useful for troubleshooting:
- `.command.run`: This is the actual script which sets environment variables and runs `.command.sh`, Nextflow submits this to LSF using bsub. You can manually resubmit it by running `bsub < .command.run`.
-- `.command.sh`: This contains the command-line calls that are defined in the corresponding process in `pipeline.nf`.
+- `.command.sh`: This contains the executable script generated by the process definition.
- `.command.log`: Contains `stdout` from the process itself and `bsub`.
- `.command.out`: `stdout` from the process.
- `.command.err`: `stderr` from the process.
diff --git a/docs/variant-annotation-and-filtering.md b/docs/variant-annotation-and-filtering.md
index cd60145f..e67b8c71 100644
--- a/docs/variant-annotation-and-filtering.md
+++ b/docs/variant-annotation-and-filtering.md
@@ -2,8 +2,9 @@
:::tip Note
Hard-coded filter thresholds are viewable and editable in the configuration files here:
-[* Exomes](../conf/exome.config)
-[* Genomes](../conf/genome.config)
+* [Exomes](https://github.com/mskcc/tempo/blob/develop/conf/exome.config)
+* [Genomes](https://github.com/mskcc/tempo/blob/develop/conf/genome.config)
+
:::
::: warning Be aware
@@ -128,4 +129,45 @@ Similar to somatic mutations, tumor zygosity of germline SNVs and indels is esti
## Somatic and Germline SVs
-_Under development._
+Tempo uses two to four callers to identify structural variants. By default, the following callers are used regardless of assay type for both somatic and germline analysis:
+ - [Delly](https://github.com/dellytools/delly)
+ - [Manta](https://github.com/Illumina/manta)
+ - [SvABA](https://github.com/walaj/svaba)
+
+When the assay type is WGS, the following caller is used in addition for somatic variant calling only:
+ - [BRASS](https://github.com/cancerit/BRASS)
+
+The SV workflow in Tempo is significantly influenced by the [2020 PCAWG publication on whole genomes](https://www.nature.com/articles/s41586-020-1969-6). Similar to the workflow described in their paper, calls from each structural variant are provided to [mergesvvcf](https://github.com/papaemmelab/mergeSVvcf/tree/master/mergesvvcf), which converts each call to a normalized representation and merges them using a fixed window size of 200bp. Any two calls for which each breakpoint is less than 200bp away and matches relative directionality can be merged.
+
+### Filtering and annotating structural variant calls
+
+Using the read support information reported in Delly and Manta, the variants from those callers are subject to the following filters:
+- `tumor_read_supp`: The variant is supported by less than 5 discordant reads or less than 2 split reads in the tumor sample.
+- `normal_read_supp`: The variant is supported by any number of reads in the normal sample.
+
+From the merged callset, any variant is filtered based on a minimum number of supporting callers (1 for exome, 2 for genome). If a caller produced a filter flag for the variant, it is not considered to be a supporting caller.
+
+The merged callset is converted from vcf to bedpe using [svtools](https://github.com/hall-lab/svtools/tree/master/svtools) and the following filters are applied:
+- `mappability` and `repeat_masker`: One or both breakends is in a repeat, low-mappability, or hard-to-sequence region. More details in the [reference file description](reference-files.md#repeatmasker-and-mappability-blacklist).
+- `pcawg_blacklist_bed`: One or both breakends is in a region that PCAWG has blacklisted.
+- `pcawg_blacklist_bedpe`: The breakpoint is blacklisted by PCAWG.
+- `pcawg_blacklist_fb_bedpe`: The breakpoint is blacklisted by PCAWG and is likely a foldback artefact.
+- `pcawg_blacklist_te_bedpe`: The breakpoint is blacklisted by PCAWG and is likely a transposable element.
+
+The bed and bedpe files used for the flags `pcawg_blacklist_bed`,`pcawg_blacklist_bedpe`, `pcawg_blacklist_fb_bedpe` and `pcawg_blacklist_te_bedpe` are sourced from the [SV merging tool used in the PCAWG paper](https://bitbucket.org/weischenfeldt/pcawg_sv_merge/src/docker/data/blacklist_files/).
+
+In addition to filtering, Tempo also annotates the merged callset using the [iAnnotateSV package](https://github.com/rhshah/iAnnotateSV), and identifies possible cDNA contamination among deletion events that span splice sites. Possible cDNA contamination sites are not filtered.
+
+### Structural Variant Classes
+
+Each breakpoint is described on a single record of the bedpe file, with the coordinates and orientation of both breakends described. The four types of breakends produced by Tempo are as follows:
+| Class | Abbreviation | Description |
+| :--- | :--- | :--- |
+| Breakend | BND | Any event that cannot be described with one of the below terms. The majority of BND are usually translocations. |
+| Deletion | DEL | Loss of a segment that is spanned by two joined breakends either side. |
+| Tandem Duplication | DUP | Extra copy of a segment immediately downstream of the template in the same orientation. |
+| Inversion | INV | A segment inserted into its original position, but in the opposite orientation. Simple inversions are balanced, but in complex inversions the second side of a dsDNA break may not be rescued. |
+
+## BEDPE Format
+
+After merging the variants from different callers, the variants are converted from vcf to bedpe file using [svtools vcftobedpe](https://github.com/hall-lab/svtools). Many downstream tools require bedpe or similar table formats. The PCAWG Working Group also makes use of the bedpe format. You can find more information about the bedpe file format [here](https://bedtools.readthedocs.io/en/latest/content/general-usage.html#bedpe-format).
diff --git a/dsl2.nf b/dsl2.nf
new file mode 100644
index 00000000..72cea33a
--- /dev/null
+++ b/dsl2.nf
@@ -0,0 +1,329 @@
+#!/usr/bin/env nextflow
+nextflow.enable.dsl = 2
+
+if (!(workflow.profile in ['juno', 'awsbatch', 'docker', 'singularity', 'test_singularity', 'test'])) {
+ println 'ERROR: You need to set -profile (values: juno, awsbatch, docker, singularity)'
+ exit 1
+}
+
+// User-set runtime parameters
+outDir = file(params.outDir).toAbsolutePath()
+outname = params.outname
+runAggregate = params.aggregate
+runConpairAll = false
+wallTimeExitCode = params.wallTimeExitCode ? params.wallTimeExitCode.split(',').collect { it.trim().toLowerCase() } : []
+multiqcWesConfig = workflow.projectDir + '/lib/multiqc_config/exome_multiqc_config.yaml'
+multiqcWgsConfig = workflow.projectDir + '/lib/multiqc_config/wgs_multiqc_config.yaml'
+multiqcTempoLogo = workflow.projectDir + '/docs/tempoLogo.png'
+params.startEpoch = new Date().getTime()
+
+
+//Utility Includes
+include { defineReferenceMap; loadTargetReferences } from './modules/function/define_maps'
+include { touchInputs; watchMapping; watchBamMapping; watchPairing; watchAggregateWithResult; watchAggregate } from './modules/function/watch_inputs'
+
+pairingQc = params.pairing
+referenceMap = defineReferenceMap()
+targetsMap = loadTargetReferences()
+
+//Sub-workflow Includes
+include { validate_wf } from './modules/subworkflow/validate_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { alignment_wf } from './modules/subworkflow/alignment_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { manta_wf } from './modules/subworkflow/manta_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { msiSensor_wf } from './modules/subworkflow/msiSensor_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { mutSig_wf } from './modules/subworkflow/mutSig_wf'
+include { mdParse_wf } from './modules/subworkflow/mdParse_wf'
+include { loh_wf } from './modules/subworkflow/loh_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { facets_wf } from './modules/subworkflow/facets_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { sv_wf } from './modules/subworkflow/sv_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { snv_wf } from './modules/subworkflow/snv_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { sampleQC_wf } from './modules/subworkflow/sampleQC_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap, multiqcWesConfig: multiqcWesConfig, multiqcWgsConfig: multiqcWgsConfig, multiqcTempoLogo: multiqcTempoLogo)
+include { samplePairingQC_wf } from './modules/subworkflow/samplePairingQC_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { somaticMultiQC_wf } from './modules/subworkflow/somaticMultiQC_wf' addParams(multiqcWesConfig: multiqcWesConfig, multiqcWgsConfig: multiqcWgsConfig, multiqcTempoLogo: multiqcTempoLogo)
+include { scatter_wf } from './modules/subworkflow/scatter_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { germlineSNV_wf } from './modules/subworkflow/germlineSNV_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { germlineSV_wf } from './modules/subworkflow/germlineSV_wf' addParams(referenceMap: referenceMap, targetsMap: targetsMap)
+include { PairTumorNormal } from './modules/subworkflow/PairTumorNormal'
+include { aggregateFromResult } from './modules/subworkflow/AggregateFromResult'
+include { aggregateFromProcess } from './modules/subworkflow/AggregateFromProcess'
+include { hrdetect_wf } from './modules/subworkflow/hrdetect_wf'
+include { clonality_wf } from './modules/subworkflow/clonality_wf'
+include { ascat_wf } from './modules/subworkflow/ascat_wf' addParams(referenceMap: referenceMap)
+
+aggregateParamIsFile = !(runAggregate instanceof Boolean)
+// check if --aggregate is a file
+
+WFs = params.workflows instanceof Boolean ? '' : params.workflows
+
+WFs = WFs.split(',').collect{it.trim().toLowerCase()}.unique()
+
+WFs = (!params.mapping && !params.bamMapping && aggregateParamIsFile) ? ['snv','sv','mutsig','germsnv','germsv','lohhla','facets','qc','msisensor'] : WFs
+
+workflow {
+ //Set flags for when each pipeline is required to run.
+ doWF_align = (params.mapping) ? true : false
+ doWF_manta = ['snv', 'sv', 'mutsig'].any(it -> it in WFs) ? true : false
+ doWF_scatter = ['snv', 'sv', 'mutsig', 'germsnv'].any(it -> it in WFs) ? true : false
+ doWF_germSNV = 'germsnv' in WFs ? true : false
+ doWF_germSV = 'germsv' in WFs ? true : false
+ doWF_facets = ['lohhla', 'facets', 'snv', 'mutsig', 'germsnv'].any(it -> it in WFs) ? true : false
+ doWF_SV = 'sv' in WFs ? true : false
+ doWF_facets = doWF_SV && params.assayType == "genome" && ["hisens","purity"].contains(params.svcnv) ? true : doWF_facets
+ doWF_loh = ['lohhla', 'snv', 'mutsig'].any(it -> it in WFs) ? true : false
+ doWF_SNV = ['snv', 'mutsig'].any(it -> it in WFs) ? true : false ? true : false
+ doWF_QC = 'qc' in WFs ? true : false
+ doWF_msiSensor = 'msisensor' in WFs ? true : false
+ doWF_mutSig = 'mutsig' in WFs ? true : false
+ doWF_mdParse = (doWF_manta && doWF_scatter && doWF_facets && doWF_loh && doWF_SNV && doWF_msiSensor && doWF_mutSig) ? true : false
+
+ doWF_AggregateFromResult = false
+ doWF_AggregateFromProcess = false
+
+ if (!params.mapping && !params.bamMapping) {
+ if (aggregateParamIsFile) { doWF_AggregateFromResult = true }
+ else {
+ println 'ERROR: (--mapping/-bamMapping [tsv]) or (--mapping/--bamMapping [tsv] & --pairing [tsv] ) or (--aggregate [tsv]) need to be provided, otherwise nothing to be run.'
+ exit 1
+ }
+ }
+ else if (WFs == [''] && runAggregate) {
+ println 'ERROR: No provided sub-workflows enabled to aggregate. Remove --aggregate or add sub-workflows"'
+ exit 1
+ }
+ else{
+ doWF_AggregateFromProcess = runAggregate ? true : false
+ }
+
+ if (!params.pairing && WFs != ['qc'] && WFs != [''] && !doWF_AggregateFromResult){
+ println "ERROR: Certain workflows cannot be performed without pairing information."
+ println "\tProvide a --pairing [tsv], or disable other sub-workflows to proceed."
+ exit 1
+ }
+
+ if (params.bamMapping && WFs == ['']){
+ println "ERROR: No sub-workflows to run.."
+ println "\tPlease provide sub-workflows using --workflow parameters."
+ exit 1
+ }
+
+ if(params.pairing && !params.mapping && !params.bamMapping){
+ println "ERROR: When --pairing [tsv], --mapping/--bamMapping [tsv] must be provided."
+ exit 1
+ }
+
+ if (params.watch == true) {
+ epochMap = [:]
+ for (i in ["mapping","bamMapping","pairing","aggregate"]) {
+ if (file(params."${i}".toString()).exists()){
+ epochMap[file(params."${i}").toRealPath()] = 0
+ }
+ }
+ touchInputs(params.chunkSizeLimit, epochMap)
+ }
+
+ if (doWF_AggregateFromResult){
+ aggregateFromResult(runAggregate, multiqcWesConfig, multiqcWgsConfig, multiqcTempoLogo)
+ }
+ else{
+ //Begin executing modules for the run.
+ validate_wf()
+ inputMapping = validate_wf.out.inputMapping
+ inputPairing = validate_wf.out.inputPairing
+
+ if (doWF_align)
+ {
+ alignment_wf(inputMapping)
+ }
+
+ //Handle input bams as coming originally from bams, or from an alignment this run.
+ if (params.bamMapping) {
+ inputBam = inputMapping
+ if (doWF_QC){
+ inputMapping.map{idSample, target, bam, bai ->
+ [ idSample,target, bam.getParent() ]
+ }.set{ locateFastP4MultiQC }
+
+ locateFastP4MultiQC.map{ idSample,target, bamFolder ->
+ [idSample, file(bamFolder + "/fastp/*json")]
+ }.set{ fastPJson }
+ }
+ }
+ else
+ {
+ inputBam = alignment_wf.out.RunBQSR_bamsBQSR
+ fastPJson = alignment_wf.out.fastPJson
+ }
+
+ //Generate several channels for tumor/normal pairing required for downstream processes.
+ if (params.pairing) {
+ PairTumorNormal(inputBam, inputPairing)
+ bamFiles = PairTumorNormal.out.bamFiles
+ bams = PairTumorNormal.out.bams
+ bamsNormal = PairTumorNormal.out.bamsNormal
+ bamsTumor = PairTumorNormal.out.bamsTumor
+ }
+
+ if(doWF_manta)
+ {
+ manta_wf(bamFiles)
+ }
+
+ if(doWF_scatter)
+ {
+ scatter_wf()
+ }
+
+ if(doWF_germSV)
+ {
+ germlineSV_wf(bams)
+ }
+
+ if(doWF_facets)
+ {
+ facets_wf(bamFiles)
+ }
+
+ if(doWF_germSNV)
+ {
+ germlineSNV_wf(bams, bamsTumor, scatter_wf.out.mergedIList, facets_wf.out.facetsForMafAnno)
+ }
+
+ if(doWF_loh)
+ {
+ loh_wf(bams, bamFiles, facets_wf.out.facetsPurity)
+ }
+
+ if(doWF_SNV)
+ {
+ snv_wf(bamFiles, scatter_wf.out.mergedIList, manta_wf.out.mantaToStrelka, loh_wf.out.hlaOutput, facets_wf.out.facetsForMafAnno)
+ }
+
+ if(doWF_SV)
+ {
+ CNVcalls = false
+ samplestatistics = false
+ if (params.assayType == "genome") {
+ if (params.svcnv == "ascat"){
+ ascat_wf(bamFiles)
+ samplestatistics = ascat_wf.out.ascatSS
+ CNVcalls = ascat_wf.out.ascatCNV
+ } else if(params.svcnv == "hisens") {
+ samplestatistics = facets_wf.out.FacetsHisensSampleStatistics4BRASS
+ } else if(params.svcnv == "purity"){
+ samplestatistics = facets_wf.out.FacetsPuritySampleStatistics4BRASS
+ }
+ }
+ if (doWF_facets && CNVcalls == false){
+ if(params.svcnv == "purity") {
+ CNVcalls = facets_wf.out.FacetsPurityCNV4HrDetectFiltered
+ } else {
+ CNVcalls = facets_wf.out.FacetsHisensCNV4HrDetectFiltered
+ }
+ }
+ sv_wf(
+ bamFiles,
+ manta_wf.out.manta4Combine,
+ samplestatistics,
+ CNVcalls
+ )
+ if (doWF_SNV && params.assayType == "genome"){
+ hrdetect_wf(CNVcalls, snv_wf.out.mafFile, sv_wf.out.SVAnnotBedpePass)
+ clonality_wf(
+ bamFiles,
+ sv_wf.out.SVAnnotBedpePass,
+ snv_wf.out.mafFile,
+ CNVcalls,
+ samplestatistics
+ )
+ }
+ }
+
+ if(doWF_QC)
+ {
+ sampleQC_wf(inputBam, fastPJson)
+ }
+
+ if(doWF_msiSensor)
+ {
+ msiSensor_wf(bamFiles)
+ }
+
+ if(doWF_mutSig)
+ {
+ mutSig_wf(snv_wf.out.mafFile)
+ }
+
+ if(doWF_mdParse)
+ {
+ facets_wf.out.facetsPurity.combine(snv_wf.out.maf4MetaDataParser, by: [0,1,2])
+ .combine(facets_wf.out.FacetsQC4MetaDataParser, by: [0,1,2])
+ .combine(msiSensor_wf.out.msi4MetaDataParser, by: [0,1,2])
+ .combine(mutSig_wf.out.mutSig4MetaDataParser, by: [0,1,2])
+ .combine(loh_wf.out.hlaOutput, by: [1,2])
+ .unique()
+ .map{ idNormal, target, idTumor, purityOut, mafFile, qcOutput, msifile, mutSig, placeHolder, polysolverFile ->
+ [idNormal, target, idTumor, purityOut, mafFile, qcOutput, msifile, mutSig, placeHolder, polysolverFile, targetsMap."$target".codingBed]
+ }.set{ mergedChannelMetaDataParser }
+
+ mdParse_wf(mergedChannelMetaDataParser)
+ }
+
+ if(doWF_QC && params.pairing)
+ {
+ samplePairingQC_wf(inputBam, inputPairing, runConpairAll)
+
+ FacetsQC4SomaticMultiQC = doWF_facets ? facets_wf.out.FacetsQC4SomaticMultiQC : inputPairing.map{ t_id, n_id -> [t_id, n_id,"",""]}
+
+ qualimap4PairedTN = inputPairing
+ .combine(sampleQC_wf.out.qualimap4Process)
+ .branch{ idTumor, idNormal, idSample, qualimapDir ->
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+
+ qualimap4PairedTN.tumor
+ .combine(qualimap4PairedTN.normal, by:[0,1])
+ .map{ idTumor,idNormal, idSample1, qualimapDir1, idSample2, qualimapDir2 ->
+ [idTumor,idNormal,qualimapDir1,qualimapDir2]
+ }.set{qualimap4SomaticMultiQC}
+
+
+ samplePairingQC_wf.out.conpairOutput
+ .join(qualimap4SomaticMultiQC, by:[0,1])
+ .join(FacetsQC4SomaticMultiQC, by:[0,1])
+ .set{ somaticMultiQCinput }
+
+ somaticMultiQC_wf(somaticMultiQCinput)
+ }
+
+ if(doWF_AggregateFromProcess)
+ {
+ aggregateFromProcess(
+ inputPairing,
+ runAggregate,
+ doWF_facets ? facets_wf : false,
+ doWF_SV ? sv_wf : false,
+ doWF_SNV ? snv_wf : false,
+ doWF_SV && doWF_SNV && params.assayType == "genome" ? hrdetect_wf : false,
+ doWF_SV && doWF_SNV && params.assayType == "genome" ? clonality_wf : false,
+ doWF_loh ? loh_wf : false,
+ doWF_mdParse ? mdParse_wf : false,
+ doWF_germSNV ? germlineSNV_wf : false,
+ doWF_germSV ? germlineSV_wf : false,
+ doWF_QC ? sampleQC_wf : false,
+ doWF_QC && params.pairing ? samplePairingQC_wf : false,
+ doWF_QC ? fastPJson : false,
+ multiqcWesConfig,
+ multiqcWgsConfig,
+ multiqcTempoLogo
+ )
+ }
+ }
+}
+workflow.onComplete {
+ file(params.fileTracking).text = ""
+ file(outDir).eachFileRecurse{
+ file(params.fileTracking).append(it + "\n")
+ }
+}
+
diff --git a/modules/function/define_maps.nf b/modules/function/define_maps.nf
new file mode 100644
index 00000000..d31363d7
--- /dev/null
+++ b/modules/function/define_maps.nf
@@ -0,0 +1,94 @@
+def checkParamReturnFile(item) {
+ params."${item}" = params.genomes[params.genome]."${item}"
+ if(params."${item}" == null){println "${item} is not found in reference map"; exit 1}
+ if(file(params."${item}", checkIfExists: false) == []){println "${item} is not found; glob pattern produces empty list"; exit 1}
+ return file(params."${item}", checkIfExists: true)
+}
+
+def defineReferenceMap() {
+ if (!(params.genome in params.genomes)) exit 1, "Genome ${params.genome} not found in configuration"
+ result_array = [
+ 'dbsnp' : checkParamReturnFile("dbsnp"),
+ 'dbsnpIndex' : checkParamReturnFile("dbsnpIndex"),
+ // genome reference dictionary
+ 'genomeDict' : checkParamReturnFile("genomeDict"),
+ // FASTA genome reference
+ 'genomeFile' : checkParamReturnFile("genomeFile"),
+ // genome .fai file
+ 'genomeIndex' : checkParamReturnFile("genomeIndex"),
+ // BWA index files
+ 'bwaIndex' : checkParamReturnFile("bwaIndex"),
+ // VCFs with known indels (such as 1000 Genomes, Mill’s gold standard)
+ 'knownIndels' : checkParamReturnFile("knownIndels"),
+ 'knownIndelsIndex' : checkParamReturnFile("knownIndelsIndex"),
+ 'msiSensorList' : checkParamReturnFile("msiSensorList"),
+ 'svCallingExcludeRegions' : checkParamReturnFile("svCallingExcludeRegions"),
+ 'svCallingIncludeRegions' : checkParamReturnFile("svCallingIncludeRegions"),
+ 'svCallingIncludeRegionsIndex' : checkParamReturnFile("svCallingIncludeRegionsIndex"),
+ ]
+
+ result_array << ['vepCache' : checkParamReturnFile("vepCache")]
+ // for SNP Pileup
+ result_array << ['facetsVcf' : checkParamReturnFile("facetsVcf")]
+ // intervals file for spread-and-gather processes
+ result_array << ['intervals' : checkParamReturnFile("intervals")]
+ // files for CombineChannel, needed by bcftools annotate
+ result_array << ['repeatMasker' : checkParamReturnFile("repeatMasker")]
+ result_array << ['repeatMaskerIndex' : checkParamReturnFile("repeatMaskerIndex")]
+ result_array << ['mapabilityBlacklist' : checkParamReturnFile("mapabilityBlacklist")]
+ result_array << ['mapabilityBlacklistIndex' : checkParamReturnFile("mapabilityBlacklistIndex")]
+ // isoforms needed by vcf2maf
+ result_array << ['isoforms' : checkParamReturnFile("isoforms")]
+ // PON files
+ result_array << ['exomePoN' : checkParamReturnFile("exomePoN")]
+ result_array << ['exomePoNIndex' : checkParamReturnFile("exomePoNIndex")]
+ result_array << ['wgsPoN' : checkParamReturnFile("wgsPoN")]
+ result_array << ['wgsPoNIndex' : checkParamReturnFile("wgsPoNIndex")]
+ // gnomAD resources
+ result_array << ['gnomadWesVcf' : checkParamReturnFile("gnomadWesVcf")]
+ result_array << ['gnomadWesVcfIndex' : checkParamReturnFile("gnomadWesVcfIndex")]
+ result_array << ['gnomadWgsVcf' : checkParamReturnFile("gnomadWgsVcf")]
+ result_array << ['gnomadWgsVcfIndex' : checkParamReturnFile("gnomadWgsVcfIndex")]
+ // HLA FASTA and *dat for LOHHLA
+ result_array << ['hlaFasta' : checkParamReturnFile("hlaFasta")]
+ result_array << ['hlaDat' : checkParamReturnFile("hlaDat")]
+ // files for neoantigen & NetMHC
+ result_array << ['neoantigenCDNA' : checkParamReturnFile("neoantigenCDNA")]
+ result_array << ['neoantigenCDS' : checkParamReturnFile("neoantigenCDS")]
+ // coding region BED files for calculating TMB
+ // splice sites for locating cDNA contam
+ result_array << ['spliceSites' : checkParamReturnFile('spliceSites')]
+ result_array << ['snpGcCorrections' : checkParamReturnFile('snpGcCorrections')]
+ if (! workflow.profile.startsWith("test") ){
+ result_array << ['brassRefDir' : checkParamReturnFile('brassRefDir')]
+ result_array << ['vagrentRefDir' : checkParamReturnFile('vagrentRefDir')]
+ }
+ result_array << ['svBlacklistBed' : checkParamReturnFile('svBlacklistBed')]
+ result_array << ['svBlacklistBedpe' : checkParamReturnFile('svBlacklistBedpe')]
+ result_array << ['svBlacklistFoldbackBedpe' : checkParamReturnFile('svBlacklistFoldbackBedpe')]
+ result_array << ['svBlacklistTEBedpe' : checkParamReturnFile('svBlacklistTEBedpe')]
+
+ return result_array
+}
+
+def loadTargetReferences(){
+ def result_array = [:]
+ new File(params.targets_base).eachDir{ i ->
+ def target_id = i.getBaseName()
+ if (params.assayType == "genome" && target_id != "wgs" ){ return }
+ if (params.assayType != "genome" && target_id == "wgs" ){ return }
+ result_array["${target_id}"] = [:]
+ for ( j in params.targets.keySet()) { // baitsInterval, targetsInterval, targetsBedGz, targetsBedGzTbi, codingBed
+ result_array."${target_id}" << [ ("$j".toString()) : evalTargetPath(j,target_id)]
+ }
+ }
+ return result_array
+}
+
+def evalTargetPath(item,target_id){
+ def templateString = params.targets."${item}"
+ if(templateString == null){println "${item} is not found in targets' map"; exit 1}
+ def res = evaluate("def targets_id=\"$target_id\" ; template=\"$templateString\"")
+ if(file(res, checkIfExists: false) == []){println "${item} is not found; glob pattern produces empty list"; exit 1}
+ return file(file(res, checkIfExists: true).toAbsolutePath().toRealPath())
+}
diff --git a/modules/function/watch_inputs.nf b/modules/function/watch_inputs.nf
new file mode 100644
index 00000000..7c0f01e7
--- /dev/null
+++ b/modules/function/watch_inputs.nf
@@ -0,0 +1,155 @@
+def touchInputs(chunkSizeLimit, epochMap) {
+ new Timer().schedule({
+ for ( i in epochMap.keySet() ){
+ fileEpoch = file(i).lastModified()
+ if (( fileEpoch > epochMap[i]) || (chunkSizeLimit > 0 )) {
+ epochMap[i] = fileEpoch
+ "touch -ca ${i}".execute()
+ }
+ }
+} as TimerTask, 15*1000, params.touchInputsInterval * 60 * 1000 ) // convert minutes to milliseconds
+}
+
+def watchMapping(tsvFile, assayType, validTargetsList) {
+ def index = 0
+ def limitInputLines = params.chunkSizeLimit
+ Channel.watchPath( tsvFile, 'create, modify' )
+ .map{ row ->
+ def timeNow = new Date().getTime()
+ limitInputLines = params.chunkSizeLimit + ( ((timeNow - params.startEpoch)/60000) * (params.chunkSizeLimit / params.touchInputsInterval) )
+ index = 0
+ row
+ }.splitCsv(sep: '\t', header: true)
+ .map{ row ->
+ [index++] + row
+ }.filter{ row ->
+ if (params.chunkSizeLimit > 0 ){
+ row[0] <= limitInputLines
+ } else { 1 }
+ }.map{ row ->
+ row[1]
+ }.unique()
+ .map{ row ->
+ def idSample = row.SAMPLE
+ def target = row.TARGET
+ def fastqFile1 = file(row.FASTQ_PE1, checkIfExists: false)
+ def fastqFile2 = file(row.FASTQ_PE2, checkIfExists: false)
+ def numOfPairs = row.NUM_OF_PAIRS.toInteger()
+ if(!TempoUtils.checkTarget(target, assayType, validTargetsList)){}
+ if(!TempoUtils.checkNumberOfItem(row, 5, tsvFile)){}
+
+ [idSample, numOfPairs, target, fastqFile1, fastqFile2]
+ }
+ .map{ idSample, numOfPairs, target, files_pe1, files_pe2
+ -> tuple( groupKey(idSample, numOfPairs), target, files_pe1, files_pe2)
+ }
+ .transpose()
+ .unique()
+}
+
+def watchBamMapping(tsvFile, assayType, validTargetsList){
+ def index = 0
+ def limitInputLines = params.chunkSizeLimit
+ Channel.watchPath( tsvFile, 'create, modify' )
+ .map{ row ->
+ def timeNow = new Date().getTime()
+ limitInputLines = params.chunkSizeLimit + ( ((timeNow - params.startEpoch)/60000) * (params.chunkSizeLimit / params.touchInputsInterval) )
+ index = 0
+ row
+ }.splitCsv(sep: '\t', header: true)
+ .map{ row ->
+ [index++] + row
+ }.filter{ row ->
+ if (params.chunkSizeLimit > 0 ){
+ row[0] <= limitInputLines
+ } else { 1 }
+ }.map{ row ->
+ row[1]
+ }.unique()
+ .map{ row ->
+ def idSample = row.SAMPLE
+ def target = row.TARGET
+ def bam = file(row.BAM, checkIfExists: false)
+ def bai = file(row.BAI, checkIfExists: false)
+ if(!TempoUtils.checkTarget(target, assayType, validTargetsList)){}
+ if(!TempoUtils.checkNumberOfItem(row, 4, tsvFile)){}
+
+ [idSample, target, bam, bai]
+ }
+ .map{ idSample, target, files_pe1, files_pe2
+ -> tuple( groupKey(idSample, 1), target, files_pe1, files_pe2)
+ }
+ .transpose()
+ .unique()
+}
+
+def watchPairing(tsvFile){
+ Channel.watchPath( tsvFile, 'create, modify' )
+ .splitCsv(sep: '\t', header: true)
+ .unique()
+ .map { row ->
+ def TUMOR_ID = row.TUMOR_ID
+ def NORMAL_ID = row.NORMAL_ID
+ if(!TempoUtils.checkNumberOfItem(row, 2, tsvFile)){}
+
+ [TUMOR_ID, NORMAL_ID]
+ }
+ .unique()
+}
+
+def watchAggregateWithResult(tsvFile) {
+ def index = 0
+ def limitInputLines = params.chunkSizeLimit
+ Channel.watchPath(tsvFile, 'create, modify')
+ .map{ row ->
+ def timeNow = new Date().getTime()
+ limitInputLines = params.chunkSizeLimit + ( ((timeNow - params.startEpoch)/60000) * (params.chunkSizeLimit / params.touchInputsInterval) )
+ index = 0
+ row
+ }.splitCsv(sep: '\t', header: true)
+ .map{ row ->
+ [index++] + row
+ }.filter{ row ->
+ if (params.chunkSizeLimit > 0 ){
+ row[0] <= limitInputLines
+ } else { 1 }
+ }.map{ row ->
+ row[1]
+ }.unique()
+ .map{ row ->
+ def idNormal = row.NORMAL_ID
+ def idTumor = row.TUMOR_ID
+ def cohort = row.COHORT
+ def cohortSize = row.COHORT_SIZE.toInteger()
+ def path = row.PATH
+ if(!TempoUtils.checkNumberOfItem(row, 5, file(runAggregate))){}
+
+ [cohort, cohortSize, idTumor, idNormal, path]
+ }
+ .map { cohort, cohortSize, idTumor, idNormal, path
+ -> tuple( groupKey(cohort, cohortSize), idTumor, idNormal, path)
+ }
+ .transpose()
+ .unique()
+}
+
+def watchAggregate(tsvFile) {
+ Channel.watchPath(tsvFile, 'create, modify')
+ .splitCsv(sep: '\t', header: true)
+ .unique()
+ .map{ row ->
+ def idNormal = row.NORMAL_ID
+ def idTumor = row.TUMOR_ID
+ def cohort = row.COHORT
+ def cohortSize = row.COHORT_SIZE.toInteger()
+ if(!TempoUtils.checkNumberOfItem(row, 4, tsvFile)){}
+
+ [cohort, cohortSize, idTumor, idNormal]
+ }
+ .map { cohort, cohortSize, idTumor, idNormal
+ -> tuple( groupKey(cohort, cohortSize), idTumor, idNormal)
+ }
+ .transpose()
+ .unique()
+}
+
diff --git a/modules/process/Aggregate/CohortRunMultiQC.nf b/modules/process/Aggregate/CohortRunMultiQC.nf
new file mode 100644
index 00000000..ebbd0e53
--- /dev/null
+++ b/modules/process/Aggregate/CohortRunMultiQC.nf
@@ -0,0 +1,82 @@
+process CohortRunMultiQC {
+ tag "${cohort}"
+ label 'multiqc_process'
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(cohort), path(fastPTumor), path(fastPNormal), path(alfredIgnoreYTumor), path(alfredIgnoreYNormal), path(alfredIgnoreNTumor), path(alfredIgnoreNNormal), path(concordFile), path(contamiFile), file(FacetsSummaryFile), file(FacetsQCFile), path(qualimapFolderTumor), path(qualimapFolderNormal), file(hsMetricsTumor), file(hsMetricsNormal)
+ tuple path("exome_multiqc_config.yaml"), path("wgs_multiqc_config.yaml"), path("tempoLogo.png")
+
+ output:
+ tuple val(cohort), path("*multiqc_report*.html"), path("*multiqc_data*.zip"), emit: cohort_multiqc_report
+
+ script:
+ if (params.assayType == "exome") {
+ assay = "exome"
+ }
+ else {
+ assay = 'wgs'
+ }
+ """
+ for i in ./*_qualimap_rawdata.tar.gz ; do
+ newFolder=\$(basename \$i | rev | cut -f 3- -d. | cut -f 3- -d_ | rev )
+ mkdir -p qualimap/\$newFolder
+ tar -xzf \$i -C qualimap/\$newFolder
+ done
+ echo -e "\\tTumor\\tNormal\\tTumor_Contamination\\tNormal_Contamination\\tConcordance" > conpair.tsv
+ for i in ./*contamination.txt ; do
+ j=./\$(basename \$i | cut -f 1 -d.).concordance.txt
+ echo -e "\$(tail -n +2 \$i | sort -r | cut -f 2| head -1)\\t\$(tail -n +2 \$i | sort -r | cut -f 2| paste -sd"\\t")\\t\$(tail -n +2 \$i | sort -r | cut -f 3| paste -sd"\\t")\\t\$(tail -1 \$j | cut -f 2 )" >> conpair.tsv
+ done
+ cp conpair.tsv conpair_genstat.tsv
+
+ mkdir -p fastp_original
+ for i in `find . -maxdepth 1 -name "*fastp.json"` ; do
+ mv \$i fastp_original
+ inname=fastp_original/\$(basename \$i)
+ clean_fastp.py \$inname \$i
+ done
+
+ for i in `find qualimap -name genome_results.txt` ; do
+ sampleName=\$(dirname \$i | xargs -n 1 basename )
+ cover=\$(grep -i "mean cover" \$i | cut -f 2 -d"=" | sed "s/\\s*//g" | tr -d "X" | tr -d "," )
+ echo -e "\${sampleName}\\t\${cover}"
+ done > flatCoverage
+ echo -e "\\tTumor_Coverage\\tNormal_Coverage" > coverage_split.txt
+ join -1 2 -2 1 -o 1.1,1.2,1.3,2.2 -t \$'\\t' <(join -1 1 -2 1 -t \$'\\t' <(cut -f2,3 conpair_genstat.tsv | tail -n +2 | sort | uniq) <(cat flatCoverage | sort | uniq)) <(cat flatCoverage | sort | uniq) | cut -f 1,3,4 >> coverage_split.txt
+ join -1 1 -2 1 -t \$'\\t' <(cut -f 3 conpair_genstat.tsv | sort | uniq | sed "s/\$/\\t/g" ) <(cat flatCoverage | sort | uniq) >> coverage_split.txt
+
+ parse_alfred.py --alfredfiles *alfred*tsv.gz
+ mkdir -p ignoreFolder
+ find . -maxdepth 1 \\( -name 'CO_ignore_mqc.yaml' -o -name 'IS_*mqc.yaml' -o -name 'GC_ignore_mqc.yaml' -o -name 'ME_aware_mqc.yaml' \\) -type f -print0 | xargs -0r mv -t ignoreFolder
+ if [[ "${params.assayType}" == "exome" ]] ; then
+ find . -maxdepth 1 -name 'CM_*mqc.yaml' -type f -print0 | xargs -0r mv -t ignoreFolder
+ fi
+ mv conpair.tsv ignoreFolder
+
+ for i in *.facets_qc.txt ; do
+ head -1 \$i | cut -f 1,28,97 | sed "s/^tumor_sample_id//g"> \$i.qc.txt
+ tail -n +2 \$i | cut -f 1,28,97 | sed "s/TRUE\$/PASS/g" | sed "s/FALSE\$/FAIL/g" >> \$i.qc.txt
+ done
+
+ cp ${assay}_multiqc_config.yaml multiqc_config.yaml
+
+ samplesNum=`for i in ./*contamination.txt ; do tail -n +2 \$i | cut -f 2 ; done | sort | uniq | wc -l`
+ fastpNum=`ls ./*fastp*json | wc -l`
+ mqcSampleNum=\$((samplesNum + fastpNum ))
+ mqcSampleNum=\$(( mqcSampleNum > 25 ? mqcSampleNum : 25 ))
+
+ multiqc . --cl_config "max_table_rows: \$(( mqcSampleNum + 1 ))" -x ignoreFolder/ -x fastp_original/
+ general_stats_parse.py --print-criteria
+ rm -rf multiqc_report.html multiqc_data
+
+ if [ \$samplesNum -gt 50 ] ; then
+ cp genstats-QC_Status.txt QC_Status.txt
+ fi
+ beeswarm_config="max_table_rows: \$(( mqcSampleNum + 1 ))"
+
+ multiqc . --cl_config "title: \\"Cohort MultiQC Report\\"" --cl_config "subtitle: \\"${cohort} QC\\"" --cl_config "intro_text: \\"Aggregate results from Tempo QC analysis\\"" --cl_config "\${beeswarm_config}" --cl_config "report_comment: \\"This report includes FASTQ and alignment for all samples in ${cohort}and Tumor/Normal pair statistics for all pairs in ${cohort}.\\"" -z -x ignoreFolder/ -x fastp_original/
+
+ """
+}
diff --git a/modules/process/Aggregate/GermlineAggregateMaf.nf b/modules/process/Aggregate/GermlineAggregateMaf.nf
new file mode 100644
index 00000000..f51a3f0c
--- /dev/null
+++ b/modules/process/Aggregate/GermlineAggregateMaf.nf
@@ -0,0 +1,29 @@
+process GermlineAggregateMaf {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumors), val(idNormals), val(cohort), val(placeHolder), path(mafFile)
+
+ output:
+ path("mut_germline.maf"), emit: mutationAggregatedGermlineOutput
+
+ script:
+ """
+ ## Making a temp directory that is needed for some reason...
+ mkdir tmp
+ TMPDIR=./tmp
+
+ ## Collect and merge MAF files
+ mkdir mut
+ mv *.maf mut/
+ for i in mut/*.maf ; do
+ if [ \$( cat \$i | wc -l ) -gt 1 ] ; then
+ cat \$i
+ fi
+ done | grep ^Hugo | head -n1 > mut_germline.maf
+ cat mut/*.maf | grep -Ev "^#|^Hugo" | sort -k5,5V -k6,6n >> mut_germline.maf
+
+ """
+}
diff --git a/modules/process/Aggregate/GermlineAggregateSv.nf b/modules/process/Aggregate/GermlineAggregateSv.nf
new file mode 100644
index 00000000..f4212ed7
--- /dev/null
+++ b/modules/process/Aggregate/GermlineAggregateSv.nf
@@ -0,0 +1,16 @@
+process GermlineAggregateSv {
+ tag "${cohortID}"
+ publishDir "${params.outDir}/cohort_level/${cohortID}", mode: params.publishDirMode
+
+input:
+ tuple val(cohortID),
+ path(bedpeFiles)
+
+output:
+ path("sv_germline.bedpe")
+
+script:
+ """
+ awk '\$1 ~ /^#/ && FNR < NR {next;}{print}' ${bedpeFiles} > sv_germline.bedpe
+ """
+}
diff --git a/modules/process/Aggregate/QcBamAggregate.nf b/modules/process/Aggregate/QcBamAggregate.nf
new file mode 100644
index 00000000..e8aa6c4c
--- /dev/null
+++ b/modules/process/Aggregate/QcBamAggregate.nf
@@ -0,0 +1,22 @@
+process QcBamAggregate {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(cohort), path(alfredIgnoreYTumor), path(alfredIgnoreYNormal), path(alfredIgnoreNTumor), path(alfredIgnoreNNormal), file(hsMetricsTumor), file(hsMetricsNormal)
+
+ output:
+ path('alignment_qc.txt'), emit: alignmentQcAggregatedOutput
+
+ script:
+ if (params.assayType == "exome") {
+ assayType = "wes"
+ }
+ else {
+ assayType = 'wgs'
+ }
+ """
+ Rscript --no-init-file /usr/bin/create-aggregate-qc-file.R -n ${task.cpus} -a ${assayType}
+ """
+}
diff --git a/modules/process/Aggregate/QcConpairAggregate.nf b/modules/process/Aggregate/QcConpairAggregate.nf
new file mode 100644
index 00000000..cf2fe562
--- /dev/null
+++ b/modules/process/Aggregate/QcConpairAggregate.nf
@@ -0,0 +1,24 @@
+process QcConpairAggregate {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(cohort), path(concordFile), path(contamiFile)
+
+ output:
+ tuple path('concordance_qc.txt'), path('contamination_qc.txt'), emit: conpairAggregatedOutput
+
+ script:
+ """
+ if ls *.concordance.txt 1> /dev/null 2>&1; then
+ echo -e "Pair\tConcordance" > concordance_qc.txt
+ grep -v "concordance" *.concordance.txt | sed 's/.concordance.txt:/\t/' | cut -f1,3 | sort -k1,1 >> concordance_qc.txt
+ fi
+ if ls *.contamination.txt 1> /dev/null 2>&1; then
+ echo -e "Pair\tSample_Type\tSample_ID\tContamination" > contamination_qc.txt
+ grep -v "Contamination" *.contamination.txt | sed 's/.contamination.txt:/\t/' | sort -k1,1 >> contamination_qc.txt
+ fi
+ touch concordance_qc.txt contamination_qc.txt
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateFacets.nf b/modules/process/Aggregate/SomaticAggregateFacets.nf
new file mode 100644
index 00000000..55c6dca7
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateFacets.nf
@@ -0,0 +1,29 @@
+process SomaticAggregateFacets {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(cohort), path(purity), path(Hisens), path(outLog), path(armLev), path(geneLev)
+
+ output:
+ tuple path("cna_hisens_run_segmentation.seg"), path("cna_purity_run_segmentation.seg"), path("cna_armlevel.txt"), path("cna_genelevel.txt"), path("cna_facets_run_info.txt"), emit: FacetsAnnotationAggregatedOutput
+
+ script:
+ """
+ # Collect and merge FACETS outputs
+ # Arm-level and gene-level output is filtered
+ mkdir facets_tmp
+ mv *_OUT.txt facets_tmp/
+ mv *{purity,hisens}.seg facets_tmp/
+
+ awk 'FNR==1 && NR!=1{next;}{print}' facets_tmp/*_hisens.seg > cna_hisens_run_segmentation.seg
+ awk 'FNR==1 && NR!=1{next;}{print}' facets_tmp/*_purity.seg > cna_purity_run_segmentation.seg
+ awk 'FNR==1 && NR!=1{next;}{print}' facets_tmp/*_OUT.txt > cna_facets_run_info.txt
+ mv *{gene_level,arm_level}.txt facets_tmp/
+ cat facets_tmp/*gene_level.txt | head -n 1 > cna_genelevel.txt
+ awk -v FS='\t' '{ if (\$24 != "DIPLOID" && (\$25 == "PASS" || \$25 == "RESCUE" )) print \$0 }' facets_tmp/*gene_level.txt >> cna_genelevel.txt
+ cat facets_tmp/*arm_level.txt | head -n 1 > cna_armlevel.txt
+ cat facets_tmp/*arm_level.txt | grep -v "DIPLOID" | grep -v "Tumor_Sample_Barcode" >> cna_armlevel.txt || [[ \$? == 1 ]]
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateHRDetect.nf b/modules/process/Aggregate/SomaticAggregateHRDetect.nf
new file mode 100644
index 00000000..be7f8eb1
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateHRDetect.nf
@@ -0,0 +1,15 @@
+process SomaticAggregateHRDetect {
+tag { cohortID }
+publishDir "${params.outDir}/cohort_level/${cohortID}", mode: params.publishDirMode
+
+input:
+ tuple val(cohortID),
+ path(hrdetectFiles)
+output:
+ path("hrdetect.tsv")
+
+script:
+ """
+ awk 'FNR==1 && NR!=1{next;}{print}' ${hrdetectFiles} > hrdetect.tsv
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateLOHHLA.nf b/modules/process/Aggregate/SomaticAggregateLOHHLA.nf
new file mode 100644
index 00000000..ee31fd9e
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateLOHHLA.nf
@@ -0,0 +1,23 @@
+process SomaticAggregateLOHHLA {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(cohort), path(preditHLA), path(intCPN)
+
+ output:
+ path("DNA.IntegerCPN_CI.txt"), emit: lohhlaDNAIntegerCPNOutput
+ path("HLAlossPrediction_CI.txt"), emit: lohhlaHLAlossPredictionAggregatedOutput
+
+ script:
+ """
+ ## Making a temp directory that is needed for some reason...
+ mkdir tmp
+ TMPDIR=./tmp
+ mkdir lohhla
+ mv *.txt lohhla/
+ awk 'FNR==1 && NR!=1{next;}{print}' lohhla/*HLAlossPrediction_CI.txt > HLAlossPrediction_CI.txt
+ awk 'FNR==1 && NR!=1{next;}{print}' lohhla/*DNA.IntegerCPN_CI.txt > DNA.IntegerCPN_CI.txt
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateMaf.nf b/modules/process/Aggregate/SomaticAggregateMaf.nf
new file mode 100644
index 00000000..017e53a7
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateMaf.nf
@@ -0,0 +1,28 @@
+process SomaticAggregateMaf {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumors), val(idNormals), val(cohort), val(placeHolder), path(mafFile)
+
+ output:
+ path("mut_somatic.maf"), emit: mutationAggregatedOutput
+
+ script:
+ """
+ ## Making a temp directory that is needed for some reason...
+ mkdir tmp
+ TMPDIR=./tmp
+
+ ## Collect and merge MAF files
+ mkdir mut
+ mv *.maf mut/
+ for i in mut/*.maf ; do
+ if [ \$( cat \$i | wc -l ) -gt 1 ] ; then
+ cat \$i
+ fi
+ done | grep ^Hugo_Symbol | head -n 1 > mut_somatic.maf
+ cat mut/*.maf | grep -Ev "^#|^Hugo_Symbol" | sort -k5,5V -k6,6n >> mut_somatic.maf
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateMetadata.nf b/modules/process/Aggregate/SomaticAggregateMetadata.nf
new file mode 100644
index 00000000..60d28be5
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateMetadata.nf
@@ -0,0 +1,23 @@
+process SomaticAggregateMetadata {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumors), val(idNormals), val(cohort), val(placeHolder), path(metaDataFile)
+
+ output:
+ path("sample_data.txt"), emit: MetaDataAggregatedOutput
+
+ script:
+ """
+ ## Making a temp directory that is needed for some reason...
+ mkdir tmp
+ TMPDIR=./tmp
+
+ ## Collect and merge metadata file
+ mkdir sample_data_tmp
+ mv *.sample_data.txt sample_data_tmp/
+ awk 'FNR==1 && NR!=1{next;}{print}' sample_data_tmp/*.sample_data.txt > sample_data.txt
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateNetMHC.nf b/modules/process/Aggregate/SomaticAggregateNetMHC.nf
new file mode 100644
index 00000000..f4ee997f
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateNetMHC.nf
@@ -0,0 +1,22 @@
+process SomaticAggregateNetMHC {
+ tag "${cohort}"
+
+ publishDir "${params.outDir}/cohort_level/${cohort}", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumors), val(idNormals), val(cohort), val(placeHolder), path(netmhcCombinedFile)
+
+ output:
+ path("mut_somatic_neoantigens.txt"), emit: NetMhcAggregatedOutput
+
+ script:
+ """
+ ## Making a temp directory that is needed for some reason...
+ mkdir tmp
+ TMPDIR=./tmp
+ ## Collect and merge neoantigen prediction
+ mkdir neoantigen
+ mv *.all_neoantigen_predictions.txt neoantigen/
+ awk 'FNR==1 && NR!=1{next;}{print}' neoantigen/*.all_neoantigen_predictions.txt > mut_somatic_neoantigens.txt
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateSVclone.nf b/modules/process/Aggregate/SomaticAggregateSVclone.nf
new file mode 100644
index 00000000..c2d345e5
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateSVclone.nf
@@ -0,0 +1,17 @@
+process SomaticAggregateSVclone {
+ tag "${cohortid}"
+ publishDir "${params.outDir}/cohort_level/${cohortid}", mode: params.publishDirMode
+
+ input:
+ tuple val(cohortid), path("sv/*"),path("snv/*")
+
+ output:
+ path("svclone_sv_cluster_certainty.tsv")
+ path("svclone_snv_cluster_certainty.tsv")
+
+ script:
+ """
+ awk 'FNR==1 && NR!=1{next;}{print}' sv/* > svclone_sv_cluster_certainty.tsv
+ awk 'FNR==1 && NR!=1{next;}{print}' snv/* > svclone_snv_cluster_certainty.tsv
+ """
+}
\ No newline at end of file
diff --git a/modules/process/Aggregate/SomaticAggregateSv.nf b/modules/process/Aggregate/SomaticAggregateSv.nf
new file mode 100644
index 00000000..34cdf0b1
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateSv.nf
@@ -0,0 +1,16 @@
+process SomaticAggregateSv {
+ tag "${cohortID}"
+ publishDir "${params.outDir}/cohort_level/${cohortID}", mode: params.publishDirMode
+
+input:
+ tuple val(cohortID),
+ path(bedpeFiles)
+
+output:
+ path("sv_somatic.bedpe")
+
+script:
+ """
+ awk '\$1 ~ /^#/ && FNR < NR {next;}{print}' ${bedpeFiles} > sv_somatic.bedpe
+ """
+}
diff --git a/modules/process/Aggregate/SomaticAggregateSvSignatures.nf b/modules/process/Aggregate/SomaticAggregateSvSignatures.nf
new file mode 100644
index 00000000..28989697
--- /dev/null
+++ b/modules/process/Aggregate/SomaticAggregateSvSignatures.nf
@@ -0,0 +1,20 @@
+process SomaticAggregateSvSignatures {
+ tag { cohortID }
+ publishDir "${params.outDir}/cohort_level/${cohortID}", mode: params.publishDirMode
+
+input:
+ tuple val(cohortID),
+ path(svCataloguePdf),
+ path(exposureFiles)
+
+output:
+ path("sv_catalogues.pdf")
+ path("sv_exposures.tsv")
+
+script:
+ """
+ awk 'FNR==1 && NR!=1{next;}{print}' *_exposures.tsv > sv_exposures.tsv
+ gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -sOutputFile=new.pdf *_catalogues.pdf
+ mv new.pdf sv_catalogues.pdf
+ """
+}
diff --git a/modules/process/Alignment/AlignReads.nf b/modules/process/Alignment/AlignReads.nf
new file mode 100644
index 00000000..2fbf8611
--- /dev/null
+++ b/modules/process/Alignment/AlignReads.nf
@@ -0,0 +1,84 @@
+process AlignReads {
+ tag "${fileID + "@" + lane}" // The tag directive allows you to associate each process executions with a custom label
+
+ publishDir "${params.outDir}/bams/${idSample}/fastp", mode: params.publishDirMode, pattern: "*.{html,json}"
+
+ input:
+ tuple val(idSample), val(target), path(fastqFile1), val(sizeFastqFile1), path(fastqFile2), val(sizeFastqFile2), val(fileID), val(lane)
+ tuple path(genomeFile), path(bwaIndex)
+
+ output:
+ tuple val(idSample), path("*.html"), emit: fastPHtml
+ tuple val(idSample), path("*.json"), val(fileID), emit: fastPJson4MultiQC
+ path("file-size.txt"), emit: laneSize
+ tuple val(idSample), val(target), path("*.sorted.bam"), val(fileID), val(lane), path("*.readId"), emit: sortedBam
+
+ script:
+ // LSF resource allocation for juno
+ // if running on juno, check the total size of the FASTQ pairs in order to allocate the runtime limit for the job, via LSF `bsub -W`
+ // if total size of the FASTQ pairs is over 20 GB, use params.maxWallTimeours
+ // if total size of the FASTQ pairs is under 12 GB, use 3h. If there is a 140 error, try again with 6h. If 6h doesn't work, try 500h.
+ inputSize = sizeFastqFile1 + sizeFastqFile2
+ if (workflow.profile == "juno") {
+ if (inputSize > 18.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (inputSize < 9.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+
+ // mem --- total size of the FASTQ pairs in MB (max memory `samtools sort` can take advantage of)
+ // memDivider --- If mem_per_core is true, use 1. Else, use task.cpus
+ // memMultiplier --- If mem_per_core is false, use 1. Else, use task.cpus
+ // originalMem -- If this is the first attempt, use task.memory. Else, use `originalMem`
+ mem = (inputSize/1024**2).round()
+ memDivider = params.mem_per_core ? 1 : task.cpus
+ memMultiplier = params.mem_per_core ? task.cpus : 1
+ originalMem = task.attempt ==1 ? task.memory : originalMem
+
+ if ( mem < 6 * 1024 / task.cpus ) {
+ // minimum total task memory requirment is 6GB because `bwa mem` need this much to run, and increase by 10% everytime retry
+ task.memory = { (6 / memMultiplier * (0.9 + 0.1 * task.attempt) + 0.5).round() + " GB" }
+ mem = (5.4 * 1024 / task.cpus).round()
+ }
+ else if ( mem / memDivider * (1 + 0.1 * task.attempt) > originalMem.toMega() ) {
+ // if file size is too big, use task.memory as the max mem for this task, and decrease -M for `samtools sort` by 10% everytime retry
+ mem = (originalMem.toMega() / memDivider * (1 - 0.1 * task.attempt) + 0.5).round()
+ }
+ else {
+ // normal situation, `samtools sort` -M = inputSize * 2, task.memory is 110% of `samtools sort` and increase by 10% everytime retry
+ task.memory = { (mem * memDivider * (1 + 0.1 * task.attempt) / 1024 + 0.5).round() + " GB" }
+ mem = mem
+ }
+
+ task.memory = task.memory.toGiga() < 1 ? { 1.GB } : task.memory
+
+ filePartNo = fastqFile1.getSimpleName().split("_R1")[-1]
+ """
+ rgID=`zcat $fastqFile1 | head -1 | tr ':/' '@' | cut -d '@' -f2-5`
+ readGroup="@RG\\tID:\${rgID}\\tSM:${idSample}\\tLB:${idSample}\\tPL:Illumina"
+ touch `zcat $fastqFile1 | head -1 | tr ':/\t ' '@' | cut -d '@' -f2-`.readId
+ set -e
+ set -o pipefail
+
+ fastq1=${fastqFile1}
+ fastq2=${fastqFile2}
+ if ${params.anonymizeFQ}; then
+ ln -s ${fastqFile1} ${idSample}@\${rgID}@R1${filePartNo}.fastq.gz
+ ln -s ${fastqFile2} ${idSample}@\${rgID}@R2${filePartNo}.fastq.gz
+ fastq1=`echo ${idSample}@\${rgID}@R1${filePartNo}.fastq.gz`
+ fastq2=`echo ${idSample}@\${rgID}@R2${filePartNo}.fastq.gz`
+ fi
+
+ fastp --html ${idSample}@\${rgID}${filePartNo}.fastp.html --json ${idSample}@\${rgID}${filePartNo}.fastp.json --in1 \${fastq1} --in2 \${fastq2}
+ bwa mem -R \"\${readGroup}\" -t ${task.cpus} -M ${genomeFile} \${fastq1} \${fastq2} | samtools view -Sb - > ${idSample}@\${rgID}${filePartNo}.bam
+
+ samtools sort -m ${mem}M -@ ${task.cpus} -o ${idSample}@\${rgID}${filePartNo}.sorted.bam ${idSample}@\${rgID}${filePartNo}.bam
+ echo -e "${fileID}@${lane}\t${inputSize}" > file-size.txt
+ """
+}
diff --git a/modules/process/Alignment/MergeBamsAndMarkDuplicates.nf b/modules/process/Alignment/MergeBamsAndMarkDuplicates.nf
new file mode 100644
index 00000000..1590bd07
--- /dev/null
+++ b/modules/process/Alignment/MergeBamsAndMarkDuplicates.nf
@@ -0,0 +1,50 @@
+process MergeBamsAndMarkDuplicates {
+ tag "${idSample}"
+
+ input:
+ tuple val(idSample), path(bam), val(target)
+
+ output:
+ tuple val(idSample), path("${idSample}.md.bam"), path("${idSample}.md.bai"), val(target), emit: mdBams
+ path("size.txt"), emit: sizeOutput
+
+ script:
+
+ bamSize = 0
+ [bam].flatten().each{ bamSize = bamSize + it.size()}
+
+ if (workflow.profile == "juno") {
+ if(bamSize > 100.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (bamSize < 80.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+
+ memMultiplier = params.mem_per_core ? task.cpus : 1
+
+ // when increase memory requested from system every time it retries, keep java Xmx steady, in order to give more memory for java garbadge collection
+ originalMem = task.attempt ==1 ? task.memory : originalMem
+ maxMem = (memMultiplier * originalMem.toString().split(" ")[0].toInteger() - 3)
+ maxMem = maxMem < 4 ? 5 : maxMem
+ javaOptions = "--java-options '-Xms4000m -Xmx" + maxMem + "g'"
+ """
+ samtools merge --threads ${task.cpus} ${idSample}.merged.bam ${bam.join(" ")}
+ gatk MarkDuplicates \
+ ${javaOptions} \
+ --TMP_DIR ./ \
+ --MAX_RECORDS_IN_RAM 50000 \
+ --INPUT ${idSample}.merged.bam \
+ --METRICS_FILE ${idSample}.bam.metrics \
+ --ASSUME_SORT_ORDER coordinate \
+ --CREATE_INDEX true \
+ --OUTPUT ${idSample}.md.bam
+
+ echo -e "${idSample}\t`du -hs ${idSample}.md.bam`" > size.txt
+ """
+}
diff --git a/modules/process/Alignment/RunBQSR.nf b/modules/process/Alignment/RunBQSR.nf
new file mode 100644
index 00000000..f71a9e7a
--- /dev/null
+++ b/modules/process/Alignment/RunBQSR.nf
@@ -0,0 +1,91 @@
+ process RunBQSR {
+ tag "${idSample}"
+
+ publishDir "${params.outDir}/bams/${idSample}", mode: params.publishDirMode, pattern: "*.bam*"
+
+ input:
+ tuple val(idSample), path(bam), path(bai), val(target)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict), path(dbsnp), path(dbsnpIndex), path(knownIndels), path(knownIndelsIndex)
+
+ output:
+ tuple val(idSample), val(target), path("${idSample}.bam"), path("${idSample}.bam.bai"), emit: bamsBQSR
+ path("file-size.txt"), emit: bamSize
+
+ script:
+ if (workflow.profile == "juno") {
+ if(bam.size() > 200.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (bam.size() < 100.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+ if (task.attempt < 3 ) {
+ sparkConf = "Spark --conf 'spark.executor.cores = " + task.cpus + "'"
+ }
+ else {
+ sparkConf=""
+ task.cpus = 4
+ task.memory = { 6.GB }
+ if (workflow.profile == "juno"){ task.time = { params.maxWallTime } }
+ }
+
+ memMultiplier = params.mem_per_core ? task.cpus : 1
+ // when increase memory requested from system every time it retries, keep java Xmx steady, in order to give more memory for java garbadge collection
+ originalMem = task.attempt ==1 ? task.memory : originalMem
+ maxMem = (memMultiplier * originalMem.toString().split(" ")[0].toInteger() - 3)
+ maxMem = maxMem < 4 ? 5 : maxMem
+ javaOptions = "--java-options '-Xmx" + originalMem.toString().split(" ")[0].toInteger() * memMultiplier + "g'"
+ knownSites = knownIndels.collect{ "--known-sites ${it}" }.join(' ')
+ if ( task.attempt < 3 )
+ """
+ gatk \
+ BQSRPipeline${sparkConf} \
+ -R ${genomeFile} \
+ -I ${bam} \
+ --known-sites ${dbsnp} \
+ ${knownSites} \
+ --verbosity INFO \
+ --create-output-bam-index true \
+ --emit-original-quals \
+ -O ${idSample}.bam
+
+ echo -e "${idSample}\t\$(du -b ${idSample}.bam)" > file-size.txt
+
+ if [[ -f ${idSample}.bai ]]; then
+ mv ${idSample}.bai ${idSample}.bam.bai
+ fi
+ """
+ else
+ """
+ gatk \
+ BaseRecalibrator${sparkConf} \
+ ${javaOptions} \
+ --reference ${genomeFile} \
+ --known-sites ${dbsnp} \
+ ${knownSites} \
+ --verbosity INFO \
+ --input ${bam} \
+ --output ${idSample}.recal.table
+
+ gatk \
+ ApplyBQSR${sparkConf} \
+ ${javaOptions} \
+ --reference ${genomeFile} \
+ --create-output-bam-index true \
+ --bqsr-recal-file ${idSample}.recal.table \
+ --emit-original-quals \
+ --input ${bam} \
+ --output ${idSample}.bam
+
+ echo -e "${idSample}\t\$(du -b ${idSample}.bam)" > file-size.txt
+
+ if [[ -f ${idSample}.bai ]]; then
+ mv ${idSample}.bai ${idSample}.bam.bai
+ fi
+ """
+ }
diff --git a/modules/process/Alignment/SplitLanes.nf b/modules/process/Alignment/SplitLanes.nf
new file mode 100644
index 00000000..0cda6d24
--- /dev/null
+++ b/modules/process/Alignment/SplitLanes.nf
@@ -0,0 +1,76 @@
+process SplitLanesR1 {
+ tag "${idSample + "@" + fileID}" // The tag directive allows you to associate each process executions with a custom label
+
+ input:
+ tuple val(idSample), val(target), file(fastqFile1), val(fileID)
+
+ output:
+ path("file-size.txt")
+ tuple val(idSample), val(target), path("*R1*.splitLanes.fastq.gz"), path("*.fcid"), path("*.laneCount"), emit: R1SplitData
+
+ when: params.splitLanes
+
+ script:
+ inputSize = fastqFile1.size()
+ if (workflow.profile == "juno") {
+ if (inputSize > 10.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (inputSize < 5.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ // if it's the last time to try, use 500h as time limit no matter for what reason it failed before
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+
+ filePartNo = fastqFile1.getSimpleName().split("_R1")[-1]
+ filePrefix = fastqFile1.getSimpleName().split("_R1")[0..-2].join("_R1")
+ """
+ fcid=`zcat $fastqFile1 | head -1 | tr ':/' '@' | cut -d '@' -f2-4`
+ touch \${fcid}.fcid
+ echo -e "${idSample}@${fileID}\t${inputSize}" > file-size.txt
+ zcat $fastqFile1 | awk -v var="\${fcid}" 'BEGIN {FS = ":"} {lane=\$4 ; print | "gzip > ${filePrefix}@"var"_L00"lane"_R1${filePartNo}.splitLanes.fastq.gz" ; for (i = 1; i <= 3; i++) {getline ; print | "gzip > ${filePrefix}@"var"_L00"lane"_R1${filePartNo}.splitLanes.fastq.gz"}}'
+ touch `ls *R1*.splitLanes.fastq.gz | wc -l`.laneCount
+ """
+}
+
+process SplitLanesR2 {
+ tag "${idSample + "@" + fileID}" // The tag directive allows you to associate each process executions with a custom label
+
+ input:
+ tuple val(idSample), val(target), file(fastqFile2), val(fileID)
+
+ output:
+ file("file-size.txt")
+ tuple val(idSample), val(target), file("*_R2*.splitLanes.fastq.gz"), file("*.fcid"), file("*.laneCount"), emit: R2SplitData
+
+ when: params.splitLanes
+
+ script:
+ inputSize = fastqFile2.size()
+ if (workflow.profile == "juno") {
+ if (inputSize > 10.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (inputSize < 5.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+
+ filePartNo = fastqFile2.getSimpleName().split("_R2")[-1]
+ filePrefix = fastqFile2.getSimpleName().split("_R2")[0..-2].join("_R2")
+ """
+ fcid=`zcat $fastqFile2 | head -1 | tr ':/' '@' | cut -d '@' -f2-4`
+ touch \${fcid}.fcid
+ echo -e "${idSample}@${fileID}\t${inputSize}" > file-size.txt
+ zcat $fastqFile2 | awk -v var="\${fcid}" 'BEGIN {FS = ":"} {lane=\$4 ; print | "gzip > ${filePrefix}@"var"_L00"lane"_R2${filePartNo}.splitLanes.fastq.gz" ; for (i = 1; i <= 3; i++) {getline ; print | "gzip > ${filePrefix}@"var"_L00"lane"_R2${filePartNo}.splitLanes.fastq.gz"}}'
+ touch `ls *_R2*.splitLanes.fastq.gz | wc -l`.laneCount
+ """
+}
diff --git a/modules/process/Ascat/runAscat.nf b/modules/process/Ascat/runAscat.nf
new file mode 100644
index 00000000..daeb2f00
--- /dev/null
+++ b/modules/process/Ascat/runAscat.nf
@@ -0,0 +1,43 @@
+process runAscat {
+ tag {idTumor + "__" + idNormal}
+ label 'ascat'
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(ascatTar), path(tumorBam), path(tumorBai), path(normalBam), path(normalBai)
+ path(genomeFile)
+ path(genomeIndex)
+ path(snpGcCorrections)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), file("ascatResults/*.copynumber.caveman.csv"), emit: caveman
+ tuple val(idTumor), val(idNormal), val(target), file("ascatResults/*.samplestatistics.txt"), emit: samplestatistics
+
+ when: params.assayType == "genome"
+
+ script:
+ if (params.genome in ["GRCh37","smallGRCh37","GRCh38"]){
+ species = "HUMAN"
+ assembly = 37
+ if (params.genome in ["GRCh38"]) { assembly = 38 }
+ } else { // not sure if run will complete with these params.
+ species = params.genome
+ assembly = params.genome
+ }
+ """
+ export TMPDIR=\$(pwd)/tmp
+ mkdir \$TMPDIR
+ for i in ascat_alleleCount_*.tar.gz ;do
+ tar -xzf \$i
+ done
+
+ ascat.pl \\
+ -o ./ascatResults \\
+ -t ${tumorBam} -n ${normalBam} \\
+ -sg ${snpGcCorrections} \\
+ -r ${genomeFile} \\
+ -q 20 -g L \\
+ -rs "${species}" -ra "${assembly}" -pr "WGS" \\
+ -c ${task.cpus} \\
+ -force
+ """
+}
diff --git a/modules/process/Ascat/runAscatAlleleCount.nf b/modules/process/Ascat/runAscatAlleleCount.nf
new file mode 100644
index 00000000..aa25312a
--- /dev/null
+++ b/modules/process/Ascat/runAscatAlleleCount.nf
@@ -0,0 +1,51 @@
+process runAscatAlleleCount {
+ tag {idTumor + "__" + idNormal + "@" + ascatIndex }
+ label 'ascat'
+
+ input:
+ each ascatIndex
+ val(ascatIndexLimit)
+ tuple val(idTumor), val(idNormal), val(target), path(tumorBam), path(tumorBai), path(normalBam), path(normalBai)
+ path(genomeFile)
+ path(genomeIndex)
+ path(snpGcCorrections)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("ascat_alleleCount_${ascatIndex}.tar.gz")
+
+ when: params.assayType == "genome"
+
+ script:
+ genome = params.genome
+ if (genome in ["GRCh37","smallGRCh37","GRCh38"]){
+ species = "HUMAN"
+ assembly = 37
+ if (genome in ["GRCh38"]) { assembly = 38 }
+ } else { // not sure if run will complete with these params.
+ species = genome
+ assembly = genome
+ }
+ if ( ascatIndexLimit == 1 ) {
+ indexParam = ""
+ } else {
+ indexParam = "-i ${ascatIndex} -x ${ascatIndexLimit}"
+ }
+ """
+ mkdir -p ascatResults
+ export TMPDIR=\$(pwd)/tmp
+ mkdir \$TMPDIR
+
+ ascat.pl \\
+ -o ./ascatResults \\
+ -t ${tumorBam} -n ${normalBam} \\
+ -sg ${snpGcCorrections} \\
+ -r ${genomeFile} \\
+ -q 20 -g L \\
+ -rs "${species}" -ra "${assembly}" -pr "WGS" \\
+ -c ${task.cpus} \\
+ -force \\
+ -p allele_count ${indexParam}
+
+ tar -czf ascat_alleleCount_${ascatIndex}.tar.gz ascatResults/
+ """
+}
diff --git a/modules/process/Facets/DoFacets.nf b/modules/process/Facets/DoFacets.nf
new file mode 100644
index 00000000..12e7b7c2
--- /dev/null
+++ b/modules/process/Facets/DoFacets.nf
@@ -0,0 +1,96 @@
+process DoFacets {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${tag}/facets/${tag}", mode: params.publishDirMode, pattern: "*.snp_pileup.gz"
+ publishDir "${params.outDir}/somatic/${tag}/facets/${tag}", mode: params.publishDirMode, pattern: "${tag}_OUT.txt"
+ publishDir "${params.outDir}/somatic/${tag}/facets/${tag}", mode: params.publishDirMode, pattern: "${outputDir}/*.{Rdata,png,out,seg,txt}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal)
+ file(facetsVcf)
+ path(custom_scripts)
+ val(outputDir)
+
+ output:
+ path("${outfile}"), emit: snpPileupOutput
+ path("${outputDir}/*"), emit: FacetsOutput
+ tuple val(idTumor), val(idNormal), path("*/*_purity.seg"), path("*/*_hisens.seg"), path("*_OUT.txt"), path("*/*.arm_level.txt"), path("*/*.gene_level.txt"), emit: facets4Aggregate
+ tuple val(idTumor), val(idNormal), val(target), path("${outputDir}/*purity.out"), emit: facetsPurity
+ tuple val(idTumor), val(idNormal), val(target), path("${outputDir}/*hisens.Rdata"), val(outputDir), emit: facetsForMafAnno
+ tuple val(idTumor), val(idNormal), val(target), path("${outputDir}/*.{Rdata,png,out,seg,txt}"), path("${idTumor}__${idNormal}.snp_pileup.gz"), val(outputDir), emit: Facets4FacetsPreview
+ tuple val(idTumor), val(idNormal), path("*/*.*_level.txt"), emit: FacetsArmGeneOutput
+ tuple val(idTumor), val(idNormal), val(target), path("*/*.qc.txt"), emit: FacetsQC4MetaDataParser
+ tuple val(idTumor), val(idNormal), path("*_OUT.txt"), emit: FacetsRunSummary
+ tuple val(idTumor), val(idNormal), val(target), path("${tag}_hisens.facets.copynumber.csv"), emit: FacetsHisensCNV4HrDetect
+ tuple val(idTumor), val(idNormal), val(target), path("${tag}_hisens.facets.filtered.copynumber.csv"), emit: FacetsHisensCNV4HrDetectFiltered
+ tuple val(idTumor), val(idNormal), val(target), path("${tag}_hisens.samplestatistics.txt"), emit: FacetsHisensSampleStatistics4BRASS
+ tuple val(idTumor), val(idNormal), val(target), path("${tag}_purity.facets.copynumber.csv"), emit: FacetsPurityCNV4HrDetect
+ tuple val(idTumor), val(idNormal), val(target), path("${tag}_purity.facets.filtered.copynumber.csv"), emit: FacetsPurityCNV4HrDetectFiltered
+ tuple val(idTumor), val(idNormal), val(target), path("${tag}_purity.samplestatistics.txt"), emit: FacetsPuritySampleStatistics4BRASS
+
+ script:
+ tag = outputFacetsSubdirectory = "${idTumor}__${idNormal}"
+ outfile = tag + ".snp_pileup.gz"
+ """
+ touch .Rprofile
+
+ export SNP_PILEUP=/usr/bin/snp-pileup
+
+ Rscript /usr/bin/facets-suite/snp-pileup-wrapper.R \
+ --pseudo-snps 50 \
+ --vcf-file ${facetsVcf} \
+ --output-prefix ${tag} \
+ --normal-bam ${bamNormal} \
+ --tumor-bam ${bamTumor}
+
+ mkdir ${outputDir}
+
+ set +e
+ i=1
+ seed=\$((${params.facets.seed}-1))
+ attemptNumber=0
+
+ while [ \$i -eq 1 ]
+ do
+ attemptNumber=\$(( attemptNumber + 1 ))
+ if [ \$attemptNumber -gt 4 ]; then
+ break
+ fi
+ seed=\$((seed+i))
+
+ Rscript /usr/bin/facets-suite/run-facets-wrapper.R \
+ --cval ${params.facets.cval} \
+ --snp-window-size ${params.facets.snp_nbhd} \
+ --normal-depth ${params.facets.ndepth} \
+ --min-nhet ${params.facets.min_nhet} \
+ --purity-cval ${params.facets.purity_cval}\
+ --purity-min-nhet ${params.facets.purity_min_nhet} \
+ --genome ${params.facets.genome} \
+ --counts-file ${outfile} \
+ --sample-id ${tag} \
+ --directory ${outputDir} \
+ --facets-lib-path /usr/local/lib/R/site-library \
+ --seed \$seed \
+ --everything \
+ --legacy-output T
+
+ i=\$?
+ done
+ set -e
+
+ python3 /usr/bin/summarize_project.py \
+ -p ${tag} \
+ -c ${outputDir}/*cncf.txt \
+ -o ${outputDir}/*out \
+ -s ${outputDir}/*seg
+
+ Rscript ${custom_scripts}/generate_samplestatistics.R \\
+ ${outputDir}/${tag}_hisens.Rdata \\
+ ${tag}_hisens
+
+ Rscript ${custom_scripts}/generate_samplestatistics.R \\
+ ${outputDir}/${tag}_purity.Rdata \\
+ ${tag}_purity
+
+ """
+}
diff --git a/modules/process/Facets/DoFacetsPreviewQC.nf b/modules/process/Facets/DoFacetsPreviewQC.nf
new file mode 100644
index 00000000..140de162
--- /dev/null
+++ b/modules/process/Facets/DoFacetsPreviewQC.nf
@@ -0,0 +1,28 @@
+process DoFacetsPreviewQC {
+ tag "${idTumor + "__" + idNormal}"
+ publishDir "${params.outDir}/somatic/${tag}/facets/${tag}/", mode: params.publishDirMode, pattern: "${idTumor}__${idNormal}.facets_qc.txt"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), file(facetsOutputFolderFiles), path(countsFile), val(facetsOutputDir)
+
+ output:
+ tuple val(idTumor), val(idNormal), path("${idTumor}__${idNormal}.facets_qc.txt"), emit: FacetsPreviewOut
+
+ script:
+ tag = "${idTumor}__${idNormal}"
+ """
+ mkdir -p ${facetsOutputDir}
+ facetsFitFiles=( ${facetsOutputFolderFiles.join(" ")} )
+ for i in "\${facetsFitFiles[@]}" ; do
+ cp \$i ${facetsOutputDir}/\$i
+ done
+ echo -e "sample_id\\tsample_path\\ttumor_id" > manifest.txt
+ echo -e "${idTumor}__${idNormal}\\t\$(pwd)\\t${idTumor}" >> manifest.txt
+ gzip manifest.txt
+ mkdir -p refit_watcher/bin/ refit_watcher/refit_jobs/
+ R -e "facetsPreview::generate_genomic_annotations('${idTumor}__${idNormal}', '\$(pwd)/', '/usr/bin/facets-preview/tempo_config.json')"
+ cp facets_qc.txt ${idTumor}__${idNormal}.facets_qc.txt
+ rm ${facetsOutputDir}/*
+ """
+
+}
diff --git a/modules/process/GermSNV/GermlineAnnotateMaf.nf b/modules/process/GermSNV/GermlineAnnotateMaf.nf
new file mode 100644
index 00000000..668fab0f
--- /dev/null
+++ b/modules/process/GermSNV/GermlineAnnotateMaf.nf
@@ -0,0 +1,45 @@
+process GermlineAnnotateMaf {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/combined_mutations", mode: params.publishDirMode, pattern: "*.unfiltered.maf"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(vcfMerged)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict), path(vepCache), path(isoforms)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.maf"), emit: mafFileGermline
+ path("${outputPrefix}.unfiltered.maf"), emit: unfilteredMafOutputGermline
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}.germline"
+ if (target == 'wgs') {
+ infoCols = "MuTect2,Strelka2,Strelka2FILTER,RepeatMasker,EncodeDacMapability,PoN,Ref_Tri,gnomAD_FILTER,AC,AF,AC_nfe_seu,AF_nfe_seu,AC_afr,AF_afr,AC_nfe_onf,AF_nfe_onf,AC_amr,AF_amr,AC_eas,AF_eas,AC_nfe_nwe,AF_nfe_nwe,AC_nfe_est,AF_nfe_est,AC_nfe,AF_nfe,AC_fin,AF_fin,AC_asj,AF_asj,AC_oth,AF_oth,AC_popmax,AN_popmax,AF_popmax"
+ }
+ else {
+ infoCols = "MuTect2,Strelka2,Strelka2FILTER,RepeatMasker,EncodeDacMapability,PoN,Ref_Tri,gnomAD_FILTER,non_cancer_AC_nfe_onf,non_cancer_AF_nfe_onf,non_cancer_AC_nfe_seu,non_cancer_AF_nfe_seu,non_cancer_AC_eas,non_cancer_AF_eas,non_cancer_AC_asj,non_cancer_AF_asj,non_cancer_AC_afr,non_cancer_AF_afr,non_cancer_AC_amr,non_cancer_AF_amr,non_cancer_AC_nfe_nwe,non_cancer_AF_nfe_nwe,non_cancer_AC_nfe,non_cancer_AF_nfe,non_cancer_AC_nfe_swe,non_cancer_AF_nfe_swe,non_cancer_AC,non_cancer_AF,non_cancer_AC_fin,non_cancer_AF_fin,non_cancer_AC_eas_oea,non_cancer_AF_eas_oea,non_cancer_AC_raw,non_cancer_AF_raw,non_cancer_AC_sas,non_cancer_AF_sas,non_cancer_AC_eas_kor,non_cancer_AF_eas_kor,non_cancer_AC_popmax,non_cancer_AF_popmax"
+ }
+ """
+ perl /opt/vcf2maf.pl \
+ --maf-center MSKCC-CMO \
+ --vep-path /usr/bin/vep \
+ --vep-data ${vepCache} \
+ --vep-forks 4 \
+ --tumor-id ${idTumor} \
+ --normal-id ${idNormal} \
+ --vcf-tumor-id ${idTumor} \
+ --vcf-normal-id ${idNormal} \
+ --input-vcf ${vcfMerged} \
+ --ref-fasta ${genomeFile} \
+ --retain-info ${infoCols} \
+ --custom-enst ${isoforms} \
+ --output-maf ${outputPrefix}.raw.maf \
+ --filter-vcf 0
+
+ Rscript --no-init-file /usr/bin/filter-germline-maf.R \
+ --normal-depth ${params.germlineVariant.normalDepth} \
+ --normal-vaf ${params.germlineVariant.normalVaf} \
+ --maf-file ${outputPrefix}.raw.maf \
+ --output-prefix ${outputPrefix}
+ """
+}
diff --git a/modules/process/GermSNV/GermlineCombineChannel.nf b/modules/process/GermSNV/GermlineCombineChannel.nf
new file mode 100644
index 00000000..2cab7761
--- /dev/null
+++ b/modules/process/GermSNV/GermlineCombineChannel.nf
@@ -0,0 +1,149 @@
+process GermlineCombineChannel {
+ tag "${idTumor + "__" + idNormal}"
+
+// 3 intermidiate files (plus 3 index files) output for step by step filter check (2 filter steps involved here)
+ publishDir "${params.outDir}/germline/${idNormal}/combined_mutations/intermediate_files/", mode: params.publishDirMode, pattern: "*.union.*"
+ publishDir "${params.outDir}/germline/${idNormal}/combined_mutations/intermediate_files/", mode: params.publishDirMode, pattern: "*.germline.vcf.gz*"
+
+ input:
+ tuple val(idNormal), val(target), val(placeHolder), path(haplotypecallercombinedVcf), path(haplotypecallercombinedVcfIndex), path(strelkaVcf), path(strelkaVcfIndex), val(idTumor), path(bamTumor), path(baiTumor)
+ tuple path(genomeFile), path(genomeIndex)
+ tuple path(repeatMasker), path(repeatMaskerIndex), path(mapabilityBlacklist), path(mapabilityBlacklistIndex)
+ tuple path(gnomadWesVcf), path(gnomadWesVcfIndex), path(gnomadWgsVcf), path(gnomadWgsVcfIndex)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${idTumor}__${idNormal}.germline.vcf"), emit: mutationMergedGermline
+ path("${idNormal}.union.vcf.gz")
+ path("${idNormal}.union.vcf.gz.tbi")
+ path("${idNormal}.union.pass.vcf.gz")
+ path("${idNormal}.union.pass.vcf.gz.tbi")
+ path("${idTumor}__${idNormal}.germline.vcf.gz")
+ path("${idTumor}__${idNormal}.germline.vcf.gz.tbi")
+
+ script:
+ isecDir = "${idNormal}.isec"
+ gnomad = gnomadWgsVcf
+ if (params.assayType == 'genome') {
+ gnomad = gnomadWgsVcf
+ }
+ else if (params.assayType == 'exome') {
+ gnomad = gnomadWesVcf
+ }
+ """
+ echo -e "##INFO=" > vcf.header
+ echo -e "##INFO=" >> vcf.header
+ echo -e "##INFO=" >> vcf.header
+ echo -e '##INFO=' > vcf.rm.header
+ echo -e '##INFO=' > vcf.map.header
+
+ bcftools isec \
+ --output-type z \
+ --prefix ${isecDir} \
+ ${haplotypecallercombinedVcf} ${strelkaVcf}
+
+ bcftools annotate \
+ --annotations ${isecDir}/0003.vcf.gz \
+ --include 'FILTER!=\"PASS\"' \
+ --mark-sites \"+Strelka2FILTER\" \
+ -k \
+ --output-type z \
+ --output ${isecDir}/0003.annot.vcf.gz \
+ ${isecDir}/0003.vcf.gz
+
+ bcftools annotate \
+ --header-lines vcf.header \
+ --annotations ${isecDir}/0000.vcf.gz \
+ --mark-sites +HaplotypeCaller \
+ --output-type z \
+ --output ${isecDir}/0000.annot.vcf.gz \
+ ${isecDir}/0000.vcf.gz
+
+ bcftools annotate \
+ --header-lines vcf.header \
+ --annotations ${isecDir}/0002.vcf.gz \
+ --mark-sites \"+HaplotypeCaller;Strelka2\" \
+ --output-type z \
+ --output ${isecDir}/0002.tmp.vcf.gz \
+ ${isecDir}/0002.vcf.gz
+
+ tabix --preset vcf ${isecDir}/0002.tmp.vcf.gz
+ tabix --preset vcf ${isecDir}/0003.annot.vcf.gz
+
+ bcftools annotate \
+ --annotations ${isecDir}/0003.annot.vcf.gz \
+ --columns +FORMAT,Strelka2FILTER \
+ --output-type z \
+ --output ${isecDir}/0002.annot.vcf.gz \
+ ${isecDir}/0002.tmp.vcf.gz
+
+ bcftools annotate \
+ --header-lines vcf.header \
+ --annotations ${isecDir}/0001.vcf.gz \
+ --mark-sites +Strelka2 \
+ --output-type z \
+ --output ${isecDir}/0001.annot.vcf.gz \
+ ${isecDir}/0001.vcf.gz
+
+ tabix --preset vcf ${isecDir}/0000.annot.vcf.gz
+ tabix --preset vcf ${isecDir}/0001.annot.vcf.gz
+ tabix --preset vcf ${isecDir}/0002.annot.vcf.gz
+
+ bcftools concat \
+ --allow-overlaps \
+ --rm-dups all \
+ ${isecDir}/0000.annot.vcf.gz \
+ ${isecDir}/0001.annot.vcf.gz \
+ ${isecDir}/0002.annot.vcf.gz | \
+ bcftools sort | \
+ bcftools annotate \
+ --header-lines vcf.rm.header \
+ --annotations ${repeatMasker} \
+ --columns CHROM,FROM,TO,RepeatMasker | \
+ bcftools annotate \
+ --header-lines vcf.map.header \
+ --annotations ${mapabilityBlacklist} \
+ --columns CHROM,FROM,TO,EncodeDacMapability \
+ --output-type z \
+ --output ${idNormal}.union.vcf.gz
+
+ tabix --preset vcf ${idNormal}.union.vcf.gz
+
+ bcftools filter \
+ --include 'FILTER=\"PASS\"' \
+ --output-type z \
+ --output ${idNormal}.union.pass.vcf.gz \
+ ${idNormal}.union.vcf.gz
+
+ tabix --preset vcf ${idNormal}.union.pass.vcf.gz
+
+ bcftools annotate \
+ --annotations ${gnomad} \
+ --columns INFO \
+ ${idNormal}.union.pass.vcf.gz | \
+ bcftools filter \
+ --exclude \"${params.germlineVariant.gnomadAf}\" \
+ --output-type v \
+ --output ${idNormal}.union.gnomad.vcf
+
+ GetBaseCountsMultiSample \
+ --fasta ${genomeFile} \
+ --bam ${idTumor}:${bamTumor} \
+ --vcf ${idNormal}.union.gnomad.vcf \
+ --output ${idTumor}.genotyped.vcf
+
+ bgzip ${idNormal}.union.gnomad.vcf
+ bgzip ${idTumor}.genotyped.vcf
+ tabix --preset vcf ${idNormal}.union.gnomad.vcf.gz
+ tabix --preset vcf ${idTumor}.genotyped.vcf.gz
+
+ bcftools merge \
+ --output ${idTumor}__${idNormal}.germline.vcf \
+ --output-type v \
+ ${idNormal}.union.gnomad.vcf.gz \
+ ${idTumor}.genotyped.vcf.gz
+
+ bgzip -c ${idTumor}__${idNormal}.germline.vcf > ${idTumor}__${idNormal}.germline.vcf.gz
+ tabix --preset vcf ${idTumor}__${idNormal}.germline.vcf.gz
+
+ """
+}
diff --git a/modules/process/GermSNV/GermlineCombineHaplotypecallerVcf.nf b/modules/process/GermSNV/GermlineCombineHaplotypecallerVcf.nf
new file mode 100644
index 00000000..7a3c985c
--- /dev/null
+++ b/modules/process/GermSNV/GermlineCombineHaplotypecallerVcf.nf
@@ -0,0 +1,32 @@
+process GermlineCombineHaplotypecallerVcf {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/haplotypecaller", mode: params.publishDirMode
+
+ input:
+ tuple val(id), val(idNormal), val(target), path(haplotypecallerSnpVcf), path(haplotypecallerSnpVcfIndex), path(haplotypecallerIndelVcf), path(haplotypecallerIndelVcfIndex)
+ tuple file(genomeFile), file(genomeIndex), file(genomeDict)
+
+ output:
+ tuple val(idNormal), val(target), path("${outfile}"), path("${outfile}.tbi"), emit: haplotypecallerCombinedVcfOutput
+
+ script:
+ idNormal = id.toString().split("@")[0]
+ target = id.toString().split("@")[1]
+ outfile = "${idNormal}.haplotypecaller.vcf.gz"
+ """
+ bcftools concat \
+ --allow-overlaps \
+ *.filter.vcf.gz | \
+ bcftools sort | \
+ bcftools norm \
+ --fasta-ref ${genomeFile} \
+ --check-ref s \
+ --multiallelics -both | \
+ bcftools norm --rm-dup all \
+ --output-type z \
+ --output ${idNormal}.haplotypecaller.vcf.gz
+
+ tabix --preset vcf ${idNormal}.haplotypecaller.vcf.gz
+ """
+}
diff --git a/modules/process/GermSNV/GermlineFacetsAnnotation.nf b/modules/process/GermSNV/GermlineFacetsAnnotation.nf
new file mode 100644
index 00000000..2e037873
--- /dev/null
+++ b/modules/process/GermSNV/GermlineFacetsAnnotation.nf
@@ -0,0 +1,27 @@
+process GermlineFacetsAnnotation {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/combined_mutations/", mode: params.publishDirMode, pattern: "*.germline.final.maf"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(hisens_rdata), val(facetsPath), path(maf)
+
+ output:
+ path("${outputPrefix}.final.maf"), emit: mafFileOutputGermline
+ tuple val(idTumor), val(idNormal), file("${outputPrefix}.final.maf"), emit: mafFile4AggregateGermline
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}.germline"
+ """
+ if [ \$( cat ${maf} | wc -l ) -gt 1 ] ; then
+ Rscript --no-init-file /usr/bin/facets-suite/annotate-maf-wrapper.R \
+ --facets-output ${hisens_rdata} \
+ --maf-file ${maf} \
+ --output ${outputPrefix}.facets.maf
+
+ Rscript --no-init-file /usr/bin/annotate-with-zygosity-germline.R ${outputPrefix}.facets.maf ${outputPrefix}.final.maf
+ else
+ cp ${maf} ${outputPrefix}.final.maf
+ fi
+ """
+}
diff --git a/modules/process/GermSNV/GermlineRunHaplotypecaller.nf b/modules/process/GermSNV/GermlineRunHaplotypecaller.nf
new file mode 100644
index 00000000..0717b916
--- /dev/null
+++ b/modules/process/GermSNV/GermlineRunHaplotypecaller.nf
@@ -0,0 +1,53 @@
+process GermlineRunHaplotypecaller {
+ tag "${idNormal + "@" + intervalBed.baseName}"
+
+ input:
+ tuple val(id), val(idNormal), val(target), path(bamNormal), path(baiNormal), path(intervalBed)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(id), val(idNormal), val(target), path("${idNormal}_${intervalBed.baseName}.snps.filter.vcf.gz"), path("${idNormal}_${intervalBed.baseName}.snps.filter.vcf.gz.tbi"), path("${idNormal}_${intervalBed.baseName}.indels.filter.vcf.gz"), path("${idNormal}_${intervalBed.baseName}.indels.filter.vcf.gz.tbi"), emit: haplotypecaller4Combine
+
+ script:
+ """
+ gatk --java-options -Xmx8g \
+ HaplotypeCaller \
+ --reference ${genomeFile} \
+ --intervals ${intervalBed} \
+ --input ${bamNormal} \
+ --output ${idNormal}_${intervalBed.baseName}.vcf.gz
+
+ gatk SelectVariants \
+ --reference ${genomeFile} \
+ --variant ${idNormal}_${intervalBed.baseName}.vcf.gz \
+ --select-type-to-include SNP \
+ --output ${idNormal}_${intervalBed.baseName}.snps.vcf.gz
+
+ gatk SelectVariants \
+ --reference ${genomeFile} \
+ --variant ${idNormal}_${intervalBed.baseName}.vcf.gz \
+ --select-type-to-include INDEL \
+ --output ${idNormal}_${intervalBed.baseName}.indels.vcf.gz
+
+ gatk VariantFiltration \
+ --reference ${genomeFile} \
+ --variant ${idNormal}_${intervalBed.baseName}.snps.vcf.gz \
+ --filter-expression \"QD < 2.0\" --filter-name \"QD2\" \
+ --filter-expression \"QUAL < 30.0\" --filter-name \"QUAL30\" \
+ --filter-expression \"SOR > 3.0\" --filter-name \"SOR3\" \
+ --filter-expression \"FS > 60.0\" --filter-name \"FS60\" \
+ --filter-expression \"MQ < 40.0\" --filter-name \"MQ40\" \
+ --filter-expression \"MQRankSum < -12.5\" --filter-name \"MQRankSum-12.5\" \
+ --filter-expression \"ReadPosRankSum < -8.0\" --filter-name \"ReadPosRankSum-8\" \
+ --output ${idNormal}_${intervalBed.baseName}.snps.filter.vcf.gz
+
+ gatk VariantFiltration \
+ --reference ${genomeFile} \
+ --variant ${idNormal}_${intervalBed.baseName}.indels.vcf.gz \
+ --filter-expression \"QD < 2.0\" --filter-name \"QD2\" \
+ --filter-expression \"QUAL < 30.0\" --filter-name \"QUAL30\" \
+ --filter-expression \"FS > 200.0\" --filter-name \"FS200\" \
+ --filter-expression \"ReadPosRankSum < -20.0\" --filter-name \"ReadPosRankSum-20\" \
+ --output ${idNormal}_${intervalBed.baseName}.indels.filter.vcf.gz
+ """
+}
diff --git a/modules/process/GermSNV/GermlineRunStrelka2.nf b/modules/process/GermSNV/GermlineRunStrelka2.nf
new file mode 100644
index 00000000..b59bf221
--- /dev/null
+++ b/modules/process/GermSNV/GermlineRunStrelka2.nf
@@ -0,0 +1,36 @@
+process GermlineRunStrelka2 {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/strelka2", mode: params.publishDirMode
+
+ input:
+ tuple val(idNormal), val(target), path(bamNormal), path(baiNormal), path(targets), path(targetsIndex)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idNormal), val(target), path("${idNormal}.strelka2.vcf.gz"), path("${idNormal}.strelka2.vcf.gz.tbi"), emit: strelkaOutputGermline
+
+ script:
+ options = ""
+ intervals = targets
+ if (params.assayType == "exome") {
+ options = "--exome"
+ }
+ """
+ configureStrelkaGermlineWorkflow.py \
+ ${options} \
+ --callRegions ${intervals} \
+ --referenceFasta ${genomeFile} \
+ --bam ${bamNormal} \
+ --runDir Strelka
+
+ python Strelka/runWorkflow.py \
+ --mode local \
+ --jobs ${task.cpus}
+
+ mv Strelka/results/variants/variants.vcf.gz ${idNormal}.strelka2.vcf.gz
+ mv Strelka/results/variants/variants.vcf.gz.tbi ${idNormal}.strelka2.vcf.gz.tbi
+
+ if [ \$(vcf-validator ${idNormal}.strelka2.vcf.gz 2>&1 | wc -l ) -gt 0 ] ; then exit 1 ; fi
+ """
+}
diff --git a/modules/process/GermSV/GermlineAnnotateSVBedpe.nf b/modules/process/GermSV/GermlineAnnotateSVBedpe.nf
new file mode 100644
index 00000000..2ab76e3b
--- /dev/null
+++ b/modules/process/GermSV/GermlineAnnotateSVBedpe.nf
@@ -0,0 +1,90 @@
+process GermlineAnnotateSVBedpe {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${outputPrefix}/combined_svs", mode: params.publishDirMode, pattern: "*.unfiltered.bedpe"
+ publishDir "${params.outDir}/germline/${outputPrefix}/combined_svs", mode: params.publishDirMode, pattern: "*.final.bedpe"
+
+ input:
+ tuple val(idNormal), val(target), path(bedpein)
+ path(repeatMasker)
+ path(mapabilityBlacklist)
+ path(svBlacklistBed)
+ path(svBlacklistBedpe)
+ path(svBlacklistFoldbackBedpe)
+ path(svBlacklistTEBedpe)
+ path(spliceSites)
+ path(custom_scripts)
+ val(genome)
+
+ output:
+ tuple val(idNormal), val(target), path("${outputPrefix}.unfiltered.bedpe"), emit: SVAnnotBedpe
+ tuple val(idNormal), val(target), path("${outputPrefix}.final.bedpe"), emit: SVAnnotBedpePass
+ tuple val("placeholder"), val("noTumor"), val(idNormal), path("${outputPrefix}.final.bedpe"), emit: SVAnnotBedpe4Aggregate
+ script:
+ outputPrefix = "${idNormal}"
+ genome_ = ["GRCh37","smallGRCh37"].contains(genome) ? "hg19" : genome == "GRCh38" ? "hg38" : "hg18"
+ """
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${mapabilityBlacklist} \\
+ --bedpe ${bedpein} \\
+ --tag mappability \\
+ --output ${outputPrefix}.combined.dac.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${repeatMasker} \\
+ --bedpe ${outputPrefix}.combined.dac.bedpe \\
+ --tag repeat_masker \\
+ --output ${outputPrefix}.combined.dac.rm.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistBed} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.bedpe \\
+ --tag pcawg_blacklist_bed \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.1.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistBedpe} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.1.bedpe \\
+ --tag pcawg_blacklist_bedpe \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.2.bedpe \\
+ --match-type both \\
+ --ignore-strand
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistFoldbackBedpe} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.2.bedpe \\
+ --tag pcawg_blacklist_fb_bedpe \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.3.bedpe \\
+ --match-type both
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistTEBedpe} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.3.bedpe \\
+ --tag pcawg_blacklist_te_bedpe \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.4.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/detect_cdna.py \\
+ --exon-junct ${spliceSites} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.4.bedpe \\
+ --out-bedpe ${outputPrefix}.combined.dac.rm.pcawg.cdna.bedpe \\
+ --out ${outputPrefix}.contamination.tsv
+
+ python ${custom_scripts}/run_iannotatesv.py \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.cdna.bedpe \\
+ --genome ${genome_} \\
+ --threads ${task.cpus * 2}
+
+ cp ${outputPrefix}.combined.dac.rm.pcawg.cdna.iannotate.bedpe \\
+ ${outputPrefix}.unfiltered.bedpe
+
+ awk -F"\\t" '\$1 ~ /#/ || \$12 == "PASS"' \\
+ ${outputPrefix}.unfiltered.bedpe > \\
+ ${outputPrefix}.final.bedpe
+
+ """
+
+}
diff --git a/modules/process/GermSV/GermlineDellyCall.nf b/modules/process/GermSV/GermlineDellyCall.nf
new file mode 100644
index 00000000..d1a205da
--- /dev/null
+++ b/modules/process/GermSV/GermlineDellyCall.nf
@@ -0,0 +1,32 @@
+process GermlineDellyCall {
+ tag "${idNormal + '@' + svType}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/delly", mode: params.publishDirMode, pattern: "*delly.vcf.{gz,gz.tbi}"
+
+ input:
+ each svType
+ tuple val(idNormal), val(target), path(bamNormal), path(baiNormal)
+ tuple path(genomeFile), path(genomeIndex), path(svCallingExcludeRegions)
+
+ output:
+ tuple val(idNormal), val(target), path("${idNormal}_${svType}.delly.vcf.gz"), path("${idNormal}_${svType}.delly.vcf.gz.tbi"), emit: dellyFilter4CombineGermline
+ tuple path("*delly.vcf.gz"), path("*delly.vcf.gz.tbi"), emit: dellyOutputGermline
+
+ script:
+ """
+ delly call \
+ --svtype ${svType} \
+ --genome ${genomeFile} \
+ --exclude ${svCallingExcludeRegions} \
+ --outfile ${idNormal}_${svType}.bcf \
+ ${bamNormal}
+
+ delly filter \
+ --filter germline \
+ --outfile ${idNormal}_${svType}.filter.bcf \
+ ${idNormal}_${svType}.bcf
+
+ bcftools view --output-type z ${idNormal}_${svType}.filter.bcf > ${idNormal}_${svType}.delly.vcf.gz
+ tabix --preset vcf ${idNormal}_${svType}.delly.vcf.gz
+ """
+}
diff --git a/modules/process/GermSV/GermlineMergeSVs.nf b/modules/process/GermSV/GermlineMergeSVs.nf
new file mode 100644
index 00000000..6072fcb1
--- /dev/null
+++ b/modules/process/GermSV/GermlineMergeSVs.nf
@@ -0,0 +1,61 @@
+process GermlineMergeSVs {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/combined_svs/intermediate_files", mode: params.publishDirMode, pattern: "*.merged.vcf.{gz,gz.tbi}"
+ publishDir "${params.outDir}/germline/${idNormal}/combined_svs/intermediate_files", mode: params.publishDirMode, pattern: "*.merged.raw.vcf.{gz,gz.tbi}"
+
+ input:
+ tuple val(idNormal), val(target),
+ path(Vcfs), path(Tbis),
+ val(callerNames)
+ path(custom_scripts)
+
+ output:
+ tuple val(idNormal), val(target), path("${idNormal}.merged.vcf.gz"), path("${idNormal}.merged.vcf.gz.tbi"), emit: SVsCombinedOutputGermline
+ path("${idNormal}.merged.raw.vcf.{gz,gz.tbi}")
+
+ script:
+ vcfMap = [:]
+ for (i in 1..callerNames.size()){
+ vcfMap.put(callerNames[i-1], Vcfs[i-1])
+ }
+ labelparam = callerNames.sort().join(",")
+ inVCFs = ""
+ for (i in callerNames.sort()){
+ inVCFs += " " + vcfMap[i]
+ }
+ passMin = callerNames.size() > 2 ? 2 : 1
+ """
+ mergesvvcf \\
+ -n -m 1 \\
+ -l ${labelparam} \\
+ -o ${idNormal}.merged.raw.vcf \\
+ -f -d -s -v \\
+ ${inVCFs}
+
+ cat ${idNormal}.merged.raw.vcf | \\
+ awk -F"\\t" -v OFS="\\t" '\$1 ~ /^#/ && \$1 !~ /^##/ && \$1 !~ /^#CHROM/{next;}{for(i=1; i<=NF; i++) if(\$i ~ /^ *\$/) \$i = "."; print \$0}' | \\
+ bcftools sort --temp-dir ./ \\
+ > ${idNormal}.merged.clean.anon.vcf
+
+ bcftools annotate \\
+ --set-id 'TEMPO_%INFO/SVTYPE\\_%CHROM\\_%POS' \\
+ -o ${idNormal}.merged.clean.vcf \\
+ ${idNormal}.merged.clean.anon.vcf
+
+ python ${custom_scripts}/filter-sv-vcf.py \\
+ --input ${idNormal}.merged.clean.vcf \\
+ --output ${idNormal}.merged.clean.corrected.vcf \\
+ --min ${passMin}
+
+ bcftools view \\
+ --samples ${idNormal} \\
+ --output-type z \\
+ --output-file ${idNormal}.merged.vcf.gz \\
+ ${idNormal}.merged.clean.corrected.vcf
+
+ tabix --preset vcf ${idNormal}.merged.vcf.gz
+
+ bcftools view -O z -o ${idNormal}.merged.raw.vcf.gz ${idNormal}.merged.raw.vcf
+ """
+}
diff --git a/modules/process/GermSV/GermlineRunManta.nf b/modules/process/GermSV/GermlineRunManta.nf
new file mode 100644
index 00000000..8bac8555
--- /dev/null
+++ b/modules/process/GermSV/GermlineRunManta.nf
@@ -0,0 +1,44 @@
+process GermlineRunManta {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${idNormal}/manta", mode: params.publishDirMode
+
+ input:
+ tuple val(idNormal), val(target), path(bamNormal), path(baiNormal)
+ tuple path(genomeFile), path(genomeIndex)
+ tuple path(svCallingIncludeRegions), path(svCallingIncludeRegionsIndex)
+
+ output:
+ tuple val(idNormal), val(target), path("${idNormal}.manta.vcf.gz"), path("${idNormal}.manta.vcf.gz.tbi"), emit: mantaOutputGermline
+
+
+ // flag with --exome if exome
+ script:
+ options = ""
+ if (params.assayType == "exome") options = "--exome"
+ """
+ configManta.py \
+ ${options} \
+ --callRegions ${svCallingIncludeRegions} \
+ --reference ${genomeFile} \
+ --bam ${bamNormal} \
+ --runDir Manta
+
+ python Manta/runWorkflow.py \
+ --mode local \
+ --jobs ${task.cpus}
+
+ mv Manta/results/variants/candidateSmallIndels.vcf.gz \
+ Manta_${idNormal}.candidateSmallIndels.vcf.gz
+ mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi \
+ Manta_${idNormal}.candidateSmallIndels.vcf.gz.tbi
+ mv Manta/results/variants/candidateSV.vcf.gz \
+ Manta_${idNormal}.candidateSV.vcf.gz
+ mv Manta/results/variants/candidateSV.vcf.gz.tbi \
+ Manta_${idNormal}.candidateSV.vcf.gz.tbi
+ mv Manta/results/variants/diploidSV.vcf.gz \
+ ${idNormal}.manta.vcf.gz
+ mv Manta/results/variants/diploidSV.vcf.gz.tbi \
+ ${idNormal}.manta.vcf.gz.tbi
+ """
+}
diff --git a/modules/process/GermSV/GermlineRunSvABA.nf b/modules/process/GermSV/GermlineRunSvABA.nf
new file mode 100644
index 00000000..38266392
--- /dev/null
+++ b/modules/process/GermSV/GermlineRunSvABA.nf
@@ -0,0 +1,39 @@
+process GermlineRunSvABA {
+ tag "${idNormal}"
+ publishDir "${params.outDir}/germline/${idNormal}/svaba", mode: params.publishDirMode, pattern: "*.{vcf.gz,vcf.gz.tbi}"
+
+ input:
+ tuple val(idNormal), val(target), path(bamNormal), path(baiNormal), path(targetsBed)
+ path(genomeFile)
+ path(genomeIndex)
+ path(genomeDict)
+ path(bwaIndex)
+
+ output:
+ tuple val(idNormal), val(target), path("${outputPrefix}.reheader.svaba.germline.sv.vcf.gz"), path("${outputPrefix}.reheader.svaba.germline.sv.vcf.gz.tbi"), emit: SvABA4Combine
+ path("*.vcf.gz*"), emit: allVcfs
+ path("*.log"), emit: logs
+ path("*.txt.gz"), emit: supportingFiles
+
+ script:
+ outputPrefix = "${idNormal}"
+ target_param = params.assayType == "genome" ? "" : "-k ${targetsBed} "
+ """
+ svaba run \\
+ -t "${bamNormal}" \\
+ -G "${genomeFile}" \\
+ -p "${task.cpus * 2}" \\
+ -I \\
+ -L 6 \\
+ --id-string "${outputPrefix}" \\
+ ${target_param} \\
+ -z
+
+ echo -e "${bamNormal} ${idNormal}" > svaba.samplenames.tsv
+ bcftools reheader \\
+ --samples svaba.samplenames.tsv \\
+ --output ${outputPrefix}.reheader.svaba.germline.sv.vcf.gz \\
+ ${outputPrefix}.svaba.sv.vcf.gz
+ bcftools index -f -t ${outputPrefix}.reheader.svaba.germline.sv.vcf.gz
+ """
+}
diff --git a/modules/process/GermSV/GermlineSVVcf2Bedpe.nf b/modules/process/GermSV/GermlineSVVcf2Bedpe.nf
new file mode 100644
index 00000000..e17c6aa1
--- /dev/null
+++ b/modules/process/GermSV/GermlineSVVcf2Bedpe.nf
@@ -0,0 +1,42 @@
+process GermlineSVVcf2Bedpe {
+ tag "${idNormal}"
+
+ publishDir "${params.outDir}/germline/${outputPrefix}/combined_svs/intermediate_files", mode: params.publishDirMode, pattern: "*.combined.bedpe"
+
+ input:
+ tuple val(idNormal), val(target), path(vcfFile), path(tbiFile)
+
+ output:
+ tuple val(idNormal), val(target), path("${outputPrefix}.combined.bedpe"), emit: GermlineCombinedUnfilteredBedpe
+
+ script:
+ outputPrefix = "${idNormal}"
+ """
+ export LC_ALL=C
+
+ echo -e "${idNormal} NORMAL" > normalize.samplenames.tsv
+ bcftools reheader \\
+ --samples normalize.samplenames.tsv \\
+ --output reheader_${vcfFile} \\
+ ${vcfFile}
+
+ svtools vcftobedpe \\
+ -i reheader_${vcfFile} \\
+ -o ${outputPrefix}.combined.tmp.bedpe \\
+ -t ${outputPrefix}_tmp
+
+ if [ ! -s ${outputPrefix}.combined.tmp.bedpe ] ; then
+ echo -e "#CHROM_A\\tSTART_A\\tEND_A\\tCHROM_B\\tSTART_B\\tEND_B\\tID\\tQUAL\\tSTRAND_A\\tSTRAND_B\\tTYPE\\tFILTER\\tNAME_A\\tREF_A\\tALT_A\\tNAME_B\\tREF_B\\tALT_B\\tINFO_A\\tINFO_B\\tFORMAT\\tNORMAL" >> ${outputPrefix}.combined.tmp.bedpe
+ fi
+
+ zgrep "^##" ${vcfFile} | sed "s/##fileformat=*/##fileformat=BEDPE/g" > ${outputPrefix}.combined.unsorted.bedpe
+ grep -v "^##" ${outputPrefix}.combined.tmp.bedpe | \\
+ awk -F"\\t" -v nid="${idNormal}" -v OFS="\\t" 'NR == 1 {print \$0,"NORMAL_ID";next;}{print \$0,nid}' \\
+ >> ${outputPrefix}.combined.unsorted.bedpe
+
+ svtools bedpesort \\
+ ${outputPrefix}.combined.unsorted.bedpe \\
+ ${outputPrefix}.combined.bedpe
+ """
+
+}
diff --git a/modules/process/HRDetect/HRDetect.nf b/modules/process/HRDetect/HRDetect.nf
new file mode 100644
index 00000000..680928cf
--- /dev/null
+++ b/modules/process/HRDetect/HRDetect.nf
@@ -0,0 +1,25 @@
+process HRDetect {
+
+ tag {idTumor + "__" + idNormal}
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/hrdetect/", mode: params.publishDirMode, pattern: "*.hrdetect.tsv"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(mafFile), path(cnvFile), path(svFile)
+ path(HRDetect_script)
+
+ output:
+ tuple val(idTumor), val(idNormal), path("${outputPrefix}.hrdetect.tsv")
+
+ when: params.assayType == "genome"
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ genome_version = params.genome == 'GRCh38' ? "hg38" : "hg19"
+ """
+ echo -e "sample\\tsv\\tmutations\\tcnv" > ${outputPrefix}.tsv
+ echo -e "${outputPrefix}\\t${svFile}\\t${mafFile}\\t${cnvFile}" >> ${outputPrefix}.tsv
+ Rscript ${HRDetect_script} ${outputPrefix}.tsv ${genome_version} ${task.cpus}
+ """
+
+}
diff --git a/modules/process/HRDetect/RunSVSignatures.nf b/modules/process/HRDetect/RunSVSignatures.nf
new file mode 100644
index 00000000..22e4a06b
--- /dev/null
+++ b/modules/process/HRDetect/RunSVSignatures.nf
@@ -0,0 +1,25 @@
+process RunSVSignatures {
+ tag {idTumor + "__" + idNormal}
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs/", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bedpe)
+ path(sv_signature_script)
+
+ output:
+ tuple val("placeholder"), val(idTumor), val(idNormal),
+ path("${outputPrefix}_catalogues.pdf"), path("${outputPrefix}_exposures.tsv")
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ genome_version = params.genome == 'GRCh38' ? "hg38" : "hg19"
+ """
+ Rscript ${sv_signature_script} \\
+ -i ${bedpe} \\
+ -g ${genome_version} \\
+ -n ${task.cpus} \\
+ -s ${outputPrefix}
+ """
+
+}
diff --git a/modules/process/LoH/RunLOHHLA.nf b/modules/process/LoH/RunLOHHLA.nf
new file mode 100644
index 00000000..25f6cf7a
--- /dev/null
+++ b/modules/process/LoH/RunLOHHLA.nf
@@ -0,0 +1,62 @@
+process RunLOHHLA {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/lohhla", mode: params.publishDirMode
+
+ input:
+ tuple val(idNormal), val(target), val(idTumor), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal), path(purityOut), val(placeHolder), path(winnersHla)
+ tuple path(hlaFasta), path(hlaDat)
+
+ output:
+ tuple path("*.DNA.HLAlossPrediction_CI.txt"), path("*DNA.IntegerCPN_CI.txt"), path("*.pdf"), path("*.RData"), optional: true, emit: lohhlaOutput
+ tuple val(placeHolder), val(idTumor), val(idNormal), file("*.DNA.HLAlossPrediction_CI.txt"), file("*DNA.IntegerCPN_CI.txt"), emit: lohhla4Aggregate
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ """
+ cat ${winnersHla} | tr "\t" "\n" | grep -v "HLA" > massaged.winners.hla.txt
+
+ PURITY=\$(grep Purity *_purity.out | grep -oP "[0-9\\.]+|NA+")
+ PLOIDY=\$(grep Ploidy *_purity.out | grep -oP "[0-9\\.]+|NA+")
+ cat <(echo -e "tumorPurity\ttumorPloidy") <(echo -e "${idTumor}\t\$PURITY\t\$PLOIDY") > tumor_purity_ploidy.txt
+
+ Rscript --no-init-file /lohhla/LOHHLAscript.R \
+ --patientId ${outputPrefix} \
+ --normalBAMfile ${bamNormal} \
+ --tumorBAMfile ${bamTumor} \
+ --HLAfastaLoc ${hlaFasta} \
+ --HLAexonLoc ${hlaDat} \
+ --CopyNumLoc tumor_purity_ploidy.txt \
+ --minCoverageFilter ${params.lohhla.minCoverageFilter} \
+ --hlaPath massaged.winners.hla.txt \
+ --gatkDir /picard-tools \
+ --novoDir /opt/conda/bin
+
+ if [[ -f ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.HLAlossPrediction_CI.txt ]]
+ then
+ sed -i "s/^${idTumor}/${outputPrefix}/g" ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.HLAlossPrediction_CI.txt
+ else
+ rm -rf *.DNA.HLAlossPrediction_CI.txt
+ fi
+
+ if [[ -f ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.IntegerCPN_CI.txt ]]
+ then
+ sed -i "s/^/${outputPrefix}\t/g" ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.IntegerCPN_CI.txt
+ sed -i "0,/^${outputPrefix}\t/s//sample\t/" ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.IntegerCPN_CI.txt
+ else
+ rm -rf *.DNA.IntegerCPN_CI.txt
+ fi
+
+ touch ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.HLAlossPrediction_CI.txt
+ touch ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.IntegerCPN_CI.txt
+
+ mv ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.HLAlossPrediction_CI.txt ${outputPrefix}.DNA.HLAlossPrediction_CI.txt
+ mv ${outputPrefix}.${params.lohhla.minCoverageFilter}.DNA.IntegerCPN_CI.txt ${outputPrefix}.DNA.IntegerCPN_CI.txt
+
+ if find Figures -mindepth 1 | read
+ then
+ mv Figures/* .
+ mv ${idTumor}.minCoverage_${params.lohhla.minCoverageFilter}.HLA.pdf ${outputPrefix}.HLA.pdf
+ fi
+ """
+}
diff --git a/modules/process/LoH/RunPolysolver.nf b/modules/process/LoH/RunPolysolver.nf
new file mode 100644
index 00000000..5042a27e
--- /dev/null
+++ b/modules/process/LoH/RunPolysolver.nf
@@ -0,0 +1,39 @@
+process RunPolysolver {
+ tag "${idNormal}"
+
+ input:
+ tuple val(idNormal), val(target), path(bamNormal), path(baiNormal)
+
+ output:
+ tuple val(idNormal), val(target), path("${outputPrefix}.hla.txt"), emit: hlaOutput
+
+ script:
+ outputPrefix = "${idNormal}"
+ outputDir = "."
+ tmpDir = "${outputDir}-nf-scratch"
+ genome_ = params.genome == "GRCh37" ? "hg19" : params.genome == 'GRCh38' ? "hg38" : params.genome == 'smallGRCh37' ? "small" : "other"
+ """
+if [ ${genome_} != "small" ] ; then
+
+ cp /home/polysolver/scripts/shell_call_hla_type .
+
+ sed -i "171s/TMP_DIR=.*/TMP_DIR=${tmpDir}/" shell_call_hla_type
+
+ bash shell_call_hla_type \
+ ${bamNormal} \
+ Unknown \
+ 1 \
+ ${genome_} \
+ STDFQ \
+ 0 \
+ ${outputDir}
+
+ mv winners.hla.txt ${outputPrefix}.hla.txt
+
+else
+
+ echo -e "HLA-A\thla_a_01_01_01_01\thla_a_01_01_01_01\nHLA-B\thla_b_15_02_01\thla_b_15_02_01\nHLA-C\thla_c_01_02_01\thla_c_01_02_01" > ${outputPrefix}.hla.txt
+
+fi
+ """
+}
diff --git a/modules/process/MSI/RunMsiSensor.nf b/modules/process/MSI/RunMsiSensor.nf
new file mode 100644
index 00000000..e2e5b76b
--- /dev/null
+++ b/modules/process/MSI/RunMsiSensor.nf
@@ -0,0 +1,20 @@
+process RunMsiSensor {
+ tag "${idTumor + "__" + idNormal}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict), path(msiSensorList)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.msisensor.tsv"), emit: msi4MetaDataParser
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ """
+ msisensor msi \
+ -d ${msiSensorList} \
+ -t ${bamTumor} \
+ -n ${bamNormal} \
+ -o ${outputPrefix}.msisensor.tsv
+ """
+}
diff --git a/modules/process/MetaParse/MetaDataParser.nf b/modules/process/MetaParse/MetaDataParser.nf
new file mode 100644
index 00000000..8d1739da
--- /dev/null
+++ b/modules/process/MetaParse/MetaDataParser.nf
@@ -0,0 +1,30 @@
+process MetaDataParser {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${idTumor}__${idNormal}/meta_data/", mode: params.publishDirMode, pattern: "*.sample_data.txt"
+
+ input:
+ tuple val(idNormal), val(target), val(idTumor), path(purityOut), path(mafFile), path(qcOutput), path(msifile), path(mutSig), val(placeHolder), path(polysolverFile), path(codingBed)
+
+ output:
+ path("*.sample_data.txt"), emit: MetaDataOutput
+ tuple val(placeHolder), val(idTumor), val(idNormal), path("*.sample_data.txt"), emit: MetaData4Aggregate
+
+ script:
+ codingRegionsBed = codingBed
+ """
+ create_metadata_file.py \
+ --sampleID ${idTumor}__${idNormal} \
+ --tumorID ${idTumor} \
+ --normalID ${idNormal} \
+ --facetsPurity_out ${purityOut} \
+ --facetsQC ${qcOutput} \
+ --MSIsensor_output ${msifile} \
+ --mutational_signatures_output ${mutSig} \
+ --polysolver_output ${polysolverFile} \
+ --MAF_input ${mafFile} \
+ --coding_baits_BED ${codingRegionsBed}
+
+ mv ${idTumor}__${idNormal}_metadata.txt ${idTumor}__${idNormal}.sample_data.txt
+ """
+}
diff --git a/modules/process/MutSig/RunMutationSignatures.nf b/modules/process/MutSig/RunMutationSignatures.nf
new file mode 100644
index 00000000..1f56ef47
--- /dev/null
+++ b/modules/process/MutSig/RunMutationSignatures.nf
@@ -0,0 +1,18 @@
+process RunMutationSignatures {
+ tag "${idTumor + "__" + idNormal}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(maf)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.mutsig.txt"), emit: mutSig4MetaDataParser
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ """
+ maf2cat2.R ${outputPrefix}.somatic.maf \
+ ${outputPrefix}.trinucmat.txt
+ tempoSig.R --cosmic_${params.cosmic} --pvalue --nperm 10000 --seed 132 ${outputPrefix}.trinucmat.txt \
+ ${outputPrefix}.mutsig.txt
+ """
+}
diff --git a/modules/process/QC/QcAlfred.nf b/modules/process/QC/QcAlfred.nf
new file mode 100644
index 00000000..deeefe29
--- /dev/null
+++ b/modules/process/QC/QcAlfred.nf
@@ -0,0 +1,43 @@
+process QcAlfred {
+ tag "${idSample + "@" + "ignore_rg_" + ignore_rg }"
+
+ publishDir "${params.outDir}/bams/${idSample}/alfred", mode: params.publishDirMode
+
+ input:
+ each ignore_rg
+ tuple val(idSample), val(target), path(bam), path(bai), path(targets), path(targetsIndex)
+ path(genomeFile)
+
+ output:
+ tuple val(idSample), path("${idSample}.alfred*tsv.gz"), emit: bamsQcStats4Aggregate
+ tuple val(idSample), path("${idSample}.alfred*tsv.gz"), path("${idSample}.alfred*tsv.gz.pdf"), emit: alfredOutput
+
+ script:
+ if (workflow.profile == "juno") {
+ if (bam.size() > 200.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (bam.size() < 100.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+
+ options = ""
+ if (params.assayType == "exome") {
+ options = "--bed ${targets}"
+ }
+ def ignore = ignore_rg ? "--ignore" : ""
+ def outfile = ignore_rg ? "${idSample}.alfred.tsv.gz" : "${idSample}.alfred.per_readgroup.tsv.gz"
+ """
+ alfred qc ${options} \
+ --reference ${genomeFile} \
+ ${ignore} \
+ --outfile ${outfile} \
+ ${bam} && \
+ Rscript --no-init-file /opt/alfred/scripts/stats.R ${outfile}
+ """
+}
diff --git a/modules/process/QC/QcCollectHsMetrics.nf b/modules/process/QC/QcCollectHsMetrics.nf
new file mode 100644
index 00000000..7824fb16
--- /dev/null
+++ b/modules/process/QC/QcCollectHsMetrics.nf
@@ -0,0 +1,45 @@
+process QcCollectHsMetrics {
+ tag "${idSample}"
+
+ publishDir "${params.outDir}/bams/${idSample}/collecthsmetrics", mode: params.publishDirMode
+
+ input:
+ tuple val(idSample), val(target), path(bam), path(bai), path(targetsList), path(baitsList)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idSample), path("${idSample}.hs_metrics.txt"), emit: collectHsMetricsOutput
+
+ when: params.assayType == "exome"
+
+ script:
+ if (workflow.profile == "juno") {
+ if (bam.size() > 200.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (bam.size() < 100.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+
+ memMultiplier = params.mem_per_core ? task.cpus : 1
+ javaOptions = "--java-options '-Xmx" + task.memory.toString().split(" ")[0].toInteger() * memMultiplier + "g'"
+
+ baitIntervals = "${baitsList}"
+ targetIntervals = "${targetsList}"
+ """
+ gatk CollectHsMetrics \
+ ${javaOptions} \
+ --COVERAGE_CAP 1000 \
+ --TMP_DIR ./ \
+ --INPUT ${bam} \
+ --OUTPUT ${idSample}.hs_metrics.txt \
+ --REFERENCE_SEQUENCE ${genomeFile} \
+ --BAIT_INTERVALS ${baitIntervals} \
+ --TARGET_INTERVALS ${targetIntervals}
+ """
+}
diff --git a/modules/process/QC/QcConpair.nf b/modules/process/QC/QcConpair.nf
new file mode 100644
index 00000000..afb8804f
--- /dev/null
+++ b/modules/process/QC/QcConpair.nf
@@ -0,0 +1,51 @@
+process QcConpair {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outPrefix}/conpair/", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumor), val(idNormal), path(pileupTumor), path(pileupNormal)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idTumor), val(idNormal), path("${outPrefix}.{concordance,contamination}.txt"), emit: conpairOutput
+ tuple val(idTumor), val(idNormal), path("${outPrefix}.concordance.txt"), path("${outPrefix}.contamination.txt"), emit: conpair4Aggregate
+
+ script:
+ outPrefix = "${idTumor}__${idNormal}"
+ conpairPath = "/usr/bin/conpair"
+
+ markersTxt = ""
+ if (params.genome == "GRCh37") {
+ markersTxt = "${conpairPath}/data/markers/GRCh37.autosomes.phase3_shapeit2_mvncall_integrated.20130502.SNV.genotype.sselect_v4_MAF_0.4_LD_0.8.txt"
+ }
+ else {
+ markersTxt = "${conpairPath}/data/markers/GRCh38.autosomes.phase3_shapeit2_mvncall_integrated.20130502.SNV.genotype.sselect_v4_MAF_0.4_LD_0.8.liftover.txt"
+ }
+
+ """
+ touch .Rprofile # calls to R inside the python scripts make this necessary to avoid loading user .Rprofile
+
+ # Make pairing file
+ echo "${idNormal}\t${idTumor}" > pairing.txt
+
+ # Verify concordance
+ ${conpairPath}/scripts/verify_concordances.py \
+ --tumor_pileup=${pileupTumor} \
+ --normal_pileup=${pileupNormal} \
+ --markers=${markersTxt} \
+ --pairing=pairing.txt \
+ --normal_homozygous_markers_only \
+ --outpre=${outPrefix}
+
+ ${conpairPath}/scripts/estimate_tumor_normal_contaminations.py \
+ --tumor_pileup=${pileupTumor} \
+ --normal_pileup=${pileupNormal} \
+ --markers=${markersTxt} \
+ --pairing=pairing.txt \
+ --outpre=${outPrefix}
+
+ mv ${outPrefix}_concordance.txt ${outPrefix}.concordance.txt
+ mv ${outPrefix}_contamination.txt ${outPrefix}.contamination.txt
+ """
+}
diff --git a/modules/process/QC/QcConpairAll.nf b/modules/process/QC/QcConpairAll.nf
new file mode 100644
index 00000000..a874123e
--- /dev/null
+++ b/modules/process/QC/QcConpairAll.nf
@@ -0,0 +1,49 @@
+process QcConpairAll {
+ tag "${idTumor + "@" + idNormal}"
+
+ input:
+ tuple val(idTumor), val(idNormal_noUse), path(pileupTumor), val(idTumor_noUse), val(idNormal), path(pileupNormal)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idTumor), val(idNormal), path("${outPrefix}.{concordance,contamination}.txt"), emit: conpairAllOutput
+ tuple val(idTumor), val(idNormal), path("${outPrefix}.concordance.txt"), path("${outPrefix}.contamination.txt"), emit: conpairAll4Aggregate
+
+ script:
+ outPrefix = "${idTumor}__${idNormal}"
+ conpairPath = "/usr/bin/conpair"
+
+ markersTxt = ""
+ if (params.genome == "GRCh37") {
+ markersTxt = "${conpairPath}/data/markers/GRCh37.autosomes.phase3_shapeit2_mvncall_integrated.20130502.SNV.genotype.sselect_v4_MAF_0.4_LD_0.8.txt"
+ }
+ else {
+ markersTxt = "${conpairPath}/data/markers/GRCh38.autosomes.phase3_shapeit2_mvncall_integrated.20130502.SNV.genotype.sselect_v4_MAF_0.4_LD_0.8.liftover.txt"
+ }
+
+ """
+ touch .Rprofile # calls to R inside the python scripts make this necessary to avoid loading user .Rprofile
+
+ # Make pairing file
+ echo "${idNormal}\t${idTumor}" > pairing.txt
+
+ # Verify concordance
+ ${conpairPath}/scripts/verify_concordances.py \
+ --tumor_pileup=${pileupTumor} \
+ --normal_pileup=${pileupNormal} \
+ --markers=${markersTxt} \
+ --pairing=pairing.txt \
+ --normal_homozygous_markers_only \
+ --outpre=${outPrefix}
+
+ ${conpairPath}/scripts/estimate_tumor_normal_contaminations.py \
+ --tumor_pileup=${pileupTumor} \
+ --normal_pileup=${pileupNormal} \
+ --markers=${markersTxt} \
+ --pairing=pairing.txt \
+ --outpre=${outPrefix}
+
+ mv ${outPrefix}_concordance.txt ${outPrefix}.concordance.txt
+ mv ${outPrefix}_contamination.txt ${outPrefix}.contamination.txt
+ """
+}
diff --git a/modules/process/QC/QcPileup.nf b/modules/process/QC/QcPileup.nf
new file mode 100644
index 00000000..8e77968a
--- /dev/null
+++ b/modules/process/QC/QcPileup.nf
@@ -0,0 +1,40 @@
+process QcPileup {
+ tag "${idSample}"
+
+ publishDir "${params.outDir}/bams/${idSample}/pileup/", mode: params.publishDirMode
+
+ input:
+ tuple val(idSample), val(target), path(bam), path(bai)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idSample), path("${idSample}.pileup"), emit: pileupOutput
+
+ script:
+ gatkPath = "/usr/bin/GenomeAnalysisTK.jar"
+ conpairPath = "/usr/bin/conpair"
+ markersBed = ""
+ if (params.genome == "GRCh37") {
+ markersBed = "${conpairPath}/data/markers/GRCh37.autosomes.phase3_shapeit2_mvncall_integrated.20130502.SNV.genotype.sselect_v4_MAF_0.4_LD_0.8.bed"
+ }
+ else {
+ markersBed = "${conpairPath}/data/markers/GRCh38.autosomes.phase3_shapeit2_mvncall_integrated.20130502.SNV.genotype.sselect_v4_MAF_0.4_LD_0.8.liftover.bed"
+ }
+
+ if (params.mem_per_core) {
+ mem = task.memory.toString().split(" ")[0].toInteger() - 1
+ }
+ else {
+ mem = (task.memory.toString().split(" ")[0].toInteger()/task.cpus).toInteger() - 1
+ }
+ javaMem = "${mem}g"
+ """
+ ${conpairPath}/scripts/run_gatk_pileup_for_sample.py \
+ --gatk=${gatkPath} \
+ --bam=${bam} \
+ --markers=${markersBed} \
+ --reference=${genomeFile} \
+ --xmx_java=${javaMem} \
+ --outfile=${idSample}.pileup
+ """
+}
diff --git a/modules/process/QC/QcQualimap.nf b/modules/process/QC/QcQualimap.nf
new file mode 100644
index 00000000..a84356fa
--- /dev/null
+++ b/modules/process/QC/QcQualimap.nf
@@ -0,0 +1,53 @@
+process QcQualimap {
+ tag "${idSample}"
+
+ publishDir "${params.outDir}/bams/${idSample}/qualimap", mode: params.publishDirMode, pattern: "*.{html,tar.gz}"
+ publishDir "${params.outDir}/bams/${idSample}/qualimap", mode: params.publishDirMode, pattern: "*/*"
+
+ input:
+ tuple val(idSample), val(target), path(bam), path(bai), path(targetsBed)
+
+ output:
+ tuple val(idSample), path("${idSample}_qualimap_rawdata.tar.gz"), emit: qualimap4Process
+ tuple val(idSample), path("*.html"), path("css/*"), path("images_qualimapReport/*"), emit: qualimapOutput
+
+
+ script:
+ if (params.assayType == "exome"){
+ gffOptions = "-gff ${targetsBed}"
+ nr = 750
+ nw = 300
+ } else {
+ gffOptions = "-gd HUMAN"
+ nr = 500
+ nw = 300
+ }
+ availMem = task.cpus * task.memory.toString().split(" ")[0].toInteger()
+ // javaMem = availMem > 20 ? availMem - 4 : ( availMem > 10 ? availMem - 2 : ( availMem > 1 ? availMem - 1 : 1 ))
+ javaMem = availMem > 20 ? (availMem * 0.75).round() : ( availMem > 1 ? availMem - 1 : 1 )
+ if (workflow.profile == "juno") {
+ if (bam.size() > 200.GB) {
+ task.time = { params.maxWallTime }
+ }
+ else if (bam.size() < 100.GB) {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
+ }
+ else {
+ task.time = task.exitStatus.toString() in params.wallTimeExitCode ? { params.maxWallTime } : { params.medWallTime }
+ }
+ task.time = task.attempt < 3 ? task.time : { params.maxWallTime }
+ }
+ """
+ qualimap bamqc \
+ -bam ${bam} \
+ ${gffOptions} \
+ -outdir ${idSample} \
+ -nt ${ task.cpus * 2 } \
+ -nw ${nw} \
+ -nr ${nr} \
+ --java-mem-size=${javaMem}G
+
+ mv ${idSample}/* .
+ tar -czf ${idSample}_qualimap_rawdata.tar.gz genome_results.txt raw_data_qualimapReport/*
+ """
+}
diff --git a/modules/process/QC/SampleRunMultiQC.nf b/modules/process/QC/SampleRunMultiQC.nf
new file mode 100644
index 00000000..9053e19f
--- /dev/null
+++ b/modules/process/QC/SampleRunMultiQC.nf
@@ -0,0 +1,57 @@
+process SampleRunMultiQC {
+ tag "${idSample}"
+ label 'multiqc_process'
+
+ publishDir "${params.outDir}/bams/${idSample}/multiqc", mode: params.publishDirMode
+
+ input:
+ tuple val(idSample), path(alfredRGNTsvFile), path(alfredRGYTsvFile), path(fastpJsonFile), path(qualimapFolder), file(hsmetricsFile)
+ tuple path("exome_multiqc_config.yaml"), path("wgs_multiqc_config.yaml"), path("tempoLogo.png")
+
+ output:
+ tuple val(idSample), path("*multiqc_report*.html"), path("*multiqc_data*.zip"), emit: sample_multiqc_report
+ tuple val(idSample), path("${idSample}.QC_Status.txt")
+
+ script:
+ if (params.assayType == "exome") {
+ assay = "exome"
+ }
+ else {
+ assay = 'wgs'
+ }
+ """
+ for i in ./*_qualimap_rawdata.tar.gz ; do
+ newFolder=\$(basename \$i | rev | cut -f 3- -d. | cut -f 3- -d_ | rev )
+ mkdir -p qualimap/\$newFolder
+ tar -xzf \$i -C qualimap/\$newFolder
+ done
+
+ parse_alfred.py --alfredfiles *alfred*tsv.gz
+ mkdir -p ignoreFolder
+ find . -maxdepth 1 \\( -name 'CO_ignore*mqc.yaml' -o -name 'IS_*mqc.yaml' -o -name 'GC_ignore*mqc.yaml' -o -name 'ME_aware_mqc.yaml' \\) -type f -print0 | xargs -0r mv -t ignoreFolder
+ if [[ "${params.assayType}" == "exome" ]] ; then
+ find . -maxdepth 1 -name 'CM_*mqc.yaml' -type f -print0 | xargs -0r mv -t ignoreFolder
+ fi
+
+ mkdir -p fastp_original
+ for i in `find . -maxdepth 1 -name "*fastp.json"` ; do
+ mv \$i fastp_original
+ inname=fastp_original/\$(basename \$i)
+ clean_fastp.py \$inname \$i
+ done
+
+ echo -e "\\tCoverage" > coverage_split.txt
+ cover=\$(grep -i "mean cover" ./qualimap/${idSample}/genome_results.txt | cut -f 2 -d"=" | sed "s/\\s*//g" | tr -d "X" | tr -d ",")
+ echo -e "${idSample}\\t\${cover}" >> coverage_split.txt
+
+ cp ${assay}_multiqc_config.yaml multiqc_config.yaml
+
+ multiqc . -x ignoreFolder/ -x fastp_original/
+ general_stats_parse.py --print-criteria
+ rm -rf multiqc_report.html multiqc_data
+
+ multiqc . --cl_config "title: \\"Sample MultiQC Report\\"" --cl_config "subtitle: \\"${idSample} QC\\"" --cl_config "intro_text: \\"Aggregate results from Tempo QC analysis\\"" --cl_config "report_comment: \\"This report includes FASTQ and alignment statistics for the sample ${idSample}." > vcf.header
+ echo -e "##INFO=" >> vcf.header
+ echo -e "##INFO=" >> vcf.header
+ echo -e "##INFO=" > vcf.rm.header
+ echo -e "##INFO=" > vcf.map.header
+ echo -e "##INFO=" > vcf.pon.header
+ echo -e "##FORMAT=" > vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+ echo -e "##FORMAT=" > vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+ echo -e "##FORMAT=" >> vcf.ad_n.header
+
+ # Get set differences of variant calls:
+ # 0000: MuTect2 only
+ # 0001: Strelka2 only
+ # 0002: MuTect2 calls shared by Strelka2
+ # 0003: Strelka2 calls shared by MuTect2
+
+ bcftools isec \
+ --output-type z \
+ --prefix ${isecDir} \
+ ${mutectCombinedVcf} ${strelkaVcf}
+
+ bcftools annotate \
+ --annotations ${isecDir}/0003.vcf.gz \
+ --include 'FILTER!=\"PASS\"' \
+ --mark-sites \"+Strelka2FILTER\" \
+ -k \
+ --output-type z \
+ --output ${isecDir}/0003.annot.vcf.gz \
+ ${isecDir}/0003.vcf.gz
+
+ bcftools annotate \
+ --header-lines vcf.header \
+ --annotations ${isecDir}/0000.vcf.gz \
+ --mark-sites +MuTect2 \
+ --output-type z \
+ --output ${isecDir}/0000.annot.vcf.gz \
+ ${isecDir}/0000.vcf.gz
+
+ bcftools annotate \
+ --header-lines vcf.header \
+ --annotations ${isecDir}/0002.vcf.gz \
+ --mark-sites \"+MuTect2;Strelka2\" \
+ --output-type z \
+ --output ${isecDir}/0002.tmp.vcf.gz \
+ ${isecDir}/0002.vcf.gz
+
+ tabix --preset vcf ${isecDir}/0002.tmp.vcf.gz
+ tabix --preset vcf ${isecDir}/0003.annot.vcf.gz
+
+ bcftools annotate \
+ --annotations ${isecDir}/0003.annot.vcf.gz \
+ --columns +INFO,+FORMAT,Strelka2FILTER \
+ --output-type z \
+ --output ${isecDir}/0002.annot.vcf.gz \
+ ${isecDir}/0002.tmp.vcf.gz
+
+ bcftools annotate \
+ --header-lines vcf.header \
+ --annotations ${isecDir}/0001.vcf.gz \
+ --mark-sites +Strelka2 \
+ --output-type z \
+ --output ${isecDir}/0001.annot.vcf.gz \
+ ${isecDir}/0001.vcf.gz
+
+ tabix --preset vcf ${isecDir}/0000.annot.vcf.gz
+ tabix --preset vcf ${isecDir}/0001.annot.vcf.gz
+ tabix --preset vcf ${isecDir}/0002.annot.vcf.gz
+
+ # Concatenate the different sets, annotate with blacklists
+ bcftools concat \
+ --allow-overlaps \
+ --rm-dups all \
+ ${isecDir}/0000.annot.vcf.gz \
+ ${isecDir}/0001.annot.vcf.gz \
+ ${isecDir}/0002.annot.vcf.gz | \
+ bcftools sort | \
+ bcftools annotate \
+ --header-lines vcf.rm.header \
+ --annotations ${repeatMasker} \
+ --columns CHROM,FROM,TO,RepeatMasker | \
+ bcftools annotate \
+ --header-lines vcf.map.header \
+ --annotations ${mapabilityBlacklist} \
+ --columns CHROM,FROM,TO,EncodeDacMapability \
+ --output-type z \
+ --output ${idTumor}.union.vcf.gz
+
+ tabix --preset vcf ${idTumor}.union.vcf.gz
+
+ # Add gnomAD annotation
+ bcftools annotate \
+ --annotations ${gnomad} \
+ --columns INFO \
+ --output-type z \
+ --output ${idTumor}.union.gnomad.vcf.gz \
+ ${idTumor}.union.vcf.gz
+
+ tabix --preset vcf ${idTumor}.union.gnomad.vcf.gz
+
+ # Add PoN annotation and flanking sequence
+ bcftools annotate \
+ --header-lines vcf.pon.header \
+ --annotations ${pon} \
+ --columns PoN:=AC_Het \
+ ${idTumor}.union.gnomad.vcf.gz | \
+ vt annotate_indels \
+ -r ${genomeFile} \
+ -o ${idTumor}.union.annot.vcf -
+
+ mv ${idTumor}.union.annot.vcf ${outputPrefix}.union.annot.vcf
+
+ # Do custom filter annotation, then filter variants
+ filter-vcf.py ${outputPrefix}.union.annot.vcf
+
+ bcftools filter \
+ --include 'FILTER=\"PASS\"' \
+ --output-type v \
+ --output ${outputPrefix}.vcf \
+ ${outputPrefix}.union.annot.filter.vcf
+
+ # Add normal read count, using all reads
+ GetBaseCountsMultiSample \
+ --thread ${task.cpus} \
+ --maq 0 \
+ --fasta ${genomeFile} \
+ --bam ${idTumor}:${bamTumor} \
+ --bam ${idNormal}:${bamNormal} \
+ --vcf ${outputPrefix}.vcf \
+ --output ${outputPrefix}.genotyped.vcf
+
+ bgzip ${outputPrefix}.vcf
+ bgzip ${outputPrefix}.genotyped.vcf
+ tabix --preset vcf ${outputPrefix}.vcf.gz
+ tabix --preset vcf ${outputPrefix}.genotyped.vcf.gz
+
+ bcftools annotate \
+ --annotations ${outputPrefix}.genotyped.vcf.gz \
+ --header-lines vcf.ad_n.header \
+ --columns FORMAT/alt_count_raw:=FORMAT/AD,FORMAT/ref_count_raw:=FORMAT/RD,FORMAT/alt_count_raw_fwd:=FORMAT/ADP,FORMAT/ref_count_raw_fwd:=FORMAT/RDP,FORMAT/alt_count_raw_rev:=FORMAT/ADN,FORMAT/ref_count_raw_rev:=FORMAT/RDN,FORMAT/depth_raw:=FORMAT/DP,FORMAT/depth_raw_fwd:=FORMAT/DPP,FORMAT/depth_raw_rev:=FORMAT/DPN \
+ --output-type v \
+ --output ${outputPrefix}.union.annot.filter.pass.vcf \
+ ${outputPrefix}.vcf.gz
+
+ cp ${outputPrefix}.union.annot.filter.pass.vcf ${outputPrefix}.pass.vcf
+ bgzip ${outputPrefix}.union.annot.vcf
+ bgzip ${outputPrefix}.union.annot.filter.vcf
+ bgzip ${outputPrefix}.union.annot.filter.pass.vcf
+ tabix --preset vcf ${outputPrefix}.union.annot.vcf.gz
+ tabix --preset vcf ${outputPrefix}.union.annot.filter.vcf.gz
+ tabix --preset vcf ${outputPrefix}.union.annot.filter.pass.vcf.gz
+
+ """
+}
diff --git a/modules/process/SNV/SomaticCombineMutect2Vcf.nf b/modules/process/SNV/SomaticCombineMutect2Vcf.nf
new file mode 100644
index 00000000..6915d253
--- /dev/null
+++ b/modules/process/SNV/SomaticCombineMutect2Vcf.nf
@@ -0,0 +1,35 @@
+process SomaticCombineMutect2Vcf {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${idTumor}__${idNormal}/mutect2", mode: params.publishDirMode
+
+ input:
+ tuple val(id), val(idTumor), val(idNormal), val(target), path(mutect2Vcf), path(mutect2VcfIndex), path(mutect2Stats)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outfile}"), path("${outfile}.tbi"), emit: mutect2CombinedVcfOutput
+
+ script:
+ idTumor = id.toString().split("__")[0]
+ idNormal = id.toString().split("@")[0].split("__")[1]
+ target = id.toString().split("@")[1]
+ outfile = "${idTumor}__${idNormal}.mutect2.vcf.gz"
+ """
+ bcftools concat \
+ --allow-overlaps \
+ *.filtered.vcf.gz | \
+ bcftools sort | \
+ bcftools norm \
+ --fasta-ref ${genomeFile} \
+ --check-ref s \
+ --multiallelics -both | \
+ bcftools norm --rm-dup all | \
+ bcftools view \
+ --samples ${idNormal},${idTumor} \
+ --output-type z \
+ --output-file ${outfile}
+
+ tabix --preset vcf ${outfile}
+ """
+}
diff --git a/modules/process/SNV/SomaticFacetsAnnotation.nf b/modules/process/SNV/SomaticFacetsAnnotation.nf
new file mode 100644
index 00000000..d2d7d7f6
--- /dev/null
+++ b/modules/process/SNV/SomaticFacetsAnnotation.nf
@@ -0,0 +1,35 @@
+process SomaticFacetsAnnotation {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_mutations/", mode: params.publishDirMode, pattern: "*.somatic.final.maf"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(hisens_rdata), val(facetsPath), path(maf)
+
+ output:
+ tuple val(idTumor), val(idNormal), path("${outputPrefix}.somatic.final.maf"), emit: finalMaf4Aggregate
+ path("file-size.txt"), emit: mafSize
+ path("${outputPrefix}.somatic.final.maf"), emit: finalMafOutput
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.somatic.final.maf"), emit: maf4MetaDataParser
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ """
+ if [ \$( cat ${maf} | wc -l ) -gt 1 ] ; then
+ Rscript --no-init-file /usr/bin/facets-suite/annotate-maf-wrapper.R \
+ --facets-output ${hisens_rdata} \
+ --maf-file ${maf} \
+ --facets-algorithm em \
+ --output ${outputPrefix}.facets.maf
+
+ Rscript --no-init-file /usr/bin/annotate-with-zygosity-somatic.R ${outputPrefix}.facets.maf ${outputPrefix}.facets.zygosity.maf
+
+ echo -e "${outputPrefix}\t`wc -l ${outputPrefix}.facets.zygosity.maf | cut -d ' ' -f1`" > file-size.txt
+
+ mv ${outputPrefix}.facets.zygosity.maf ${outputPrefix}.somatic.final.maf
+ else
+ cp ${maf} ${outputPrefix}.somatic.final.maf
+ echo -e "${outputPrefix}\t0" > file-size.txt
+ fi
+ """
+}
diff --git a/modules/process/SNV/SomaticRunStrelka2.nf b/modules/process/SNV/SomaticRunStrelka2.nf
new file mode 100644
index 00000000..36d27bd4
--- /dev/null
+++ b/modules/process/SNV/SomaticRunStrelka2.nf
@@ -0,0 +1,64 @@
+process SomaticRunStrelka2 {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/strelka2", mode: params.publishDirMode, pattern: "*.vcf.{gz,gz.tbi}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal), path(mantaCSI), path(mantaCSIi), path(targets), path(targetsIndex)
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path('*strelka2.vcf.gz'), path('*strelka2.vcf.gz.tbi'), emit: strelka4Combine
+ tuple path('*strelka2.vcf.gz'), path('*strelka2.vcf.gz.tbi'), emit: strelkaOutput
+
+ script:
+ options = ""
+ intervals = targets
+ if (params.assayType == "exome") {
+ options = "--exome"
+ }
+ outputPrefix = "${idTumor}__${idNormal}"
+ outfile = "${outputPrefix}.strelka2.vcf.gz"
+ """
+ configureStrelkaSomaticWorkflow.py \
+ ${options} \
+ --reportEVSFeatures \
+ --callRegions ${intervals} \
+ --referenceFasta ${genomeFile} \
+ --indelCandidates ${mantaCSI} \
+ --tumorBam ${bamTumor} \
+ --normalBam ${bamNormal} \
+ --runDir Strelka
+
+ python Strelka/runWorkflow.py \
+ --mode local \
+ --jobs ${task.cpus}
+
+ mv Strelka/results/variants/somatic.indels.vcf.gz \
+ Strelka_${outputPrefix}_somatic_indels.vcf.gz
+ mv Strelka/results/variants/somatic.indels.vcf.gz.tbi \
+ Strelka_${outputPrefix}_somatic_indels.vcf.gz.tbi
+ mv Strelka/results/variants/somatic.snvs.vcf.gz \
+ Strelka_${outputPrefix}_somatic_snvs.vcf.gz
+ mv Strelka/results/variants/somatic.snvs.vcf.gz.tbi \
+ Strelka_${outputPrefix}_somatic_snvs.vcf.gz.tbi
+
+ echo -e 'TUMOR ${idTumor}\\nNORMAL ${idNormal}' > samples.txt
+
+ bcftools concat \
+ --allow-overlaps \
+ Strelka_${outputPrefix}_somatic_indels.vcf.gz Strelka_${outputPrefix}_somatic_snvs.vcf.gz | \
+ bcftools reheader \
+ --samples samples.txt | \
+ bcftools sort | \
+ bcftools norm \
+ --fasta-ref ${genomeFile} \
+ --check-ref s \
+ --output-type z \
+ --output ${outfile}
+
+ tabix --preset vcf ${outfile}
+
+ if [ \$(vcf-validator ${outfile} 2>&1 | wc -l ) -gt 0 ] ; then exit 1 ; fi
+ """
+}
diff --git a/modules/process/SV/BRASS/SomaticRunBRASS.nf b/modules/process/SV/BRASS/SomaticRunBRASS.nf
new file mode 100644
index 00000000..bd24b06a
--- /dev/null
+++ b/modules/process/SV/BRASS/SomaticRunBRASS.nf
@@ -0,0 +1,83 @@
+process runBRASS {
+ tag "${idTumor}__${idNormal}"
+ label 'BRASS'
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/", mode: params.publishDirMode, pattern: "brass/*.{gz,tbi}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target),
+ path(bamTumor), path(baiTumor), path(basTumor),
+ path(bamNormal), path(baiNormal), path(basNormal),
+ path(BrassInputTmp),
+ path(BrassInputProgress),
+ path(BrassCoverTmp),
+ path(BrassCoverProgress),
+ path(ascatSampleStatistics)
+ path(genomeFile)
+ path(genomeIndex)
+ path("brassRefDir")
+ path(vagrentRefDir)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("brass/*.{vcf.gz,vcf.gz.tbi}"), emit: BRASSOutput
+ tuple val(idTumor), val(idNormal), val(target), path("${idTumor}__${idNormal}.brass.annot.vcf.gz"), path("${idTumor}__${idNormal}.brass.annot.vcf.gz.tbi"), emit: BRASS4Combine
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ if (params.genome in ["GRCh37","smallGRCh37"]){
+ species = "HUMAN"
+ assembly = 37
+ }
+ else if (params.genome in ["GRCh38"]) {
+ species = "HUMAN"
+ assembly = 38
+ } else { // not sure if run will complete with these params.
+ species = params.genome
+ assembly = params.genome
+ }
+ """
+ BrassInputResults=( ${BrassInputTmp.join(" ")} )
+ BrassInputProgress=( ${BrassInputProgress.join(" ")} )
+ BrassCoverCover=( ${BrassCoverTmp.join(" ")} )
+ BrassCoverProgress=( ${BrassCoverProgress.join(" ")} )
+
+ export TMPDIR=\$(pwd)/tmp ; mkdir -p \$TMPDIR brass/tmpBrass/progress brass/tmpBrass/cover
+ for i in "\${BrassCoverCover[@]}" ; do
+ mv \$i brass/tmpBrass/cover
+ done
+ for i in "\${BrassCoverProgress[@]}" ; do
+ mv \$i brass/tmpBrass/progress
+ done
+ for i in "\${BrassInputResults[@]}" ; do
+ mv \$i brass/tmpBrass
+ done
+ for i in "\${BrassInputProgress[@]}" ; do
+ mv \$i brass/tmpBrass/progress
+ done
+ brass.pl -j 4 -k 4 -c ${task.cpus} \\
+ -d brassRefDir/HiDepth.bed.gz \\
+ -f brassRefDir/brass_np.groups.gz \\
+ -g ${genomeFile} \\
+ -s "${species}" -as "${assembly}" -pr "WGS" \\
+ -g_cache ${vagrentRefDir}/vagrent.cache.gz \\
+ -vi brassRefDir/viral.genomic.fa.2bit \\
+ -mi brassRefDir/all_ncbi_bacteria \\
+ -b brassRefDir/500bp_windows.gc.bed.gz \\
+ -ct brassRefDir/CentTelo.tsv \\
+ -cb brassRefDir/cytoband.txt \\
+ -t ${bamTumor} \\
+ -n ${bamNormal} \\
+ -ss ${ascatSampleStatistics} \\
+ -o brass
+
+ echo -e "TUMOUR ${idTumor}\\nNORMAL ${idNormal}" > brass.samplenames.tsv
+ bcftools reheader \\
+ --samples brass.samplenames.tsv \\
+ brass/${idTumor}_vs_${idNormal}.annot.vcf.gz | \\
+ bcftools filter \\
+ -O z \\
+ -e "POS=0 | ( INFO/BKDIST > 0 & INFO/BKDIST = POS + 1 ) " \\
+ > ${idTumor}__${idNormal}.brass.annot.vcf.gz
+ bcftools index -f -t ${idTumor}__${idNormal}.brass.annot.vcf.gz
+ """
+}
diff --git a/modules/process/SV/BRASS/SomaticRunBRASSCover.nf b/modules/process/SV/BRASS/SomaticRunBRASSCover.nf
new file mode 100644
index 00000000..5c44994a
--- /dev/null
+++ b/modules/process/SV/BRASS/SomaticRunBRASSCover.nf
@@ -0,0 +1,56 @@
+process runBRASSCover {
+ tag "${idTumor}__${idNormal}@${brassCoverIndex}"
+ label 'BRASS'
+
+ input:
+ each brassCoverIndex
+ val(brassCoverLimit)
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(basTumor), path(bamNormal), path(baiNormal), path(basNormal)
+ path(genomeFile)
+ path(genomeIndex)
+ path("brassRefDir")
+ path(vagrentRefDir)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("brass/tmpBrass/cover/*.*"), path("brass/tmpBrass/progress/*.*")
+
+ script:
+ if (params.genome in ["GRCh37","smallGRCh37"]){
+ species = "HUMAN"
+ assembly = 37
+ }
+ else if (params.genome in ["GRCh38"]) {
+ species = "HUMAN"
+ assembly = 38
+ } else { // not sure if run will complete with these params.
+ species = params.genome
+ assembly = params.genome
+ }
+ if (brassCoverLimit == 1 ) {
+ indexParam = ""
+ } else {
+ indexParam = "-i ${brassCoverIndex} -l ${brassCoverLimit}"
+ }
+ """
+ export TMPDIR=\$(pwd)/tmp ; mkdir -p \$TMPDIR brass
+ for i in rho Ploidy GenderChr GenderChrFound ; do echo \$i ;done > samplestatistics.txt
+ brass.pl -j 4 -k 4 -c ${task.cpus} \\
+ -d brassRefDir/HiDepth.bed.gz \\
+ -f brassRefDir/brass_np.groups.gz \\
+ -g ${genomeFile} \\
+ -s "${species}" -as "${assembly}" -pr "WGS" \\
+ -g_cache ${vagrentRefDir}/vagrent.cache.gz \\
+ -vi brassRefDir/viral.genomic.fa.2bit \\
+ -mi brassRefDir/all_ncbi_bacteria \\
+ -b brassRefDir/500bp_windows.gc.bed.gz \\
+ -ct brassRefDir/CentTelo.tsv \\
+ -cb brassRefDir/cytoband.txt \\
+ -t ${bamTumor} \\
+ -n ${bamNormal} \\
+ -ss samplestatistics.txt \\
+ -o brass \\
+ -p cover \\
+ ${indexParam}
+ """
+}
+
diff --git a/modules/process/SV/BRASS/SomaticRunBRASSInput.nf b/modules/process/SV/BRASS/SomaticRunBRASSInput.nf
new file mode 100644
index 00000000..9a6eb41b
--- /dev/null
+++ b/modules/process/SV/BRASS/SomaticRunBRASSInput.nf
@@ -0,0 +1,50 @@
+process runBRASSInput {
+ tag "${idTumor}__${idNormal}@${inputIndex}"
+ label 'BRASS'
+
+ input:
+ each inputIndex
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(basTumor), path(bamNormal), path(baiNormal), path(basNormal)
+ path(genomeFile)
+ path(genomeIndex)
+ path("brassRefDir")
+ path(vagrentRefDir)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("brass/tmpBrass/*.*"), path("brass/tmpBrass/progress/*.*")
+
+ script:
+ if (params.genome in ["GRCh37","smallGRCh37"]){
+ species = "HUMAN"
+ assembly = 37
+ }
+ else if (params.genome in ["GRCh38"]) {
+ species = "HUMAN"
+ assembly = 38
+ } else { // not sure if run will complete with these params.
+ species = params.genome
+ assembly = params.genome
+ }
+ """
+ export TMPDIR=\$(pwd)/tmp ; mkdir -p \$TMPDIR brass
+ for i in rho Ploidy GenderChr GenderChrFound ; do echo \$i ;done > samplestatistics.txt
+ brass.pl -j 4 -k 4 -c ${task.cpus} \\
+ -d brassRefDir/HiDepth.bed.gz \\
+ -f brassRefDir/brass_np.groups.gz \\
+ -g ${genomeFile} \\
+ -s "${species}" -as "${assembly}" -pr "WGS" \\
+ -g_cache ${vagrentRefDir}/vagrent.cache.gz \\
+ -vi brassRefDir/viral.genomic.fa.2bit \\
+ -mi brassRefDir/all_ncbi_bacteria \\
+ -b brassRefDir/500bp_windows.gc.bed.gz \\
+ -ct brassRefDir/CentTelo.tsv \\
+ -cb brassRefDir/cytoband.txt \\
+ -t ${bamTumor} \\
+ -n ${bamNormal} \\
+ -ss samplestatistics.txt \\
+ -o brass \\
+ -p input \\
+ -i ${inputIndex} -l 2
+ """
+}
+
diff --git a/modules/process/SV/BRASS/generateBasFile.nf b/modules/process/SV/BRASS/generateBasFile.nf
new file mode 100644
index 00000000..275dc588
--- /dev/null
+++ b/modules/process/SV/BRASS/generateBasFile.nf
@@ -0,0 +1,21 @@
+process generateBasFile {
+ tag "${idSample}"
+
+ input:
+ tuple val(idSample), val(target), path(bam), path(bai)
+ path(genomeFile)
+ path(genomeIndex)
+
+ output:
+ tuple val(idSample), val(target), path("*.bas")
+
+ script:
+ """
+ bam_stats \\
+ -i ${bam} \\
+ -o ${bam}.bas \\
+ -r ${genomeIndex} \\
+ -@ ${ task.cpus > 1 ? task.cpus - 1 : task.cpus }
+ """
+}
+
diff --git a/modules/process/SV/DellyCombine.nf b/modules/process/SV/DellyCombine.nf
new file mode 100644
index 00000000..d62d7a53
--- /dev/null
+++ b/modules/process/SV/DellyCombine.nf
@@ -0,0 +1,26 @@
+process DellyCombine {
+ tag { outputPrefix }
+
+ publishDir "${params.outDir}/${mode}/${outputPrefix}/delly", mode: params.publishDirMode, pattern: "*.delly.vcf.{gz,gz.tbi}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(inputVcfs), path(inputTbis)
+ val(mode)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.delly.vcf.gz"), path("${outputPrefix}.delly.vcf.gz.tbi")
+
+ script:
+ outputPrefix = [idTumor,idNormal].unique()
+ outputPrefix.remove("")
+ outputPrefix = outputPrefix.join("__")
+ """
+ bcftools concat \\
+ --allow-overlaps \\
+ --output-type z \\
+ --output ${outputPrefix}.delly.vcf.gz \\
+ *.delly.vcf.gz
+
+ tabix --preset vcf ${outputPrefix}.delly.vcf.gz
+ """
+}
diff --git a/modules/process/SV/SomaticAnnotateSVBedpe.nf b/modules/process/SV/SomaticAnnotateSVBedpe.nf
new file mode 100644
index 00000000..ecd3491d
--- /dev/null
+++ b/modules/process/SV/SomaticAnnotateSVBedpe.nf
@@ -0,0 +1,92 @@
+process SomaticAnnotateSVBedpe {
+ tag "${idTumor}__${idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs", mode: params.publishDirMode, pattern: "*.unfiltered.bedpe"
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs", mode: params.publishDirMode, pattern: "*.final.bedpe"
+
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bedpein)
+ path(repeatMasker)
+ path(mapabilityBlacklist)
+ path(svBlacklistBed)
+ path(svBlacklistBedpe)
+ path(svBlacklistFoldbackBedpe)
+ path(svBlacklistTEBedpe)
+ path(spliceSites)
+ path(custom_scripts)
+ val(genome)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.unfiltered.bedpe"), emit: SVAnnotBedpe
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.final.bedpe"), emit: SVAnnotBedpePass
+ tuple val(idTumor), val(idNormal), path("${outputPrefix}.final.bedpe"), emit: SVAnnotBedpe4Aggregate
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ genome_ = ["GRCh37","smallGRCh37"].contains(genome) ? "hg19" : genome == "GRCh38" ? "hg38" : "hg18"
+ """
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${mapabilityBlacklist} \\
+ --bedpe ${bedpein} \\
+ --tag mappability \\
+ --output ${outputPrefix}.combined.dac.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${repeatMasker} \\
+ --bedpe ${outputPrefix}.combined.dac.bedpe \\
+ --tag repeat_masker \\
+ --output ${outputPrefix}.combined.dac.rm.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistBed} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.bedpe \\
+ --tag pcawg_blacklist_bed \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.1.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistBedpe} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.1.bedpe \\
+ --tag pcawg_blacklist_bedpe \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.2.bedpe \\
+ --match-type both \\
+ --ignore-strand
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistFoldbackBedpe} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.2.bedpe \\
+ --tag pcawg_blacklist_fb_bedpe \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.3.bedpe \\
+ --match-type both
+
+ python ${custom_scripts}/filter_regions_bedpe.py \\
+ --blacklist-regions ${svBlacklistTEBedpe} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.3.bedpe \\
+ --tag pcawg_blacklist_te_bedpe \\
+ --output ${outputPrefix}.combined.dac.rm.pcawg.4.bedpe \\
+ --match-type either
+
+ python ${custom_scripts}/detect_cdna.py \\
+ --exon-junct ${spliceSites} \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.4.bedpe \\
+ --out-bedpe ${outputPrefix}.combined.dac.rm.pcawg.cdna.bedpe \\
+ --out ${outputPrefix}.contamination.tsv
+
+ python ${custom_scripts}/run_iannotatesv.py \\
+ --bedpe ${outputPrefix}.combined.dac.rm.pcawg.cdna.bedpe \\
+ --genome ${genome_} \\
+ --threads ${task.cpus * 2}
+
+ cp ${outputPrefix}.combined.dac.rm.pcawg.cdna.iannotate.bedpe \\
+ ${outputPrefix}.unfiltered.bedpe
+
+ awk -F"\\t" '\$1 ~ /#/ || \$12 == "PASS"' \\
+ ${outputPrefix}.unfiltered.bedpe > \\
+ ${outputPrefix}.final.bedpe
+
+ """
+
+}
diff --git a/modules/process/SV/SomaticDellyCall.nf b/modules/process/SV/SomaticDellyCall.nf
new file mode 100644
index 00000000..646b5d5f
--- /dev/null
+++ b/modules/process/SV/SomaticDellyCall.nf
@@ -0,0 +1,50 @@
+process SomaticDellyCall {
+ tag "${idTumor + "__" + idNormal + '@' + svType}"
+
+ publishDir "${params.outDir}/somatic/${idTumor}__${idNormal}/delly", mode: params.publishDirMode, pattern: "*.delly.vcf.{gz,gz.tbi}"
+
+ input:
+ each svType
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal)
+ tuple path(genomeFile), path(genomeIndex), path(svCallingExcludeRegions)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${idTumor}__${idNormal}_${svType}.delly.vcf.gz"), path("${idTumor}__${idNormal}_${svType}.delly.vcf.gz.tbi"), emit: dellyFilter4Combine
+ tuple path("${idTumor}__${idNormal}_${svType}.delly.vcf.gz"), path("${idTumor}__${idNormal}_${svType}.delly.vcf.gz.tbi"), emit: dellyOutput
+
+ script:
+ """
+ delly call \
+ --svtype ${svType} \
+ --genome ${genomeFile} \
+ --exclude ${svCallingExcludeRegions} \
+ --outfile ${idTumor}__${idNormal}_${svType}.bcf \
+ ${bamTumor} ${bamNormal}
+
+ echo "${idTumor}\ttumor\n${idNormal}\tcontrol" > samples.tsv
+
+ delly filter \
+ --filter somatic \
+ -a .05 \
+ --samples samples.tsv \
+ --outfile ${idTumor}__${idNormal}_${svType}.filter.bcf \
+ ${idTumor}__${idNormal}_${svType}.bcf
+
+ # Filter variants that have low supporting reads in the tumor or high supporting reads in the normal
+ # DV = discordant reads
+ # RV = split reads
+ bcftools view \\
+ -s ${idTumor},${idNormal} \\
+ ${idTumor}__${idNormal}_${svType}.filter.bcf | \\
+ bcftools filter \\
+ --soft-filter tumor_read_supp -m + \\
+ -e "FORMAT/DV[0] < 5 | FORMAT/RV[0] < 2" | \\
+ bcftools filter \\
+ --soft-filter normal_read_supp -m + \\
+ -e "FORMAT/DV[1] > 0 | FORMAT/RV[1] > 0" | \\
+ bcftools view --output-type z > \\
+ ${idTumor}__${idNormal}_${svType}.delly.vcf.gz
+
+ tabix --preset vcf ${idTumor}__${idNormal}_${svType}.delly.vcf.gz
+ """
+}
diff --git a/modules/process/SV/SomaticMergeSVs.nf b/modules/process/SV/SomaticMergeSVs.nf
new file mode 100644
index 00000000..484717e6
--- /dev/null
+++ b/modules/process/SV/SomaticMergeSVs.nf
@@ -0,0 +1,62 @@
+process SomaticMergeSVs {
+ tag "${idTumor}__${idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs/intermediate_files", mode: params.publishDirMode, pattern: "*.merged.vcf.{gz,gz.tbi}"
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs/intermediate_files", mode: params.publishDirMode, pattern: "*.merged.raw.vcf.{gz,gz.tbi}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target),
+ path(Vcfs), path(Tbis),
+ val(callerNames)
+ path(custom_scripts)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.merged.vcf.gz"), path("${outputPrefix}.merged.vcf.gz.tbi"), emit: SVCallsCombinedVcf
+ path("${outputPrefix}.merged.raw.vcf.{gz,gz.tbi}")
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ vcfMap = [:]
+ for (i in 1..callerNames.size()){
+ vcfMap.put(callerNames[i-1], Vcfs[i-1])
+ }
+ labelparam = callerNames.sort().join(",")
+ inVCFs = ""
+ for (i in callerNames.sort()){
+ inVCFs += " " + vcfMap[i]
+ }
+ passMin = callerNames.size() > 2 ? 2 : 1
+ """
+ mergesvvcf \\
+ -n -m 1 \\
+ -l ${labelparam} \\
+ -o ${outputPrefix}.merged.raw.vcf \\
+ -f -d -s -v \\
+ ${inVCFs}
+
+ cat ${outputPrefix}.merged.raw.vcf | \\
+ awk -F"\\t" -v OFS="\\t" '\$1 ~ /^#/ && \$1 !~ /^##/ && \$1 !~ /^#CHROM/{next;}{for(i=1; i<=NF; i++) if(\$i ~ /^ *\$/) \$i = "."; print \$0}' | \\
+ bcftools sort --temp-dir ./ \\
+ > ${outputPrefix}.merged.clean.anon.vcf
+
+ bcftools annotate \\
+ --set-id 'TEMPO_%INFO/SVTYPE\\_%CHROM\\_%POS\\_%INFO/CHR2\\_%INFO/END\\_%INFO/STRANDS' \\
+ -o ${outputPrefix}.merged.clean.vcf \\
+ ${outputPrefix}.merged.clean.anon.vcf
+
+ python ${custom_scripts}/filter-sv-vcf.py \\
+ --input ${outputPrefix}.merged.clean.vcf \\
+ --output ${outputPrefix}.merged.clean.corrected.vcf \\
+ --min ${passMin}
+
+ bcftools view \\
+ --samples ${idTumor},${idNormal} \\
+ --output-type z \\
+ --output-file ${outputPrefix}.merged.vcf.gz \\
+ ${outputPrefix}.merged.clean.corrected.vcf
+
+ tabix --preset vcf ${outputPrefix}.merged.vcf.gz
+
+ bcftools view -O z -o ${outputPrefix}.merged.raw.vcf.gz ${outputPrefix}.merged.raw.vcf
+ """
+}
diff --git a/modules/process/SV/SomaticRunClusterSV.nf b/modules/process/SV/SomaticRunClusterSV.nf
new file mode 100644
index 00000000..f79b20c9
--- /dev/null
+++ b/modules/process/SV/SomaticRunClusterSV.nf
@@ -0,0 +1,46 @@
+process SomaticRunClusterSV {
+ tag { outputPrefix }
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs", mode: params.publishDirMode, pattern: "*.clustered.bedpe"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bedpe)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target),
+ path("${outputPrefix}.sv_clusters_and_footprints.tsv"),
+ path("${outputPrefix}.sv_distance_pvals"),
+ path("${clusteredBedpe}"), emit: clusterSvOutput
+ tuple val(idTumor), val(idNormal),
+ path("${clusteredBedpe}"), emit: Bedpe4Aggregate
+
+ when: ["GRCh37","GRCh38","smallGRCh37"].contains(params.genome)
+
+ script:
+ outputPrefix = [idTumor,idNormal].unique()
+ outputPrefix.remove("")
+ outputPrefix = outputPrefix.join("__")
+ genome_ = params.genome == "GRCh37" || params.genome == 'smallGRCh37' ? "hs37d5" : "hg38"
+ clusteredBedpe = bedpe.getBaseName() + ".clustered.bedpe"
+ """
+ mkdir -p tmp
+ grep -v "^#" ${bedpe} | cut -f 1-10 > tmp/${outputPrefix}.bedpe
+ if [ \$(cat tmp/${outputPrefix}.bedpe | wc -l ) -lt 1 ] ; then
+ touch ${outputPrefix}.sv_clusters_and_footprints.tsv
+ touch ${outputPrefix}.sv_distance_pvals
+ else
+ Rscript /opt/ClusterSV/R/run_cluster_sv.R \\
+ -chr /opt/ClusterSV/references/${genome_}.chrom_sizes \\
+ -cen_telo /opt/ClusterSV/references/${genome_}_centromere_and_telomere_coords.txt \
+ -out ${outputPrefix} \
+ -bedpe tmp/${outputPrefix}.bedpe
+ fi
+
+ grep "^##" ${bedpe} > ${clusteredBedpe}
+ grep "^#CHROM" ${bedpe} | tr "\\n" "\\t" >> ${clusteredBedpe}
+ echo -e "cluster_id\\tcluster_total_count\\tfootprint_id_low\\tfootprint_id_high\\tcoord_footprint_id_low\\tcoord_footprint_id_high\\tclustersv_pval" \\
+ >> ${clusteredBedpe}
+ paste <(grep -v "^#" ${bedpe} ) \
+ <( cut -f 11- ${outputPrefix}.sv_clusters_and_footprints.tsv) >> ${clusteredBedpe}
+ """
+}
diff --git a/modules/process/SV/SomaticRunManta.nf b/modules/process/SV/SomaticRunManta.nf
new file mode 100644
index 00000000..b2561199
--- /dev/null
+++ b/modules/process/SV/SomaticRunManta.nf
@@ -0,0 +1,67 @@
+process SomaticRunManta {
+ tag "${idTumor + "__" + idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/manta", mode: params.publishDirMode, pattern: "*.manta.vcf.{gz,gz.tbi}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal)
+ tuple path(genomeFile), path(genomeIndex)
+ tuple path(svCallingIncludeRegions), path(svCallingIncludeRegionsIndex)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.manta.vcf.gz"), path("${outputPrefix}.manta.vcf.gz.tbi"), emit: manta4Combine
+ tuple val(idTumor), val(idNormal), val(target), path("*.candidateSmallIndels.vcf.gz"), path("*.candidateSmallIndels.vcf.gz.tbi"), emit: mantaToStrelka
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ options = ""
+ if (params.assayType == "exome") options = "--exome"
+ """
+ configManta.py \
+ ${options} \
+ --callRegions ${svCallingIncludeRegions} \
+ --referenceFasta ${genomeFile} \
+ --normalBam ${bamNormal} \
+ --tumorBam ${bamTumor} \
+ --runDir Manta
+
+ python Manta/runWorkflow.py \
+ --mode local \
+ --jobs ${task.cpus}
+
+ mv Manta/results/variants/candidateSmallIndels.vcf.gz \
+ Manta_${outputPrefix}.candidateSmallIndels.vcf.gz
+ mv Manta/results/variants/candidateSmallIndels.vcf.gz.tbi \
+ Manta_${outputPrefix}.candidateSmallIndels.vcf.gz.tbi
+ mv Manta/results/variants/candidateSV.vcf.gz \
+ Manta_${outputPrefix}.candidateSV.vcf.gz
+ mv Manta/results/variants/candidateSV.vcf.gz.tbi \
+ Manta_${outputPrefix}.candidateSV.vcf.gz.tbi
+ mv Manta/results/variants/diploidSV.vcf.gz \
+ Manta_${outputPrefix}.diploidSV.vcf.gz
+ mv Manta/results/variants/diploidSV.vcf.gz.tbi \
+ Manta_${outputPrefix}.diploidSV.vcf.gz.tbi
+ mv Manta/results/variants/somaticSV.vcf.gz \
+ ${outputPrefix}.manta.raw.vcf.gz
+ mv Manta/results/variants/somaticSV.vcf.gz.tbi \
+ ${outputPrefix}.manta.raw.vcf.gz.tbi
+
+
+ # Filter variants that have low supporting reads in the tumor or high supporting reads in the normal
+ # PR = discordant reads
+ # SR = split reads
+ bcftools view \\
+ -s ${idTumor},${idNormal} \\
+ ${outputPrefix}.manta.raw.vcf.gz | \\
+ bcftools filter \\
+ --soft-filter tumor_read_supp -m + \\
+ -e "FORMAT/PR[0:1] < 5 | FORMAT/SR[0:1] < 2" | \\
+ bcftools filter \\
+ --soft-filter normal_read_supp -m + \\
+ -e "FORMAT/PR[1:1] > 0 | FORMAT/SR[1:1] > 0" | \\
+ bcftools view --output-type z > \\
+ ${outputPrefix}.manta.vcf.gz
+
+ tabix --preset vcf ${outputPrefix}.manta.vcf.gz
+ """
+}
diff --git a/modules/process/SV/SomaticRunSVCircos.nf b/modules/process/SV/SomaticRunSVCircos.nf
new file mode 100644
index 00000000..482c5e6c
--- /dev/null
+++ b/modules/process/SV/SomaticRunSVCircos.nf
@@ -0,0 +1,27 @@
+process SomaticRunSVCircos {
+ tag "${idTumor}__${idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs/", mode: params.publishDirMode
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bedpe), path(cnv)
+ path(biocircos_script)
+ path(biocircos_Rmd)
+
+ output:
+ path("${outputPrefix}.circos.html")
+
+ when: ["GRCh37","GRCh38"].contains(params.genome)
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ genome_version = params.genome == 'GRCh38' ? "hg38" : "hg19"
+ """
+ Rscript ${biocircos_script} \\
+ -b ${bedpe} \\
+ -c ${cnv} \\
+ -s ${outputPrefix} \\
+ -g ${genome_version}
+
+ """
+}
diff --git a/modules/process/SV/SomaticRunSvABA.nf b/modules/process/SV/SomaticRunSvABA.nf
new file mode 100644
index 00000000..370e5ded
--- /dev/null
+++ b/modules/process/SV/SomaticRunSvABA.nf
@@ -0,0 +1,40 @@
+process SomaticRunSvABA {
+ tag "${idTumor}__${idNormal}"
+ publishDir "${params.outDir}/somatic/${idTumor}__${idNormal}/svaba", mode: params.publishDirMode, pattern: "*.{vcf.gz,vcf.gz.tbi}"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(bamTumor), path(baiTumor), path(bamNormal), path(baiNormal), path(targetsBed)
+ path(genomeFile)
+ path(genomeIndex)
+ path(genomeDict)
+ path(bwaIndex)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.reheader.svaba.somatic.sv.vcf.gz"), path("${outputPrefix}.reheader.svaba.somatic.sv.vcf.gz.tbi"), emit: SvABA4Combine
+ path("*.vcf.gz*"), emit: allVcfs
+ path("*.log"), emit: logs
+ path("*.txt.gz"), emit: supportingFiles
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ target_param = params.assayType == "genome" ? "" : "-k ${targetsBed} "
+ """
+ svaba run \\
+ -t "${bamTumor}" \\
+ -n "${bamNormal}" \\
+ -G "${genomeFile}" \\
+ -p "${task.cpus * 2}" \\
+ --id-string "${outputPrefix}" \\
+ ${target_param} \\
+ -z
+
+ rm -f *germline*
+
+ echo -e "${bamTumor} ${idTumor}\\n${bamNormal} ${idNormal}" > svaba.samplenames.tsv
+ bcftools reheader \\
+ --samples svaba.samplenames.tsv \\
+ --output ${outputPrefix}.reheader.svaba.somatic.sv.vcf.gz \\
+ ${outputPrefix}.svaba.somatic.sv.vcf.gz
+ bcftools index -f -t ${outputPrefix}.reheader.svaba.somatic.sv.vcf.gz
+ """
+}
diff --git a/modules/process/SV/SomaticSVVcf2Bedpe.nf b/modules/process/SV/SomaticSVVcf2Bedpe.nf
new file mode 100644
index 00000000..4c2c6378
--- /dev/null
+++ b/modules/process/SV/SomaticSVVcf2Bedpe.nf
@@ -0,0 +1,42 @@
+process SomaticSVVcf2Bedpe {
+ tag "${idTumor}__${idNormal}"
+
+ publishDir "${params.outDir}/somatic/${outputPrefix}/combined_svs/intermediate_files", mode: params.publishDirMode, pattern: "*.combined.bedpe"
+
+ input:
+ tuple val(idTumor), val(idNormal), val(target), path(vcfFile), path(tbiFile)
+
+ output:
+ tuple val(idTumor), val(idNormal), val(target), path("${outputPrefix}.combined.bedpe"), emit: SomaticCombinedUnfilteredBedpe
+
+ script:
+ outputPrefix = "${idTumor}__${idNormal}"
+ """
+ export LC_ALL=C
+
+ echo -e "${idTumor} TUMOR\\n${idNormal} NORMAL" > normalize.samplenames.tsv
+ bcftools reheader \\
+ --samples normalize.samplenames.tsv \\
+ --output reheader_${vcfFile} \\
+ ${vcfFile}
+
+ svtools vcftobedpe \\
+ -i reheader_${vcfFile} \\
+ -o ${outputPrefix}.combined.tmp.bedpe \\
+ -t ${outputPrefix}_tmp
+
+ if [ ! -s ${outputPrefix}.combined.tmp.bedpe ] ; then
+ echo -e "#CHROM_A\\tSTART_A\\tEND_A\\tCHROM_B\\tSTART_B\\tEND_B\\tID\\tQUAL\\tSTRAND_A\\tSTRAND_B\\tTYPE\\tFILTER\\tNAME_A\\tREF_A\\tALT_A\\tNAME_B\\tREF_B\\tALT_B\\tINFO_A\\tINFO_B\\tFORMAT\\tTUMOR\\tNORMAL" >> ${outputPrefix}.combined.tmp.bedpe
+ fi
+
+ zgrep "^##" ${vcfFile} | sed "s/##fileformat=*/##fileformat=BEDPE/g" > ${outputPrefix}.combined.unsorted.bedpe
+ grep -v "^##" ${outputPrefix}.combined.tmp.bedpe | \\
+ awk -F"\\t" -v tid="${idTumor}" -v nid="${idNormal}" -v OFS="\\t" 'NR == 1 {print \$0,"TUMOR_ID","NORMAL_ID";next;}{print \$0,tid,nid}' \\
+ >> ${outputPrefix}.combined.unsorted.bedpe
+
+ svtools bedpesort \\
+ ${outputPrefix}.combined.unsorted.bedpe \\
+ ${outputPrefix}.combined.bedpe
+ """
+
+}
diff --git a/modules/process/SVclone/SomaticRunSVclone.nf b/modules/process/SVclone/SomaticRunSVclone.nf
new file mode 100644
index 00000000..12571752
--- /dev/null
+++ b/modules/process/SVclone/SomaticRunSVclone.nf
@@ -0,0 +1,47 @@
+process SomaticRunSVclone {
+tag "${idTumor}__${idNormal}"
+
+publishDir "${params.outDir}/somatic/${outputPrefix}/", mode: params.publishDirMode, pattern: "svclone/*"
+
+input:
+ tuple val(idTumor), val(idNormal), val(target),
+ path(bamTumor), path(baiTumor),
+ path(bamNormal), path(baiNormal),
+ path(inBedpe),
+ path(mafFiltered),
+ path(cnv),
+ path(ploidyIn)
+ path(svclone_wrapper)
+output:
+ tuple val(idTumor), val(idNormal), val(target),
+ path("${outputPrefix}"), emit: SVcloneOutput
+ tuple val(idTumor), val(idNormal), val(target),
+ path("svclone/*"), emit: SVclonePublish
+ tuple val(idTumor), val(idNormal),
+ path("svclone/svs/*cluster_certainty.txt"),
+ path("svclone/snvs/*cluster_certainty.txt"), emit: SVclone4Aggregate
+
+script:
+outputPrefix = "${idTumor}__${idNormal}"
+"""
+python ${svclone_wrapper} \\
+ --cfg_template /config/svclone_config.ini \\
+ --bedpe ${inBedpe} \\
+ --maf ${mafFiltered} \\
+ --purity_ploidy ${ploidyIn} \\
+ --out_dir svclone_in \\
+ --sampleid ${outputPrefix} \\
+ --bam ${bamTumor} \\
+ --cnv ${cnv}
+
+mkdir -p svclone/svs svclone/snvs
+cp ${outputPrefix}/ccube_out/post_assign/*.RData ${outputPrefix}/ccube_out/post_assign/*.pdf svclone/svs
+cp ${outputPrefix}/ccube_out/post_assign/snvs/*.RData ${outputPrefix}/ccube_out/post_assign/snvs/*.pdf svclone/snvs
+for i in ${outputPrefix}/ccube_out/post_assign/*.txt ; do
+ sed "s/^/${outputPrefix}\\t/g" \$i | sed "0,/^${outputPrefix}\\t/s//sampleid\\t/" > svclone/svs/\$(basename \$i)
+done
+for i in ${outputPrefix}/ccube_out/post_assign/snvs/*.txt ; do
+ sed "s/^/${outputPrefix}\\t/g" \$i | sed "0,/^${outputPrefix}\\t/s//sampleid\\t/" > svclone/snvs/\$(basename \$i)
+done
+"""
+}
diff --git a/modules/process/SampleValidation/CrossValidateSamples.nf b/modules/process/SampleValidation/CrossValidateSamples.nf
new file mode 100644
index 00000000..6f7e1606
--- /dev/null
+++ b/modules/process/SampleValidation/CrossValidateSamples.nf
@@ -0,0 +1,83 @@
+process CrossValidateSamples {
+ input:
+ val(inputMapping)
+ val(inputPairing)
+
+ output:
+ val(validSamples), emit: validSamples
+ val(invalidSamples), emit: invalidSamples
+ val(validPairings), emit: validPairings
+
+ exec:
+ //Restructure pairs.
+ pairsList = []
+ curPair = []
+ mod = 0
+ for(String pairItem : inputPairing)
+ {
+ curPair.add(pairItem)
+ if(mod == 1)
+ {
+ pairsList.add(curPair)
+ mod = 0
+ curPair = []
+ continue
+ }
+ mod = mod + 1
+ }
+
+ //Restructure mappings.
+ mappingsList = []
+ curMapping = []
+ mod = 0
+ for(String mapItem : inputMapping)
+ {
+ curMapping.add(mapItem)
+ if(mod == 3)
+ {
+ mappingsList.add(curMapping)
+ mod = 0
+ curMapping = []
+ continue
+ }
+ mod = mod + 1
+ }
+
+ //Identify samples where both pairing ids have valid mappings.
+ validSamplesList = []
+ validPairings = []
+ for(i in 0..pairsList.size()-1)
+ {
+ if (inputMapping.toString().contains(pairsList[i][0]) && inputMapping.toString().contains(pairsList[i][1]))
+ {
+ validSamplesList.add(pairsList[i][0])
+ validSamplesList.add(pairsList[i][1])
+ validPairings.add([pairsList[i][0],pairsList[i][1]])
+ }
+ }
+
+ //If nothing is valid throw and error and stop execution.
+ if(validSamplesList.size() == 0)
+ {
+ println "Error: CrossValidateSamples - No valid samples identified between pairing and mapping files."
+ sleep(500)
+ exit 1
+ }
+
+ //Create valid inputMappings.
+ validInputMappings = []
+ invalidInputMappings = []
+ for(i in 0..mappingsList.size()-1)
+ {
+ if(validSamplesList.toString().contains(mappingsList[i][0]))
+ {
+ validInputMappings.add(mappingsList[i])
+ }
+ else{
+ invalidInputMappings.add(mappingsList[i])
+ }
+ }
+ validSamples = validInputMappings
+ invalidSamples = invalidInputMappings
+
+}
diff --git a/modules/process/Scatter/CreateScatteredIntervals.nf b/modules/process/Scatter/CreateScatteredIntervals.nf
new file mode 100644
index 00000000..cbbbc23c
--- /dev/null
+++ b/modules/process/Scatter/CreateScatteredIntervals.nf
@@ -0,0 +1,28 @@
+process CreateScatteredIntervals {
+ tag "${targetId}"
+
+ input:
+ tuple path(genomeFile), path(genomeIndex), path(genomeDict)
+ tuple val(targetId), path(targets), path(targetsIndex)
+
+ output:
+ tuple path("*.interval_list"), val(targetId), val(targetId), emit: mergedIList
+
+ script:
+ scatterCount = params.scatterCount
+ subdivision_mode = targetId == "wgs" ? "INTERVAL_SUBDIVISION" : "BALANCING_WITHOUT_INTERVAL_SUBDIVISION_WITH_OVERFLOW"
+ """
+ gatk SplitIntervals \
+ --reference ${genomeFile} \
+ --intervals ${targets} \
+ --scatter-count ${scatterCount} \
+ --subdivision-mode ${subdivision_mode} \
+ --output $targetId
+
+ for i in $targetId/*.interval_list;
+ do
+ BASENAME=`basename \$i`
+ mv \$i ${targetId}-\$BASENAME
+ done
+ """
+}
diff --git a/modules/subworkflow/AggregateFromProcess.nf b/modules/subworkflow/AggregateFromProcess.nf
new file mode 100644
index 00000000..d4d9326a
--- /dev/null
+++ b/modules/subworkflow/AggregateFromProcess.nf
@@ -0,0 +1,300 @@
+include { GermlineAggregateMaf } from '../process/Aggregate/GermlineAggregateMaf'
+include { GermlineAggregateSv } from '../process/Aggregate/GermlineAggregateSv'
+include { QcBamAggregate } from '../process/Aggregate/QcBamAggregate'
+include { QcConpairAggregate } from '../process/Aggregate/QcConpairAggregate'
+include { SomaticAggregateFacets } from '../process/Aggregate/SomaticAggregateFacets'
+include { SomaticAggregateLOHHLA } from '../process/Aggregate/SomaticAggregateLOHHLA'
+include { SomaticAggregateMaf } from '../process/Aggregate/SomaticAggregateMaf'
+include { SomaticAggregateMetadata } from '../process/Aggregate/SomaticAggregateMetadata'
+include { SomaticAggregateNetMHC } from '../process/Aggregate/SomaticAggregateNetMHC'
+include { SomaticAggregateSv } from '../process/Aggregate/SomaticAggregateSv'
+include { SomaticAggregateSvSignatures } from '../process/Aggregate/SomaticAggregateSvSignatures'
+include { SomaticAggregateHRDetect } from '../process/Aggregate/SomaticAggregateHRDetect'
+include { SomaticAggregateSVclone } from '../process/Aggregate/SomaticAggregateSVclone'
+include { CohortRunMultiQC } from '../process/Aggregate/CohortRunMultiQC'
+include { watchMapping; watchBamMapping; watchPairing; watchAggregateWithResult; watchAggregate } from '../function/watch_inputs.nf'
+
+workflow aggregateFromProcess
+{
+ take:
+ inputPairing
+ runAggregate
+ facets4Aggregate
+ sv4Aggregate
+ snv4Aggregate
+ hrd4Aggregate
+ svclone4Aggregate
+ lohhla4Aggregate
+ MetaData4Aggregate
+ snv4AggregateGermline
+ sv4AggregateGermline
+ sampleQC4Aggregate
+ conpair4Aggregate
+ fastPJson
+ multiqcWesConfig
+ multiqcWgsConfig
+ multiqcTempoLogo
+
+ main:
+ if (runAggregate != true){
+ if (!params.watch){
+ TempoUtils.extractCohort(file(runAggregate, checkIfExists: true))
+ .groupTuple()
+ .map{ cohort, idTumor, idNormal, pathNoUse
+ -> tuple( groupKey(cohort, idTumor instanceof Collection ? idTumor.size() : 1), idTumor, idNormal)
+ }
+ .transpose()
+ .set{inputAggregate}
+ }
+ else{
+ watchAggregate(file(runAggregate, checkIfExists: false))
+ .set{inputAggregate}
+ }
+ }
+ else {
+ inputPairing.set{ cohortTable }
+ cohortTable.map{ idTumor, idNormal -> ["default_cohort", idTumor, idNormal]}
+ .set{ inputAggregate }
+ }
+
+ if (facets4Aggregate){
+ input4AggregateFacets = inputAggregate.combine(facets4Aggregate.out.facets4Aggregate, by:[1,2]).groupTuple(by:[2]).map{[it[2],it[4],it[5],it[6],it[7],it[8]]}
+ }
+ if (sv4Aggregate){
+ inputSomaticAggregateSv =
+ inputAggregate.combine(sv4Aggregate.out.sv4Aggregate, by:[1,2])
+ .groupTuple(by:[2])
+ .map{[it[2], it[4]]}
+
+ inputSomaticAggregateSvSignatures =
+ inputAggregate.combine(sv4Aggregate.out.SVSignatures, by:[1,2])
+ .groupTuple(by:[2])
+ .map{[it[2], it[4], it[5]]}
+ }
+ if (snv4Aggregate){
+ inputSomaticAggregateNetMHC = inputAggregate.combine(snv4Aggregate.out.NetMhcStats4Aggregate, by:[1,2]).groupTuple(by:[2])
+ inputSomaticAggregateMaf = inputAggregate.combine(snv4Aggregate.out.finalMaf4Aggregate, by:[1,2]).groupTuple(by:[2])
+ }
+ if (hrd4Aggregate){
+ inputSomaticAggregateHrd = inputAggregate.combine(
+ hrd4Aggregate.out.HRDetect, by:[1,2]).groupTuple(by:[2]).map{[it[2], it[4]]}
+ }
+ if (svclone4Aggregate){
+ inputSomaticAggregateSVclone = inputAggregate
+ .combine(svclone4Aggregate.out.svclone4Aggregate, by:[1,2])
+ .groupTuple(by:[2])
+ .map{[it[2], it[4], it[5]]}
+ }
+ if (lohhla4Aggregate){
+ inputSomaticAggregateLOHHLA = inputAggregate.combine(lohhla4Aggregate.out.lohhla4Aggregate, by:[1,2]).groupTuple(by:[2]).map{[it[2], it[4], it[5]]}
+ }
+ if (MetaData4Aggregate){
+ inputSomaticAggregateMetadata = inputAggregate.combine(MetaData4Aggregate.out.MetaData4Aggregate, by:[1,2]).groupTuple(by:[2])
+ }
+ if (snv4AggregateGermline){
+ inputGermlineAggregateMaf = inputAggregate.combine(snv4AggregateGermline.out.snv4AggregateGermline, by:[1,2]).groupTuple(by:[2])
+ }
+ if (sv4AggregateGermline)
+ {
+ inputGermlineAggregateSv = inputAggregate.combine(sv4AggregateGermline.out.sv4AggregateGermline, by:[2]).groupTuple(by:[1]).map{[it[1], it[5].unique()]}
+ }
+
+ if (sampleQC4Aggregate){
+ sampleQC4Aggregate.out.bamsQcStats4Aggregate.branch{ item ->
+ def idSample = item[0]
+ def alfred = item[1]
+ ignoreY: alfred =~ /.+\.alfred\.tsv\.gz/
+ ignoreN: alfred =~ /.+\.alfred\.per_readgroup\.tsv\.gz/
+ }
+ .set{ bamsQcStats4Aggregate }
+
+ inputPairing.combine(bamsQcStats4Aggregate.ignoreY)
+ .branch { item ->
+ def idTumor = item[0]
+ def idNormal = item[1]
+ def idSample = item[2]
+ def alfred = item[3]
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+ .set{ alfredIgnoreY }
+
+ alfredIgnoreY.tumor.combine(alfredIgnoreY.normal, by:[0,1])
+ .combine(inputAggregate.map{ item -> [item[1], item[2], item[0]]}, by:[0,1])
+ .map{ item -> [item[6], item[0], item[1], item[3], item[5]]}
+ .groupTuple(by:[0])
+ .map{ item ->
+ def cohort = item[0]
+ def idTumors = item[1].unique()
+ def idNormals = item[2].unique()
+ def fileTumor = item[3].unique()
+ def fileNormal = item[4].unique()
+ [cohort, fileTumor, fileNormal]
+ }
+ .unique()
+ .set{ alfredIgnoreY }
+
+ inputPairing.combine(bamsQcStats4Aggregate.ignoreN)
+ .branch { item ->
+ def idTumor = item[0]
+ def idNormal = item[1]
+ def idSample = item[2]
+ def alfred = item[3]
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+ .set{ alfredIgnoreN }
+
+ alfredIgnoreN.tumor.combine(alfredIgnoreN.normal, by:[0,1])
+ .combine(inputAggregate.map{ item -> [item[1], item[2], item[0]]}, by:[0,1])
+ .map{ item -> [item[6], item[0], item[1], item[3], item[5]]}
+ .groupTuple(by:[0])
+ .map{ item ->
+ def cohort = item[0]
+ def idTumors = item[1].unique()
+ def idNormals = item[2].unique()
+ def fileTumor = item[3].unique()
+ def fileNormal = item[4].unique()
+ [cohort, fileTumor, fileNormal]
+ }
+ .unique()
+ .set{ alfredIgnoreN }
+
+ inputPairing.combine(sampleQC4Aggregate.out.collectHsMetricsOutput)
+ .branch { item ->
+ def idTumor = item[0]
+ def idNormal = item[1]
+ def idSample = item[2]
+ def hsMetrics = item[3]
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+ .set{ hsMetrics }
+
+ hsMetrics.tumor.combine(hsMetrics.normal, by:[0,1])
+ .combine(inputAggregate.map{ item -> [item[1], item[2], item[0]]}, by:[0,1])
+ .map{ item -> [item[6], item[0], item[1], item[3], item[5]]}
+ .groupTuple(by:[0])
+ .map{ item ->
+ def cohort = item[0]
+ def idTumors = item[1].unique()
+ def idNormals = item[2].unique()
+ def fileTumor = item[3].unique()
+ def fileNormal = item[4].unique()
+ [cohort, fileTumor, fileNormal]
+ }
+ .unique()
+ .set{ hsMetrics }
+
+ inputHsMetrics = hsMetrics
+ inputPairing.combine(fastPJson)
+ .branch { item ->
+ def idTumor = item[0]
+ def idNormal = item[1]
+ def idSample = item[2]
+ def jsonFiles = item[3]
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+ .set{ fastPMetrics }
+
+ fastPMetrics.tumor.combine(fastPMetrics.normal, by:[0,1])
+ .combine(inputAggregate.map{ item -> [item[1], item[2], item[0]]}, by:[0,1])
+ .map{ item -> [item[6], item[0], item[1], item[3], item[5]]}
+ .groupTuple(by:[0])
+ .map{ item ->
+ def cohort = item[0]
+ def idTumors = item[1].unique()
+ def idNormals = item[2].unique()
+ def fileTumor = item[3].flatten().unique()
+ def fileNormal = item[4].flatten().unique()
+ [cohort, fileTumor, fileNormal]
+ }
+ .unique()
+ .set{ fastPMetrics }
+
+ inputPairing.combine(sampleQC4Aggregate.out.qualimap4Process)
+ .branch { idTumor, idNormal, idSample, qualimapDir ->
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+ .set{ qualimap4AggregateTN }
+
+ qualimap4AggregateTN.tumor.combine(qualimap4AggregateTN.normal, by:[0,1])
+ .combine(inputAggregate.map{ item -> [item[1], item[2], item[0]]}, by:[0,1])
+ .map{ item -> [item[6], item[3], item[5]]}
+ .groupTuple(by:[0])
+ .map{ cohort, fileTumor, fileNormal ->
+ [cohort, fileTumor.unique(), fileNormal.unique()]
+ }
+ .unique()
+ .set{ inputQualimap4CohortMultiQC }
+
+ inputAlfredIgnoreY = alfredIgnoreY
+ inputAlfredIgnoreN = alfredIgnoreN
+ inputFastP4MultiQC = fastPMetrics
+ }
+
+ if (conpair4Aggregate){
+ inputQcConpairAggregate = inputAggregate.combine(conpair4Aggregate.out.conpair4Aggregate, by:[1,2]).groupTuple(by:[2]).map{[it[2], it[4], it[5]]}
+ FacetsQC4Aggregate = facets4Aggregate ? facets4Aggregate.out.FacetsQC4Aggregate : inputPairing.map{ idTumor, idNormal -> ["placeHolder",idTumor, idNormal,"",""]}
+ inputFacetsQC4CohortMultiQC = inputAggregate.combine(FacetsQC4Aggregate,by:[1,2]).groupTuple(by:[2]).map{[it[2], it[4], it[5]]}
+ }
+
+ if (facets4Aggregate){
+ SomaticAggregateFacets(input4AggregateFacets)
+ }
+ if (snv4Aggregate){
+ SomaticAggregateMaf(inputSomaticAggregateMaf)
+ SomaticAggregateNetMHC(inputSomaticAggregateNetMHC)
+ }
+ if (sv4Aggregate){
+ SomaticAggregateSv(inputSomaticAggregateSv)
+ if (params.assayType == "genome"){
+ SomaticAggregateSvSignatures(inputSomaticAggregateSvSignatures)
+ }
+ }
+ if (hrd4Aggregate){
+ SomaticAggregateHRDetect(inputSomaticAggregateHrd)
+ }
+ if (svclone4Aggregate){
+ SomaticAggregateSVclone(inputSomaticAggregateSVclone)
+ }
+ if (lohhla4Aggregate){
+ SomaticAggregateLOHHLA(inputSomaticAggregateLOHHLA)
+ }
+ if(MetaData4Aggregate)
+ {
+ SomaticAggregateMetadata(inputSomaticAggregateMetadata)
+ }
+ if (snv4AggregateGermline){
+ GermlineAggregateMaf(inputGermlineAggregateMaf)
+ }
+ if (sv4AggregateGermline){
+ GermlineAggregateSv(inputGermlineAggregateSv)
+ }
+
+ if (sampleQC4Aggregate){
+ inputAlfredIgnoreY.join(inputAlfredIgnoreN)
+ .join(inputHsMetrics)
+ .set{ inputQcBamAggregate }
+
+ QcBamAggregate(inputQcBamAggregate)
+ }
+
+ if (conpair4Aggregate) {
+ QcConpairAggregate(inputQcConpairAggregate)
+
+ inputFastP4MultiQC
+ .join(inputAlfredIgnoreY,by:0)
+ .join(inputAlfredIgnoreN,by:0)
+ .join(inputQcConpairAggregate,by:0)
+ .join(inputFacetsQC4CohortMultiQC,by:0)
+ .join(inputQualimap4CohortMultiQC,by:0)
+ .join(inputHsMetrics, by:0)
+ .set{ inputCohortRunMultiQC }
+
+ CohortRunMultiQC(inputCohortRunMultiQC,
+ Channel.value([multiqcWesConfig, multiqcWgsConfig, multiqcTempoLogo]))
+ }
+}
diff --git a/modules/subworkflow/AggregateFromResult.nf b/modules/subworkflow/AggregateFromResult.nf
new file mode 100644
index 00000000..8a5a679b
--- /dev/null
+++ b/modules/subworkflow/AggregateFromResult.nf
@@ -0,0 +1,156 @@
+include { GermlineAggregateMaf } from '../process/Aggregate/GermlineAggregateMaf'
+include { GermlineAggregateSv } from '../process/Aggregate/GermlineAggregateSv'
+include { QcBamAggregate } from '../process/Aggregate/QcBamAggregate'
+include { QcConpairAggregate } from '../process/Aggregate/QcConpairAggregate'
+include { SomaticAggregateFacets } from '../process/Aggregate/SomaticAggregateFacets'
+include { SomaticAggregateLOHHLA } from '../process/Aggregate/SomaticAggregateLOHHLA'
+include { SomaticAggregateMaf } from '../process/Aggregate/SomaticAggregateMaf'
+include { SomaticAggregateMetadata } from '../process/Aggregate/SomaticAggregateMetadata'
+include { SomaticAggregateNetMHC } from '../process/Aggregate/SomaticAggregateNetMHC'
+include { SomaticAggregateSv } from '../process/Aggregate/SomaticAggregateSv'
+include { SomaticAggregateSvSignatures } from '../process/Aggregate/SomaticAggregateSvSignatures'
+include { SomaticAggregateHRDetect } from '../process/Aggregate/SomaticAggregateHRDetect'
+include { SomaticAggregateSVclone } from '../process/Aggregate/SomaticAggregateSVclone'
+include { CohortRunMultiQC } from '../process/Aggregate/CohortRunMultiQC'
+include { watchMapping; watchBamMapping; watchPairing; watchAggregateWithResult; watchAggregate } from '../function/watch_inputs.nf'
+
+workflow aggregateFromResult
+{
+ take:
+ aggregateFile
+ multiqcWesConfig
+ multiqcWgsConfig
+ multiqcTempoLogo
+
+ main:
+ doWF_validate = true
+ doWF_align = true
+ doWF_manta = true
+ doWF_scatter = true
+ doWF_germSNV = true
+ doWF_germSV = true
+ doWF_facets = true
+ doWF_SV = true
+ doWF_loh = true
+ doWF_SNV = true
+ doWF_sampleQC = true
+ doWF_msiSensor = true
+ doWF_mutSig = true
+ doWF_mdParse = true
+ doWF_samplePairingQC = true
+
+ if(!params.watch){
+ TempoUtils.extractCohort(file(aggregateFile, checkIfExists: true))
+ .set{ inputAggregate }
+ }
+ else{
+ watchAggregateWithResult(file(aggregateFile, checkIfExists: true))
+ .set{ inputAggregate }
+ }
+
+ inputAggregate.multiMap{ cohort, idTumor, idNormal, path ->
+ finalMaf4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.final.maf" )]
+ NetMhcStats4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.all_neoantigen_predictions.txt")]
+ FacetsPurity4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*/*_purity.seg")]
+ FacetsHisens4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*/*_hisens.seg")]
+ FacetsOutLog4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*_OUT.txt")]
+ FacetsArmLev4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*/*.arm_level.txt")]
+ FacetsGeneLev4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*/*.gene_level.txt")]
+ FacetsQC4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*_OUT.txt"), file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*/*.facets_qc.txt")]
+ sv4Aggregate: params.assayType == "genome" ?
+ [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.final.clustered.bedpe")] :
+ [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.final.bedpe")]
+ svSignatures4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*_exposures.tsv"), file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*_catalogues.pdf")]
+ hrd4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.hrdetect.tsv")]
+ svclone4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/svs/*_cluster_certainty.txt"), file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/snvs/*_cluster_certainty.txt")]
+ predictHLA4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.DNA.HLAlossPrediction_CI.txt")]
+ intCPN4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*DNA.IntegerCPN_CI.txt")]
+ MetaData4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/*.sample_data.txt")]
+ mafFile4AggregateGermline: [idTumor, idNormal, cohort, "placeHolder", file(path + "/germline/" + idNormal + "/*/" + idTumor + "__" + idNormal + ".germline.final.maf")]
+ sv4AggregateGermline: [idNormal, cohort, idTumor, "placeHolder", "noTumor", file(path + "/germline/" + idNormal + "/*/*.final.bedpe")]
+ conpairConcord4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/" + idTumor + "__" + idNormal + ".concordance.txt")]
+ conpairContami4Aggregate: [idTumor, idNormal, cohort, "placeHolder", file(path + "/somatic/" + idTumor + "__" + idNormal + "/*/" + idTumor + "__" + idNormal + ".contamination.txt")]
+ alfredIgnoreYTumor: [cohort, idTumor, idNormal, file(path + "/bams/" + idTumor + "/*/*.alfred.tsv.gz/")]
+ alfredIgnoreYNormal: [cohort, idTumor, idNormal, file(path + "/bams/" + idNormal + "/*/*.alfred.tsv.gz/")]
+ alfredIgnoreNTumor: [cohort, idTumor, idNormal, file(path + "/bams/" + idTumor + "/*/*.alfred.per_readgroup.tsv.gz/")]
+ alfredIgnoreNNormal: [cohort, idTumor, idNormal, file(path + "/bams/" + idNormal + "/*/*.alfred.per_readgroup.tsv.gz/")]
+ qualimapTumor: [cohort, idTumor, idNormal, file(path + "/bams/" + idTumor + "/qualimap/${idTumor}_qualimap_rawdata.tar.gz")]
+ qualimapNormal: [cohort, idTumor, idNormal, file(path + "/bams/" + idNormal + "/qualimap/${idNormal}_qualimap_rawdata.tar.gz")]
+ hsMetricsTumor: params.assayType == "exome" ? [cohort, idTumor, idNormal, file(path + "/bams/" + idTumor + "/*/*.hs_metrics.txt")] : [cohort, idTumor, idNormal, ""]
+ hsMetricsNormal: params.assayType == "exome" ? [cohort, idTumor, idNormal, file(path + "/bams/" + idNormal + "/*/*.hs_metrics.txt")] : [cohort, idTumor, idNormal, ""]
+ fastpTumor: [cohort, idTumor, idNormal, file(path + "/bams/" + idTumor + "/*/*.fastp.json")]
+ fastpNormal: [cohort, idTumor, idNormal, file(path + "/bams/" + idNormal + "/*/*.fastp.json")]
+ }
+ .set { aggregateList }
+
+ inputSomaticAggregateMaf = aggregateList.finalMaf4Aggregate.transpose().groupTuple(by:[2])
+ inputSomaticAggregateNetMHC = aggregateList.NetMhcStats4Aggregate.transpose().groupTuple(by:[2])
+ inputPurity4Aggregate = aggregateList.FacetsPurity4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputHisens4Aggregate = aggregateList.FacetsHisens4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputOutLog4Aggregate = aggregateList.FacetsOutLog4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputArmLev4Aggregate = aggregateList.FacetsArmLev4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputGeneLev4Aggregate = aggregateList.FacetsGeneLev4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputFacetsQC4CohortMultiQC = aggregateList.FacetsQC4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4], it[5]]}
+ inputSomaticAggregateSv = aggregateList.sv4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputSomaticAggregateSvSignatures = aggregateList.svSignatures4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4], it[5]]}
+ inputSomaticAggregateHrd = aggregateList.hrd4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputSomaticAggregateSVclone = aggregateList.svclone4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4], it[5]]}
+ inputPredictHLA4Aggregate = aggregateList.predictHLA4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputIntCPN4Aggregate = aggregateList.intCPN4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}
+ inputSomaticAggregateMetadata = aggregateList.MetaData4Aggregate.transpose().groupTuple(by:[2])
+ inputGermlineAggregateMaf = aggregateList.mafFile4AggregateGermline.transpose().groupTuple(by:[2])
+ inputGermlineAggregateSv = aggregateList.sv4AggregateGermline.transpose().groupTuple(by:[1]).map{[it[1], it[5].unique()]}
+ inputAlfredIgnoreY = aggregateList.alfredIgnoreYTumor.unique().combine(aggregateList.alfredIgnoreYNormal.unique(), by:[0,1,2]).transpose().groupTuple(by:[0]).map{ [it[0], it[3].unique(), it[4].unique()]}
+ inputAlfredIgnoreN = aggregateList.alfredIgnoreNTumor.unique().combine(aggregateList.alfredIgnoreNNormal.unique(), by:[0,1,2]).transpose().groupTuple(by:[0]).map{ [it[0], it[3].unique(), it[4].unique()]}
+ inputQualimap4CohortMultiQC = aggregateList.qualimapTumor.unique().combine(aggregateList.qualimapNormal.unique(), by:[0,1,2]).transpose().groupTuple(by:[0]).map{ [it[0], it[3].unique(), it[4].unique()]}
+ inputHsMetrics = aggregateList.hsMetricsTumor.unique().combine(aggregateList.hsMetricsNormal.unique(), by:[0,1,2]).transpose().groupTuple(by:[0]).map{ [it[0], it[3].unique(), it[4].unique()]}
+ aggregateList.conpairConcord4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}.set{inputConpairConcord4Aggregate}
+ aggregateList.conpairContami4Aggregate.transpose().groupTuple(by:[2]).map{[it[2], it[4]]}.set{inputConpairContami4Aggregate}
+ aggregateList.fastpTumor.unique().combine(aggregateList.fastpNormal.unique(), by:[0,1,2]).transpose().groupTuple(by:[0]).map{ [it[0], it[3].unique(), it[4].unique()]}.set{inputFastP4MultiQC}
+
+ SomaticAggregateMaf(inputSomaticAggregateMaf)
+
+ inputPurity4Aggregate.join(inputHisens4Aggregate, by:[0])
+ .join(inputOutLog4Aggregate, by:[0])
+ .join(inputArmLev4Aggregate, by:[0])
+ .join(inputGeneLev4Aggregate, by:[0])
+ .set{ inputSomaticAggregateFacets }
+ SomaticAggregateFacets(inputSomaticAggregateFacets)
+ SomaticAggregateNetMHC(inputSomaticAggregateNetMHC)
+
+ SomaticAggregateSv(inputSomaticAggregateSv)
+ SomaticAggregateSvSignatures(inputSomaticAggregateSvSignatures)
+ SomaticAggregateHRDetect(inputSomaticAggregateHrd)
+ SomaticAggregateSVclone(inputSomaticAggregateSVclone)
+
+ inputPredictHLA4Aggregate.join(inputIntCPN4Aggregate)
+ .set{ inputSomaticAggregateLOHHLA }
+ SomaticAggregateLOHHLA(inputSomaticAggregateLOHHLA)
+
+ SomaticAggregateMetadata(inputSomaticAggregateMetadata)
+
+ GermlineAggregateMaf(inputGermlineAggregateMaf)
+
+ GermlineAggregateSv(inputGermlineAggregateSv)
+
+ inputAlfredIgnoreY.join(inputAlfredIgnoreN)
+ .join(inputHsMetrics)
+ .set{ inputQcBamAggregate }
+ QcBamAggregate(inputQcBamAggregate)
+
+ inputConpairConcord4Aggregate.join(inputConpairContami4Aggregate)
+ .set{ inputQcConpairAggregate }
+ QcConpairAggregate(inputQcConpairAggregate)
+
+ inputFastP4MultiQC
+ .join(inputAlfredIgnoreY,by:0)
+ .join(inputAlfredIgnoreN,by:0)
+ .join(inputConpairConcord4Aggregate,by:0)
+ .join(inputConpairContami4Aggregate,by:0)
+ .join(inputFacetsQC4CohortMultiQC,by:0)
+ .join(inputQualimap4CohortMultiQC,by:0)
+ .join(inputHsMetrics, by:0)
+ .set{ inputCohortRunMultiQC }
+ CohortRunMultiQC(inputCohortRunMultiQC,
+ Channel.value([multiqcWesConfig, multiqcWgsConfig, multiqcTempoLogo]))
+}
diff --git a/modules/subworkflow/PairTumorNormal.nf b/modules/subworkflow/PairTumorNormal.nf
new file mode 100644
index 00000000..7d6809e4
--- /dev/null
+++ b/modules/subworkflow/PairTumorNormal.nf
@@ -0,0 +1,86 @@
+
+workflow PairTumorNormal
+{
+ take:
+ inputBam
+ inputPairing
+
+ main:
+ if (params.pairing) {
+ // Parse input FASTQ mapping
+ inputBam.combine(inputPairing)
+ .filter { item ->
+ def idSample = item[0]
+ def target = item[1]
+ def sampleBam = item[2]
+ def sampleBai = item[3]
+ def idTumor = item[4]
+ def idNormal = item[5]
+ idSample == idTumor
+ }.map { item ->
+ def idTumor = item[4]
+ def idNormal = item[5]
+ def tumorBam = item[2]
+ def tumorBai = item[3]
+ def target = item[1]
+ return [ idTumor, idNormal, target, tumorBam, tumorBai ]
+ }
+ .unique()
+ .set{ bamsTumor }
+
+ inputBam.combine(inputPairing)
+ .filter { item ->
+ def idSample = item[0]
+ def target = item[1]
+ def sampleBam = item[2]
+ def sampleBai = item[3]
+ def idTumor = item[4]
+ def idNormal = item[5]
+ idSample == idNormal
+ }.map { item ->
+ def idTumor = item[4]
+ def idNormal = item[5]
+ def normalBam = item[2]
+ def normalBai = item[3]
+ def target = item[1]
+ return [ idTumor, idNormal, target, normalBam, normalBai ]
+ }
+ .unique()
+ .set{ bamsNormal }
+
+ bamsNormal.map { item ->
+ def idNormal = item[1]
+ def target = item[2]
+ def normalBam = item[3]
+ def normalBai = item[4]
+ return [ idNormal, target, normalBam, normalBai ] }
+ .unique()
+ .set{ bams }
+
+ bamsTumor.combine(bamsNormal, by: [0,1,2])
+ .map { item -> // re-order the elements
+ def idTumor = item[0]
+ def idNormal = item[1]
+ def target = item[2]
+ def bamTumor = item[3]
+ def baiTumor = item[4]
+ def bamNormal = item[5]
+ def baiNormal = item[6]
+
+ return [ idTumor, idNormal, target, bamTumor, baiTumor, bamNormal, baiNormal ]
+ }
+ .set{ bamFiles }
+ }
+ else
+ {
+ println "Error: PairTumorNormal - pairing file not provided."
+ exit 1
+ }
+
+
+ emit:
+ bamFiles = bamFiles
+ bams = bams
+ bamsNormal = bamsNormal
+ bamsTumor = bamsTumor
+}
diff --git a/modules/subworkflow/alignment_wf.nf b/modules/subworkflow/alignment_wf.nf
new file mode 100644
index 00000000..5813d994
--- /dev/null
+++ b/modules/subworkflow/alignment_wf.nf
@@ -0,0 +1,198 @@
+include { SplitLanesR1; SplitLanesR2 } from '../process/Alignment/SplitLanes'
+include { AlignReads } from '../process/Alignment/AlignReads'
+include { MergeBamsAndMarkDuplicates } from '../process/Alignment/MergeBamsAndMarkDuplicates'
+include { RunBQSR } from '../process/Alignment/RunBQSR'
+
+workflow alignment_wf
+{
+ take:
+ inputMapping
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ if (params.bamMapping)
+ {
+ println "Alignment workflow cannot accept bam files for input."
+ exit 1
+ }
+ if(params.mapping)
+ {
+ // Parse input FASTQ mapping
+ if (params.watch != true) {
+ inputMapping.groupTuple(by: [0])
+ .map { idSample, targets, files_pe1, files_pe2
+ -> tuple(groupKey(idSample, targets.size()), targets, files_pe1, files_pe2)
+ }
+ .transpose()
+ .set { inputMapping }
+ }
+
+ inputMapping.map { idSample, target, file_pe1, file_pe2 ->
+ [idSample, target, file_pe1, file_pe2, idSample + '@' + file_pe1.getSimpleName(), file_pe2.getSimpleName()]
+ }
+ .set { inputFastqs }
+
+ if (params.splitLanes) {
+ inputFastqs.set { fastqsNeedSplit }
+ inputFastqs.set { fastqsNoNeedSplit }
+
+ fastqsNeedSplit
+ .filter { item -> !(item[2].getName() =~ /_L(\d){3}_/) }
+ .multiMap { idSample, target, file_pe1, file_pe2, fileID, lane ->
+ inputFastqR1: [idSample, target, file_pe1, file_pe1.toString()]
+ inputFastqR2: [idSample, target, file_pe2, file_pe2.toString()]
+ }
+ .set { fastqsNeedSplit }
+
+ fastqsNoNeedSplit
+ .filter { item -> item[2].getName() =~ /_L(\d){3}_/ }
+ .map { idSample, target, file_pe1, file_pe2, fileID, lane
+ -> tuple(idSample, target, file_pe1, file_pe1.size(), file_pe2, file_pe2.size(), groupKey(fileID, 1), lane)
+ }
+ .set { fastqsNoNeedSplit }
+
+ perLaneFastqsR1 = SplitLanesR1(fastqsNeedSplit.inputFastqR1).R1SplitData
+ perLaneFastqsR2 = SplitLanesR2(fastqsNeedSplit.inputFastqR2).R2SplitData
+
+ def fastqR1fileIDs = [:]
+ perLaneFastqsR1 = perLaneFastqsR1.transpose()
+ .map { item ->
+ def idSample = item[0]
+ def target = item[1]
+ def fastq = item[2]
+ def fileID = idSample + '@' + item[3].getSimpleName()
+ def lane = fastq.getSimpleName().split('_L00')[1].split('_')[0]
+ def laneCount = item[4].getSimpleName().toInteger()
+
+ // This only checks if same read groups appears in two or more fastq files which belongs to the same sample. Cross sample check will be performed after AlignReads since the read group info is not available for fastqs which does not need to be split.
+ if ( !params.watch ) {
+ if (!TempoUtils.checkDuplicates(fastqR1fileIDs, fileID + '@' + lane, fileID + "\t" + fastq, 'the following fastq files since they contain the same RGID')) { exit 1 }
+ }
+ [idSample, target, fastq, fileID, lane, laneCount]
+ }
+
+ def fastqR2fileIDs = [:]
+ perLaneFastqsR2 = perLaneFastqsR2.transpose()
+ .map { item ->
+ def idSample = item[0]
+ def target = item[1]
+ def fastq = item[2]
+ def fileID = idSample + '@' + item[3].getSimpleName()
+ def lane = fastq.getSimpleName().split('_L00')[1].split('_')[0]
+ def laneCount = item[4].getSimpleName().toInteger()
+ if ( !params.watch ) {
+ if (!TempoUtils.checkDuplicates(fastqR2fileIDs, fileID + '@' + lane, fileID + "\t" + fastq, 'the follwoing fastq files since they contain the same RGID')) { exit 1 }
+ }
+ [idSample, target, fastq, fileID, lane, laneCount]
+ }
+
+ fastqFiles = perLaneFastqsR1
+ .mix(perLaneFastqsR2)
+ .groupTuple(by: [0, 1, 3, 4, 5], size: 2, sort: true)
+ .map { idSample, target, fastqPairs, fileID, lanes, laneCount ->
+ tuple(idSample, target, fastqPairs, groupKey(fileID, laneCount), lanes)
+ }
+ .map { idSample, target, fastqPairs, fileID, lane ->
+ [idSample, target, fastqPairs[0], fastqPairs[1], fileID, lane]
+ }
+ .map { item ->
+ def idSample = item[0]
+ def target = item[1]
+ def fastqPair1 = item[2]
+ def fastqPair2 = item[3]
+ if (item[2].toString().split('_R1').size() < item[3].toString().split('_R1').size()) {
+ fastqPair1 = item[3]
+ fastqPair2 = item[2]
+ }
+ def fileID = item[4]
+ def lane = item[5]
+ [idSample, target, fastqPair1, fastqPair1.size(), fastqPair2, fastqPair2.size(), fileID, lane]
+ }
+ .mix(fastqsNoNeedSplit)
+ }
+ else {
+ fastqFiles = inputFastqs.map { idSample, target, file_pe1, file_pe2, fileID, lane
+ -> tuple(idSample, target, file_pe1, file_pe1.size(),
+ file_pe2, file_pe2.size(), groupKey(fileID, 1), lane)
+ }
+ }
+
+ //Align reads to reference.
+ AlignReads(fastqFiles, Channel.value([referenceMap.genomeFile, referenceMap.bwaIndex]))
+
+ AlignReads.out.fastPJson4MultiQC
+ .groupTuple(by:[2])
+ .map{idSample, jsonFile, fileID ->
+ def idSampleout = idSample[0] instanceof Collection ? idSample[0].first() : idSample[0]
+ [idSampleout, jsonFile]
+ }.groupTuple(by: [0])
+ .map{ idSample, jsonFile ->
+ [idSample, jsonFile.flatten()]
+ }.set{ fastPJson }
+
+ // Check for FASTQ files which might have different path but contains the same reads, based only on the name of the first read.
+ def allReadIds = [:]
+ AlignReads.out.sortedBam
+ .groupTuple(by:[3])
+ .map { idSample, target, bam, fileID, lane, readIdFile ->
+ def idSample_first = idSample instanceof Collection ? idSample.first() : idSample
+ def target_first = target instanceof Collection ? target.first() : target
+ if ( !params.watch ){
+ for (i in readIdFile.flatten().unique()){
+ def readId = "@" + i.getSimpleName().replaceAll("@", ":")
+ if(!TempoUtils.checkDuplicates(allReadIds, readId, idSample_first + "\t" + fileID, "the following samples, since they contain the same read: \n${readId}")){exit 1}
+ }
+ }
+ [idSample_first, target_first, bam.flatten().unique()]
+ }
+ .groupTuple(by: [0])
+ .map{ item ->
+ def idSample = item[0]
+ def target = item[1] instanceof Collection ? item[1].first() : item[1]
+ def bams = item[2].flatten().unique()
+ [idSample, bams, target]
+ }
+ .set { groupedBam }
+
+ MergeBamsAndMarkDuplicates(groupedBam)
+ RunBQSR(MergeBamsAndMarkDuplicates.out.mdBams,
+ Channel.value([
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.genomeDict,
+ referenceMap.dbsnp,
+ referenceMap.dbsnpIndex,
+ referenceMap.knownIndels,
+ referenceMap.knownIndelsIndex
+ ]))
+
+
+ File file_bammapping = new File(params.outname)
+ file_bammapping.newWriter().withWriter { w ->
+ w << "SAMPLE\tTARGET\tBAM\tBAI\n"
+ }
+
+ RunBQSR.out.bamsBQSR
+ .map{ idSample, target, bam, bai ->
+ [ idSample, target, "${file(params.outDir).toString()}/bams/${idSample}/${idSample}.bam", "${file(params.outDir).toString()}/bams/${idSample}/${idSample}.bam.bai" ]
+ }.subscribe { Object obj ->
+ file_bammapping.withWriterAppend { out ->
+ out.println "${obj[0]}\t${obj[1]}\t${obj[2]}\t${obj[3]}"
+ }
+ }
+ }
+ else{
+ if(params.pairing){
+ println "ERROR: When --pairing [tsv], --mapping [tsv] must be provided."
+ exit 1
+ }
+ }
+
+
+ emit:
+ RunBQSR_bamsBQSR = RunBQSR.out.bamsBQSR
+ RunBQSR_bamSize = RunBQSR.out.bamSize
+ fastPJson = fastPJson
+}
diff --git a/modules/subworkflow/ascat_wf.nf b/modules/subworkflow/ascat_wf.nf
new file mode 100644
index 00000000..f1da5afa
--- /dev/null
+++ b/modules/subworkflow/ascat_wf.nf
@@ -0,0 +1,39 @@
+include { runAscatAlleleCount } from '../process/Ascat/runAscatAlleleCount'
+include { runAscat } from '../process/Ascat/runAscat'
+
+workflow ascat_wf {
+ take:
+ paired_bams
+
+ main:
+ referenceMap = params.referenceMap
+
+ ascatAlleleCountLimit = ["GRCh37","smallGRCh37","GRCh38"].contains(params.genome) ? 48 : 1
+ ascatAlleleCountSegments = Channel.from(1..ascatAlleleCountLimit)
+ runAscatAlleleCount(
+ ascatAlleleCountSegments,
+ ascatAlleleCountLimit,
+ paired_bams,
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.snpGcCorrections
+ )
+
+ runAscat(
+ runAscatAlleleCount.out
+ .groupTuple(by:[0,1,2], size: ascatAlleleCountLimit)
+ .map{idTumor, idNormal, target, ascatTar ->
+ [idTumor, idNormal, target, ascatTar.flatten() ]
+ }.combine(paired_bams, by:[0,1,2]),
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.snpGcCorrections
+ )
+
+ emit:
+ ascatCNV = runAscat.out.caveman
+ ascatSS = runAscat.out.samplestatistics
+
+}
+
+
diff --git a/modules/subworkflow/brass_wf.nf b/modules/subworkflow/brass_wf.nf
new file mode 100644
index 00000000..5264e44a
--- /dev/null
+++ b/modules/subworkflow/brass_wf.nf
@@ -0,0 +1,98 @@
+include { generateBasFile } from '../process/SV/BRASS/generateBasFile'
+include { runBRASSInput } from '../process/SV/BRASS/SomaticRunBRASSInput'
+include { runBRASSCover } from '../process/SV/BRASS/SomaticRunBRASSCover'
+include { runBRASS } from '../process/SV/BRASS/SomaticRunBRASS'
+
+workflow brass_wf
+{
+ take:
+ bamFiles
+ sampleStatistics // from ascat
+
+ main:
+ referenceMap = params.referenceMap
+ inputPairing = bamFiles
+ .map{ row ->
+ [row[0],row[1]]
+ }
+ bamFilesUnpaired = bamFiles
+ .map{ row -> [row[0], row[2], row[3],row[4]]}
+ .mix(
+ bamFiles
+ .map{ row -> [row[1], row[2], row[5],row[6]]}
+ .unique()
+ )
+
+ generateBasFile(
+ bamFilesUnpaired,
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex
+ )
+
+ basPairing = inputPairing
+ .combine(generateBasFile.out)
+ .branch{ idTumor, idNormal, idSample, target, basFile ->
+ tumor: idSample == idTumor
+ normal: idSample == idNormal
+ }
+
+ brassInfiles = bamFiles
+ .combine(
+ basPairing.tumor
+ .combine(basPairing.normal, by:[0,1])
+ .map{ idTumor,idNormal, idSample1, target1, basFile1, idSample2, target2, basFile2 ->
+ [idTumor,idNormal,target1,basFile1,basFile2]
+ }, by:[0,1,2]
+ ).map{ idTumor, idNormal, target, bamTumor, baiTumor, bamNormal, baiNormal, basTumor, basNormal ->
+ [idTumor, idNormal, target, bamTumor, baiTumor, basTumor, bamNormal, baiNormal, basNormal]
+ }
+
+ BRASSInputSegments = Channel.from(1..2)
+ runBRASSInput(
+ BRASSInputSegments,
+ brassInfiles,
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.brassRefDir,
+ referenceMap.vagrentRefDir
+ )
+
+ brassCoverLimit = params.genome in ["GRCh37","smallGRCh37","GRCh37"] ? 24 : 1
+ brassCoverSegments = Channel.from(1..brassCoverLimit)
+ runBRASSCover(
+ brassCoverSegments,
+ brassCoverLimit,
+ brassInfiles,
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.brassRefDir,
+ referenceMap.vagrentRefDir
+ )
+
+ runBRASSInput_flat = runBRASSInput.out.groupTuple(by:[0,1,2],size:2)
+ .map{ idTumor, idNormal, target, tmp, progress ->
+ [ idTumor, idNormal, target, tmp.flatten(), progress.flatten() ]
+ }
+ runBRASSCover_flat = runBRASSCover.out.groupTuple(by:[0,1,2],size:brassCoverLimit)
+ .map{ idTumor, idNormal, target, tmp, progress ->
+ [ idTumor, idNormal, target, tmp.flatten(), progress.flatten() ]
+ }
+
+ brassInfilesWithPreprocess = brassInfiles
+ .combine(runBRASSInput_flat, by:[0,1,2])
+ .combine(runBRASSCover_flat, by:[0,1,2])
+ .combine(sampleStatistics, by:[0,1,2])
+
+ runBRASS(
+ brassInfilesWithPreprocess,
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.brassRefDir,
+ referenceMap.vagrentRefDir
+ )
+
+ emit:
+ brassOutput = runBRASS.out.BRASSOutput
+ BRASS4Combine = runBRASS.out.BRASS4Combine
+
+}
diff --git a/modules/subworkflow/clonality_wf.nf b/modules/subworkflow/clonality_wf.nf
new file mode 100644
index 00000000..6a92e63c
--- /dev/null
+++ b/modules/subworkflow/clonality_wf.nf
@@ -0,0 +1,26 @@
+include { SomaticRunSVclone } from '../process/SVclone/SomaticRunSVclone'
+
+workflow clonality_wf {
+ take:
+ bams
+ final_bedpe
+ final_maf
+ final_cnv
+ final_purity_ploidy
+
+ main:
+ SVcloneInput = bams
+ .combine(final_bedpe,by:[0,1,2])
+ .combine(final_maf, by:[0,1,2])
+ .combine(final_cnv, by:[0,1,2])
+ .combine(final_purity_ploidy, by:[0,1,2])
+
+ SomaticRunSVclone(
+ SVcloneInput,
+ workflow.projectDir + "/containers/svclone/svclone_wrapper.py"
+ )
+ svclone4Aggregate = SomaticRunSVclone.out.SVclone4Aggregate.map{ ["placeHolder"] + it }
+
+ emit:
+ svclone4Aggregate = svclone4Aggregate
+}
diff --git a/modules/subworkflow/facets_wf.nf b/modules/subworkflow/facets_wf.nf
new file mode 100644
index 00000000..c6909aa4
--- /dev/null
+++ b/modules/subworkflow/facets_wf.nf
@@ -0,0 +1,59 @@
+include { DoFacets } from '../process/Facets/DoFacets'
+include { DoFacetsPreviewQC } from '../process/Facets/DoFacetsPreviewQC'
+
+workflow facets_wf
+{
+ take:
+ bamFiles
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+ outputDir = "facets${params.facets.R_lib}c${params.facets.cval}pc${params.facets.purity_cval}"
+
+ DoFacets(
+ bamFiles,
+ referenceMap.facetsVcf,
+ workflow.projectDir + "/containers/facets-suite-preview-htstools",
+ outputDir
+ )
+
+ DoFacetsPreviewQC(DoFacets.out.Facets4FacetsPreview)
+
+ DoFacets.out.FacetsRunSummary.combine(DoFacetsPreviewQC.out.FacetsPreviewOut, by:[0,1]).set{ FacetsQC4Aggregate } // idTumor, idNormal, summaryFiles, qcFiles
+ DoFacets.out.FacetsRunSummary.combine(DoFacetsPreviewQC.out.FacetsPreviewOut, by:[0,1]).set{ FacetsQC4SomaticMultiQC } // idTumor, idNormal, summaryFiles, qcFiles
+ FacetsQC4Aggregate.map{ idTumor, idNormal, summaryFiles, qcFiles ->
+ ["placeholder",idTumor, idNormal, summaryFiles, qcFiles]
+ }.set{ FacetsQC4Aggregate }
+
+ DoFacets.out.facets4Aggregate
+ .map{
+ ["placeHolder"] + it
+ }.set{facets4Aggregate}
+
+ DoFacets.out.FacetsArmGeneOutput
+ .map{
+ ["placeHolder"] + it
+ }.set{FacetsArmGeneOutput}
+
+ emit:
+ snpPileupOutput = DoFacets.out.snpPileupOutput
+ FacetsOutput = DoFacets.out.FacetsOutput
+ facets4Aggregate = facets4Aggregate
+ facetsPurity = DoFacets.out.facetsPurity
+ facetsForMafAnno = DoFacets.out.facetsForMafAnno
+ Facets4FacetsPreview = DoFacets.out.Facets4FacetsPreview
+ FacetsArmGeneOutput = FacetsArmGeneOutput
+ FacetsQC4MetaDataParser = DoFacets.out.FacetsQC4MetaDataParser
+ FacetsRunSummary = DoFacets.out.FacetsRunSummary
+ FacetsPreviewOut = DoFacetsPreviewQC.out.FacetsPreviewOut
+ FacetsQC4Aggregate = FacetsQC4Aggregate
+ FacetsQC4SomaticMultiQC = FacetsQC4SomaticMultiQC
+ FacetsHisensCNV4HrDetect = DoFacets.out.FacetsHisensCNV4HrDetect
+ FacetsHisensCNV4HrDetectFiltered = DoFacets.out.FacetsHisensCNV4HrDetectFiltered
+ FacetsHisensSampleStatistics4BRASS = DoFacets.out.FacetsHisensSampleStatistics4BRASS
+ FacetsPurityCNV4HrDetect = DoFacets.out.FacetsPurityCNV4HrDetect
+ FacetsPurityCNV4HrDetectFiltered = DoFacets.out.FacetsPurityCNV4HrDetectFiltered
+ FacetsPuritySampleStatistics4BRASS = DoFacets.out.FacetsPuritySampleStatistics4BRASS
+
+}
diff --git a/modules/subworkflow/germlineSNV_wf.nf b/modules/subworkflow/germlineSNV_wf.nf
new file mode 100644
index 00000000..905df1df
--- /dev/null
+++ b/modules/subworkflow/germlineSNV_wf.nf
@@ -0,0 +1,83 @@
+include { GermlineRunHaplotypecaller } from '../process/GermSNV/GermlineRunHaplotypecaller'
+include { GermlineCombineHaplotypecallerVcf } from '../process/GermSNV/GermlineCombineHaplotypecallerVcf'
+include { GermlineRunStrelka2 } from '../process/GermSNV/GermlineRunStrelka2'
+include { GermlineCombineChannel } from '../process/GermSNV/GermlineCombineChannel'
+include { GermlineAnnotateMaf } from '../process/GermSNV/GermlineAnnotateMaf'
+include { GermlineFacetsAnnotation } from '../process/GermSNV/GermlineFacetsAnnotation'
+
+workflow germlineSNV_wf
+{
+ take:
+ bams
+ bamsTumor
+ mergedIList
+ facetsForMafAnno
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ bams.combine(mergedIList, by: 1)
+ .map{
+ item ->
+ def idNormal = item[1]
+ def target = item[0]
+ def normalBam = item[2]
+ def normalBai = item[3]
+ def intervalBed = item[4]
+ def key = idNormal+"@"+target // adding one unique key
+ return [ key, idNormal, target, normalBam, normalBai, intervalBed ]
+ }.map{
+ key, idNormal, target, normalBam, normalBai, intervalBed ->
+ tuple (
+ groupKey(key, intervalBed.size()), // adding numbers so that each sample only wait for it's own children processes
+ idNormal, target, normalBam, normalBai, intervalBed
+ )
+ }
+ .transpose()
+ .set{ mergedChannelGermline }
+
+
+ GermlineRunHaplotypecaller(mergedChannelGermline,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ GermlineRunHaplotypecaller.out.haplotypecaller4Combine.groupTuple().set{ haplotypecaller4Combine }
+
+ GermlineCombineHaplotypecallerVcf(haplotypecaller4Combine,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ bams.map{ idNormal, target, bamNormal, baiNormal ->
+ [idNormal, target, bamNormal, baiNormal, targetsMap."$target".targetsBedGz, targetsMap."$target".targetsBedGzTbi]
+ }.set{ bamsForStrelkaGermline }
+
+ GermlineRunStrelka2(bamsForStrelkaGermline,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ // Join HaploTypeCaller and Strelka outputs, bcftools.
+ GermlineCombineHaplotypecallerVcf.out.haplotypecallerCombinedVcfOutput
+ .map{ ["placeHolder"] + it }
+ .combine(
+ GermlineRunStrelka2.out.strelkaOutputGermline.map{ ["placeHolder"] + it },
+ by: [0,1,2]
+ )
+ .combine(bamsTumor, by: [1,2])
+ .set{ mergedChannelVcfCombine }
+
+ GermlineCombineChannel(mergedChannelVcfCombine,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex,]),
+ Channel.value([referenceMap.repeatMasker, referenceMap.repeatMaskerIndex, referenceMap.mapabilityBlacklist, referenceMap.mapabilityBlacklistIndex]),
+ Channel.value([referenceMap.gnomadWesVcf, referenceMap.gnomadWesVcfIndex, referenceMap.gnomadWgsVcf, referenceMap.gnomadWgsVcfIndex]))
+
+ GermlineAnnotateMaf(GermlineCombineChannel.out.mutationMergedGermline,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict,
+ referenceMap.vepCache, referenceMap.isoforms]))
+
+ facetsForMafAnno.combine(GermlineAnnotateMaf.out.mafFileGermline, by: [0,1,2])
+ .set{ facetsMafFileGermline }
+
+ GermlineFacetsAnnotation(facetsMafFileGermline)
+ snv4AggregateGermline = GermlineFacetsAnnotation.out.mafFile4AggregateGermline.map{ ["placeHolder"] + it }
+
+ emit:
+ snv4AggregateGermline = snv4AggregateGermline
+}
diff --git a/modules/subworkflow/germlineSV_wf.nf b/modules/subworkflow/germlineSV_wf.nf
new file mode 100644
index 00000000..3adebe28
--- /dev/null
+++ b/modules/subworkflow/germlineSV_wf.nf
@@ -0,0 +1,89 @@
+include { GermlineDellyCall } from '../process/GermSV/GermlineDellyCall'
+include { DellyCombine
+ as GermlineDellyCombine } from '../process/SV/DellyCombine'
+include { GermlineRunManta } from '../process/GermSV/GermlineRunManta'
+include { GermlineMergeSVs } from '../process/GermSV/GermlineMergeSVs'
+include { GermlineSVVcf2Bedpe } from '../process/GermSV/GermlineSVVcf2Bedpe'
+include { GermlineAnnotateSVBedpe } from '../process/GermSV/GermlineAnnotateSVBedpe'
+include { GermlineRunSvABA } from '../process/GermSV/GermlineRunSvABA'
+
+workflow germlineSV_wf
+{
+ take:
+ bams
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ Channel.from("DUP", "BND", "DEL", "INS", "INV").set{ svTypesGermline }
+
+ GermlineDellyCall(
+ svTypesGermline,
+ bams,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.svCallingExcludeRegions])
+ )
+ GermlineDellyCombine(
+ GermlineDellyCall.out.dellyFilter4CombineGermline
+ .groupTuple( by: [0,1], size: 5 )
+ .map{ normal_id, target, vcf, tbi ->
+ [ "", normal_id, target, vcf, tbi ]
+ }
+ , "germline"
+ )
+
+ GermlineRunManta(
+ bams,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex]),
+ Channel.value([referenceMap.svCallingIncludeRegions, referenceMap.svCallingIncludeRegionsIndex])
+ )
+
+ GermlineRunSvABA(
+ bams
+ .map{ idNormal, target, bamNormal, baiNormal ->
+ [ idNormal, target, bamNormal, baiNormal ] + [targetsMap."$target".targetsBed]
+ },
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.genomeDict,
+ referenceMap.bwaIndex
+ )
+
+ GermlineDellyCombine.out
+ .map{ tumor_id, normal_id, target, vcf, tbi -> [normal_id, target, vcf, tbi, "delly" ] }
+ .mix(GermlineRunManta.out.mantaOutputGermline.map{ it + ["manta"]})
+ .mix(GermlineRunSvABA.out.SvABA4Combine.map{ it + ["svaba"]})
+ .groupTuple( by:[0,1], size:3 )
+ .set{allSvCallsCombineChannel}
+
+ GermlineMergeSVs(
+ allSvCallsCombineChannel,
+ workflow.projectDir + "/containers/bcftools-vt-mergesvvcf"
+ )
+
+ GermlineSVVcf2Bedpe(
+ GermlineMergeSVs.out.SVsCombinedOutputGermline
+ )
+ GermlineAnnotateSVBedpe(
+ GermlineSVVcf2Bedpe.out.GermlineCombinedUnfilteredBedpe,
+ referenceMap.repeatMasker,
+ referenceMap.mapabilityBlacklist,
+ referenceMap.svBlacklistBed,
+ referenceMap.svBlacklistBedpe,
+ referenceMap.svBlacklistFoldbackBedpe,
+ referenceMap.svBlacklistTEBedpe,
+ referenceMap.spliceSites,
+ workflow.projectDir + "/containers/iannotatesv",
+ params.genome
+ )
+
+ GermlineMergeSVs.out.SVsCombinedOutputGermline
+ .map{ idNormal, target, vcfFile, tbiFile ->
+ ["placeHolder", "noTumor", idNormal, vcfFile, tbiFile]
+ }.set{sv4AggregateGermline}
+
+ emit:
+ SVAnnotBedpe = GermlineAnnotateSVBedpe.out.SVAnnotBedpe
+ SVAnnotBedpePass = GermlineAnnotateSVBedpe.out.SVAnnotBedpePass
+ sv4AggregateGermline = GermlineAnnotateSVBedpe.out.SVAnnotBedpe4Aggregate
+}
diff --git a/modules/subworkflow/hrdetect_wf.nf b/modules/subworkflow/hrdetect_wf.nf
new file mode 100644
index 00000000..f48aae30
--- /dev/null
+++ b/modules/subworkflow/hrdetect_wf.nf
@@ -0,0 +1,24 @@
+include { HRDetect } from '../process/HRDetect/HRDetect'
+
+workflow hrdetect_wf {
+ take:
+ CNVoutput
+ MAFoutput
+ SVoutput
+
+ main:
+ HRDetectVariantsIn = MAFoutput
+ .combine(CNVoutput,by:[0,1,2])
+ .combine(SVoutput, by:[0,1,2])
+
+ HRDetect(
+ HRDetectVariantsIn,
+ workflow.projectDir + "/containers/signaturetoolslib/HRDetect_wrapper.R"
+ )
+
+ HRDetectOut = HRDetect.out.map{ ["placeHolder"] + it }
+
+ emit:
+ HRDetect = HRDetectOut
+
+}
diff --git a/modules/subworkflow/loh_wf.nf b/modules/subworkflow/loh_wf.nf
new file mode 100644
index 00000000..fd7d88bf
--- /dev/null
+++ b/modules/subworkflow/loh_wf.nf
@@ -0,0 +1,28 @@
+include { RunPolysolver } from '../process/LoH/RunPolysolver'
+include { RunLOHHLA } from '../process/LoH/RunLOHHLA'
+
+workflow loh_wf
+{
+ take:
+ bams
+ bamFiles
+ facetsPurity
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ RunPolysolver(bams)
+ hlaOutput = RunPolysolver.out.hlaOutput.map{ ["placeHolder"] + it }
+
+ bamFiles.combine(facetsPurity, by: [0,1,2])
+ .combine(hlaOutput, by: [1,2])
+ .set{ mergedChannelLOHHLA }
+
+ RunLOHHLA(mergedChannelLOHHLA,
+ Channel.value([referenceMap.hlaFasta, referenceMap.hlaDat]))
+
+ emit:
+ hlaOutput = hlaOutput
+ lohhla4Aggregate = RunLOHHLA.out.lohhla4Aggregate
+}
diff --git a/modules/subworkflow/manta_wf.nf b/modules/subworkflow/manta_wf.nf
new file mode 100644
index 00000000..7195de85
--- /dev/null
+++ b/modules/subworkflow/manta_wf.nf
@@ -0,0 +1,18 @@
+include { SomaticRunManta } from '../process/SV/SomaticRunManta'
+
+workflow manta_wf
+{
+ take: bamFiles
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ SomaticRunManta(bamFiles,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex]),
+ Channel.value([referenceMap.svCallingIncludeRegions, referenceMap.svCallingIncludeRegionsIndex]))
+ emit:
+ manta4Combine = SomaticRunManta.out.manta4Combine
+ //mantaOutput = SomaticRunManta.out.mantaOutput
+ mantaToStrelka = SomaticRunManta.out.mantaToStrelka
+}
diff --git a/modules/subworkflow/mdParse_wf.nf b/modules/subworkflow/mdParse_wf.nf
new file mode 100644
index 00000000..6f29087b
--- /dev/null
+++ b/modules/subworkflow/mdParse_wf.nf
@@ -0,0 +1,13 @@
+include { MetaDataParser } from '../process/MetaParse/MetaDataParser'
+
+workflow mdParse_wf
+{
+ take:
+ mergedChannelMetaDataParser
+
+ main:
+ MetaDataParser(mergedChannelMetaDataParser)
+
+ emit:
+ MetaData4Aggregate = MetaDataParser.out.MetaData4Aggregate
+}
diff --git a/modules/subworkflow/msiSensor_wf.nf b/modules/subworkflow/msiSensor_wf.nf
new file mode 100644
index 00000000..a2f6c9c3
--- /dev/null
+++ b/modules/subworkflow/msiSensor_wf.nf
@@ -0,0 +1,16 @@
+include { RunMsiSensor } from '../process/MSI/RunMsiSensor'
+
+workflow msiSensor_wf
+{
+ take:
+ bamFiles
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+ RunMsiSensor(bamFiles,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict, referenceMap.msiSensorList]))
+
+ emit:
+ msi4MetaDataParser = RunMsiSensor.out.msi4MetaDataParser
+}
diff --git a/modules/subworkflow/mutSig_wf.nf b/modules/subworkflow/mutSig_wf.nf
new file mode 100644
index 00000000..93f9e786
--- /dev/null
+++ b/modules/subworkflow/mutSig_wf.nf
@@ -0,0 +1,17 @@
+include { RunMutationSignatures } from '../process/MutSig/RunMutationSignatures'
+
+workflow mutSig_wf
+{
+ take:
+ mafFile
+
+ main:
+ if (!(params.cosmic in ['v2', 'v3'])) {
+ println "ERROR: Possible values of mutational signature reference --cosmic is 'v2', 'v3'"
+ exit 1
+ }
+ RunMutationSignatures(mafFile)
+
+ emit:
+ mutSig4MetaDataParser = RunMutationSignatures.out.mutSig4MetaDataParser
+}
diff --git a/modules/subworkflow/samplePairingQC_wf.nf b/modules/subworkflow/samplePairingQC_wf.nf
new file mode 100644
index 00000000..bc7c3a9f
--- /dev/null
+++ b/modules/subworkflow/samplePairingQC_wf.nf
@@ -0,0 +1,67 @@
+include { QcPileup } from '../process/QC/QcPileup'
+include { QcConpair } from '../process/QC/QcConpair'
+include { QcConpairAll } from '../process/QC/QcConpairAll'
+
+workflow samplePairingQC_wf
+{
+ take:
+ inputChannel
+ inputPairing
+ runConpairAll
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ QcPileup(inputChannel, Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ QcPileup.out.pileupOutput.combine(inputPairing)
+ .filter { item ->
+ def idSample = item[0]
+ def samplePileup = item[1]
+ def idTumor = item[2]
+ def idNormal = item[3]
+ idSample == idTumor
+ }.map { item ->
+ def idTumor = item[2]
+ def idNormal = item[3]
+ def tumorPileup = item[1]
+ return [ idTumor, idNormal, tumorPileup ]
+ }
+ .unique()
+ .set{ pileupT }
+
+ QcPileup.out.pileupOutput.combine(inputPairing)
+ .filter { item ->
+ def idSample = item[0]
+ def samplePileup = item[1]
+ def idTumor = item[2]
+ def idNormal = item[3]
+ idSample == idNormal
+ }.map { item ->
+ def idTumor = item[2]
+ def idNormal = item[3]
+ def normalPileup = item[1]
+ return [ idTumor, idNormal, normalPileup ]
+ }
+ .unique()
+ .set{ pileupN }
+
+ pileupT.combine(pileupN, by: [0, 1]).unique().set{ pileupConpair }
+
+ QcConpair(pileupConpair, Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+ conpair4Aggregate = QcConpair.out.conpair4Aggregate.map{ ["placeHolder"] + it }
+
+ if(runConpairAll){
+ pileupT.combine(pileupN).unique().set{ pileupConpairAll }
+
+ QcConpairAll(pileupConpairAll,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+ conpairAll4Aggregate = QcConpairAll.out.conpairAll4Aggregate.map{ ["placeHolder"] + it }
+ }
+
+ emit:
+ // -- Run based on QcConpairAll channels or the single QcConpair channels
+ conpair4Aggregate = (!runConpairAll ? conpair4Aggregate : conpairAll4Aggregate)
+ conpairOutput = QcConpair.out.conpairOutput
+}
diff --git a/modules/subworkflow/sampleQC_wf.nf b/modules/subworkflow/sampleQC_wf.nf
new file mode 100644
index 00000000..822bee4e
--- /dev/null
+++ b/modules/subworkflow/sampleQC_wf.nf
@@ -0,0 +1,61 @@
+include { QcCollectHsMetrics } from '../process/QC/QcCollectHsMetrics'
+include { QcQualimap } from '../process/QC/QcQualimap'
+include { QcAlfred } from '../process/QC/QcAlfred'
+include { SampleRunMultiQC } from '../process/QC/SampleRunMultiQC'
+
+workflow sampleQC_wf
+{
+ take:
+ inputChannel
+ fastPJson
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ if (params.assayType != "genome"){
+ inputChannel.map{ idSample, target, bam, bai ->
+ [idSample, target, bam, bai, targetsMap."$target".targetsInterval, targetsMap."$target".baitsInterval]
+ }.set{ bamsBQSR4HsMetrics }
+
+ QcCollectHsMetrics(bamsBQSR4HsMetrics,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict])
+ )
+ collectHsMetricsOutput = QcCollectHsMetrics.out.collectHsMetricsOutput
+ } else {
+ inputChannel
+ .map{ idSample, target, bam, bai -> [idSample, ""]}
+ .set{ collectHsMetricsOutput }
+ }
+
+ inputChannel
+ .map{ idSample, target, bam, bai -> [ idSample, target, bam, bai, file(targetsMap."$target".targetsBed) ]}
+ .set{ bamsBQSR4Qualimap }
+
+ QcQualimap(bamsBQSR4Qualimap)
+
+ Channel.from(true, false).set{ ignore_read_groups }
+ inputChannel
+ .map{ idSample, target, bam, bai ->
+ [ idSample, target, bam, bai, targetsMap."$target".targetsBedGz, targetsMap."$target".targetsBedGzTbi ]
+ }.set{ bamsBQSR4Alfred }
+
+ QcAlfred(ignore_read_groups,
+ bamsBQSR4Alfred,
+ Channel.value([referenceMap.genomeFile]))
+
+ QcAlfred.out.alfredOutput
+ .groupTuple(size:2, by:0)
+ .join(fastPJson, by:0)
+ .join(QcQualimap.out.qualimap4Process, by:0)
+ .join(collectHsMetricsOutput, by:0)
+ .set{ sampleMetrics4MultiQC }
+
+ SampleRunMultiQC(sampleMetrics4MultiQC,
+ Channel.value([params.multiqcWesConfig, params.multiqcWgsConfig, params.multiqcTempoLogo]))
+
+ emit:
+ bamsQcStats4Aggregate = QcAlfred.out.bamsQcStats4Aggregate
+ collectHsMetricsOutput = collectHsMetricsOutput
+ qualimap4Process = QcQualimap.out.qualimap4Process
+}
diff --git a/modules/subworkflow/scatter_wf.nf b/modules/subworkflow/scatter_wf.nf
new file mode 100644
index 00000000..990f0be0
--- /dev/null
+++ b/modules/subworkflow/scatter_wf.nf
@@ -0,0 +1,20 @@
+include { CreateScatteredIntervals } from '../process/Scatter/CreateScatteredIntervals'
+
+workflow scatter_wf
+{
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ targets4Intervals = Channel.from(targetsMap.keySet())
+ .map{ targetId ->
+ [ targetId, targetsMap."${targetId}".targetsBedGz, targetsMap."${targetId}".targetsBedGzTbi ]
+ }
+
+ CreateScatteredIntervals(Channel.value([referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.genomeDict]),
+ targets4Intervals)
+ emit:
+ mergedIList = CreateScatteredIntervals.out.mergedIList
+}
diff --git a/modules/subworkflow/snv_wf.nf b/modules/subworkflow/snv_wf.nf
new file mode 100644
index 00000000..a22d574b
--- /dev/null
+++ b/modules/subworkflow/snv_wf.nf
@@ -0,0 +1,89 @@
+include { SomaticRunStrelka2 } from '../process/SNV/SomaticRunStrelka2'
+include { RunMutect2 } from '../process/SNV/RunMutect2'
+include { SomaticCombineMutect2Vcf } from '../process/SNV/SomaticCombineMutect2Vcf'
+include { SomaticCombineChannel } from '../process/SNV/SomaticCombineChannel'
+include { SomaticAnnotateMaf } from '../process/SNV/SomaticAnnotateMaf'
+include { RunNeoantigen } from '../process/SNV/RunNeoantigen'
+include { SomaticFacetsAnnotation } from '../process/SNV/SomaticFacetsAnnotation'
+
+workflow snv_wf
+{
+ take:
+ bamFiles
+ mergedIList
+ mantaToStrelka
+ hlaOutput
+ facetsForMafAnno
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ bamFiles.combine(mergedIList, by: 2)
+ .map{
+ item ->
+ def idTumor = item[1]
+ def idNormal = item[2]
+ def target = item[0]
+ def tumorBam = item[3]
+ def normalBam = item[4]
+ def tumorBai = item[5]
+ def normalBai = item[6]
+ def intervalBed = item[7]
+ def key = idTumor+"__"+idNormal+"@"+target // adding one unique key
+
+ return [ key, idTumor, idNormal, target, tumorBam, normalBam, tumorBai, normalBai, intervalBed ]
+ }.map{
+ key, idTumor, idNormal, target, tumorBam, normalBam, tumorBai, normalBai, intervalBed ->
+ tuple (
+ groupKey(key, intervalBed.size()), // adding numbers so that each sample only wait for it's own children processes
+ idTumor, idNormal, target, tumorBam, normalBam, tumorBai, normalBai, intervalBed
+ )
+ }
+ .transpose()
+ .set{ mergedChannelSomatic }
+
+ RunMutect2(mergedChannelSomatic,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ RunMutect2.out.forMutect2Combine.groupTuple().set{ forMutect2Combine }
+
+ SomaticCombineMutect2Vcf(forMutect2Combine,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ bamFiles.combine(mantaToStrelka, by: [0, 1, 2])
+ .map{ idTumor, idNormal, target, bamTumor, baiTumor, bamNormal, baiNormal, mantaCSI, mantaCSIi ->
+ [idTumor, idNormal, target, bamTumor, baiTumor, bamNormal, baiNormal, mantaCSI, mantaCSIi, targetsMap."$target".targetsBedGz, targetsMap."$target".targetsBedGzTbi]
+ }.set{ input4Strelka }
+
+ SomaticRunStrelka2(input4Strelka,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict]))
+
+ SomaticCombineMutect2Vcf.out.mutect2CombinedVcfOutput.combine(bamFiles, by: [0,1,2]).combine(SomaticRunStrelka2.out.strelka4Combine, by: [0,1,2]).set{ mutectStrelkaChannel }
+
+ SomaticCombineChannel(mutectStrelkaChannel,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex]),
+ Channel.value([referenceMap.repeatMasker, referenceMap.repeatMaskerIndex, referenceMap.mapabilityBlacklist, referenceMap.mapabilityBlacklistIndex]),
+ Channel.value([referenceMap.exomePoN, referenceMap.wgsPoN,referenceMap.exomePoNIndex, referenceMap.wgsPoNIndex,]),
+ Channel.value([referenceMap.gnomadWesVcf, referenceMap.gnomadWesVcfIndex,referenceMap.gnomadWgsVcf, referenceMap.gnomadWgsVcfIndex]))
+
+ SomaticAnnotateMaf(SomaticCombineChannel.out.mutationMergedVcf,
+ Channel.value([referenceMap.genomeFile, referenceMap.genomeIndex, referenceMap.genomeDict,
+ referenceMap.vepCache, referenceMap.isoforms]))
+
+
+ hlaOutput.combine(SomaticAnnotateMaf.out.mafFile, by: [1,2]).set{ input4Neoantigen }
+
+ RunNeoantigen(input4Neoantigen, Channel.value([referenceMap.neoantigenCDNA, referenceMap.neoantigenCDS]))
+
+ facetsForMafAnno.combine(RunNeoantigen.out.mafFileForMafAnno, by: [0,1,2]).set{ facetsMafFileSomatic }
+
+ SomaticFacetsAnnotation(facetsMafFileSomatic)
+ finalMaf4Aggregate = SomaticFacetsAnnotation.out.finalMaf4Aggregate.map { ["placeHolder"] + it }
+
+ emit:
+ mafFile = SomaticAnnotateMaf.out.mafFile
+ maf4MetaDataParser = SomaticFacetsAnnotation.out.maf4MetaDataParser
+ NetMhcStats4Aggregate = RunNeoantigen.out.NetMhcStats4Aggregate
+ finalMaf4Aggregate = finalMaf4Aggregate
+}
diff --git a/modules/subworkflow/somaticMultiQC_wf.nf b/modules/subworkflow/somaticMultiQC_wf.nf
new file mode 100644
index 00000000..08448c06
--- /dev/null
+++ b/modules/subworkflow/somaticMultiQC_wf.nf
@@ -0,0 +1,10 @@
+include { SomaticRunMultiQC } from '../process/QC/SomaticRunMultiQC'
+
+workflow somaticMultiQC_wf
+{
+ take:
+ somaticMultiQCinput
+
+ main:
+ SomaticRunMultiQC(somaticMultiQCinput, Channel.value([params.multiqcWesConfig, params.multiqcWgsConfig, params.multiqcTempoLogo]))
+}
diff --git a/modules/subworkflow/sv_wf.nf b/modules/subworkflow/sv_wf.nf
new file mode 100644
index 00000000..9f9f8441
--- /dev/null
+++ b/modules/subworkflow/sv_wf.nf
@@ -0,0 +1,128 @@
+include { SomaticDellyCall } from '../process/SV/SomaticDellyCall'
+include { DellyCombine
+ as SomaticDellyCombine } from '../process/SV/DellyCombine'
+include { SomaticRunSvABA } from '../process/SV/SomaticRunSvABA'
+include { brass_wf } from './brass_wf' addParams(referenceMap: params.referenceMap)
+include { SomaticMergeSVs } from '../process/SV/SomaticMergeSVs'
+include { SomaticSVVcf2Bedpe } from '../process/SV/SomaticSVVcf2Bedpe'
+include { SomaticAnnotateSVBedpe } from '../process/SV/SomaticAnnotateSVBedpe'
+include { SomaticRunClusterSV } from '../process/SV/SomaticRunClusterSV'
+include { RunSVSignatures } from '../process/HRDetect/RunSVSignatures'
+include { SomaticRunSVCircos } from '../process/SV/SomaticRunSVCircos'
+
+workflow sv_wf
+{
+ take:
+ bamFiles
+ manta4Combine
+ sampleStatistics
+ cnvCalls
+
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ Channel.from("DUP", "BND", "DEL", "INS", "INV").set{ svTypes }
+ SomaticDellyCall(
+ svTypes,
+ bamFiles,
+ Channel.value([
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.svCallingExcludeRegions
+ ])
+ )
+
+ // Put manta output and delly output into the same channel so they can be processed together in the group key
+ // that they came in with i.e. (`idTumor`, `idNormal`, and `target`)
+ SomaticDellyCombine(
+ SomaticDellyCall.out.dellyFilter4Combine
+ .groupTuple( by: [0,1,2], size: 5 )
+ , "somatic"
+ )
+
+ SomaticRunSvABA(
+ bamFiles
+ .map{ idTumor, idNormal, target, bamTumor, baiTumor, bamNormal, baiNormal ->
+ [ idTumor, idNormal, target, bamTumor, baiTumor, bamNormal, baiNormal ] + [targetsMap."$target".targetsBed]
+ },
+ referenceMap.genomeFile,
+ referenceMap.genomeIndex,
+ referenceMap.genomeDict,
+ referenceMap.bwaIndex
+ )
+
+ if (params.assayType == "genome" && workflow.profile != "test") {
+ brass_wf(
+ bamFiles,
+ sampleStatistics // from facets (default) or ascat
+ )
+
+ SomaticDellyCombine.out.map{ it + ["delly"]}
+ .mix(manta4Combine.map{ it + ["manta"]})
+ .mix(SomaticRunSvABA.out.SvABA4Combine.map{ it + ["svaba"]})
+ .mix(brass_wf.out.BRASS4Combine.map{ it + ["brass"]})
+ .groupTuple( by:[0,1,2], size:4 )
+ .set{allSvCallsCombineChannel}
+ } else {
+ SomaticDellyCombine.out.map{ it + ["delly"]}
+ .mix(manta4Combine.map{ it + ["manta"]})
+ .mix(SomaticRunSvABA.out.SvABA4Combine.map{ it + ["svaba"]})
+ .groupTuple( by:[0,1,2], size:3 )
+ .set{allSvCallsCombineChannel}
+ }
+
+ // --- Process SV VCFs
+ // Merge VCFs
+ SomaticMergeSVs(
+ allSvCallsCombineChannel,
+ workflow.projectDir + "/containers/bcftools-vt-mergesvvcf"
+ )
+
+ // Convert VCF to Bedpe
+ SomaticSVVcf2Bedpe(
+ SomaticMergeSVs.out.SVCallsCombinedVcf
+ )
+
+ // Annotate Bedpe
+ SomaticAnnotateSVBedpe(
+ SomaticSVVcf2Bedpe.out.SomaticCombinedUnfilteredBedpe,
+ referenceMap.repeatMasker,
+ referenceMap.mapabilityBlacklist,
+ referenceMap.svBlacklistBed,
+ referenceMap.svBlacklistBedpe,
+ referenceMap.svBlacklistFoldbackBedpe,
+ referenceMap.svBlacklistTEBedpe,
+ referenceMap.spliceSites,
+ workflow.projectDir + "/containers/iannotatesv",
+ params.genome
+ )
+
+ if (params.assayType == "genome") {
+ RunSVSignatures(
+ SomaticAnnotateSVBedpe.out.SVAnnotBedpePass,
+ workflow.projectDir + "/containers/signaturetoolslib/sv_signatures_wrapper.R"
+ )
+ SVSignatures = RunSVSignatures.out
+ SomaticRunClusterSV( SomaticAnnotateSVBedpe.out.SVAnnotBedpePass )
+ sv4Aggregate = SomaticRunClusterSV.out.Bedpe4Aggregate.map{ ["placeHolder"] + it }
+ } else {
+ SVSignatures = Channel.empty()
+ sv4Aggregate = SomaticAnnotateSVBedpe.out.SVAnnotBedpe4Aggregate.map{ ["placeHolder"] + it }
+ }
+
+ if (cnvCalls){
+ SomaticRunSVCircos(
+ SomaticAnnotateSVBedpe.out.SVAnnotBedpePass
+ .combine(cnvCalls, by: [0,1,2]),
+ workflow.projectDir + "/containers/biocircos/generate_biocircos.R",
+ workflow.projectDir + "/containers/biocircos/biocircos.Rmd"
+ )
+ }
+
+ emit:
+ SVSignatures = SVSignatures
+ SVAnnotBedpe = SomaticAnnotateSVBedpe.out.SVAnnotBedpe
+ SVAnnotBedpePass = SomaticAnnotateSVBedpe.out.SVAnnotBedpePass
+ sv4Aggregate = sv4Aggregate
+}
diff --git a/modules/subworkflow/validate_wf.nf b/modules/subworkflow/validate_wf.nf
new file mode 100644
index 00000000..110e5310
--- /dev/null
+++ b/modules/subworkflow/validate_wf.nf
@@ -0,0 +1,50 @@
+include { CrossValidateSamples } from '../process/SampleValidation/CrossValidateSamples'
+include { watchMapping; watchBamMapping; watchPairing; watchAggregateWithResult; watchAggregate } from '../function/watch_inputs.nf'
+
+workflow validate_wf
+{
+ main:
+ referenceMap = params.referenceMap
+ targetsMap = params.targetsMap
+
+ TempoUtils.checkAssayType(params.assayType)
+ target_id_list = targetsMap.keySet()
+ if (params.watch == false) {
+ mappingFile = params.mapping ? file(params.mapping, checkIfExists: true) : file(params.bamMapping, checkIfExists: true)
+ inputMapping = params.mapping ? TempoUtils.extractFastq(mappingFile, params.assayType, target_id_list) : TempoUtils.extractBAM(mappingFile, params.assayType, target_id_list)
+ }
+ else if (params.watch == true) {
+ mappingFile = params.mapping ? file(params.mapping, checkIfExists: false) : file(params.bamMapping, checkIfExists: false)
+ inputMapping = params.mapping ? watchMapping(mappingFile, params.assayType, target_id_list) : watchBamMapping(mappingFile, params.assayType, target_id_list)
+ }
+ else{}
+ if(params.pairing){
+ if (params.watch == false) {
+ pairingFile = file(params.pairing, checkIfExists: true)
+ inputPairing = TempoUtils.extractPairing(pairingFile)
+ TempoUtils.crossValidateTargets(inputMapping, inputPairing)
+
+ CrossValidateSamples(inputMapping.collect(), inputPairing.collect())
+ CrossValidateSamples.out.validSamples
+ .flatten()
+ .collate(4)
+ .map { idSample, target, file_pe1, file_pe2 ->
+ [idSample, target, file(file_pe1), file(file_pe2)]
+ }
+ .set { inputMapping }
+ CrossValidateSamples.out.validPairings.flatten().collate(2).set{ inputPairing }
+ }
+ else if (params.watch == true) {
+ pairingFile = file(params.pairing, checkIfExists: false)
+ inputPairing = watchPairing(pairingFile)
+ }
+ else{}
+ }
+ else {
+ inputPairing = Channel.empty()
+ }
+
+ emit:
+ inputMapping = inputMapping
+ inputPairing = inputPairing
+}
diff --git a/nextflow.config b/nextflow.config
index d76310cd..8856eda0 100755
--- a/nextflow.config
+++ b/nextflow.config
@@ -13,10 +13,10 @@
manifest {
description = 'WES & WGS pipeline'
homePage = 'https://github.com/mskcc/tempo'
- mainScript = 'pipeline.nf'
+ mainScript = 'dsl2.nf'
name = 'tempo'
- nextflowVersion = '>=20.01.0'
- version = '1.3.0'
+ nextflowVersion = '>=25.04.7'
+ version = '2.0'
}
params {
@@ -28,6 +28,7 @@ params {
// see https://github.com/SciLifeLab/Sarek/blob/master/conf/base.config
publishDirMode = 'link' // publishDir mode is hard 'link' by default
tools = 'lohhla,delly,facets,mutect2,manta,strelka2,msisensor,haplotypecaller,polysolver,mutsig,neoantigen,pileup,conpair'
+ workflows = 'snv,sv,mutsig,lohhla,facets,qc,msisensor'
assayType = "exome" // either 'exome' or 'genome'; default exome
somatic = false
germline = false
@@ -45,6 +46,7 @@ params {
chunkSizeLimit = 0 // set > 0 to tune number of lines read from mapping, bamMapping or aggregate
anonymizeFQ = false
cosmic = 'v3'
+ svcnv = 'hisens'
}
// Run profiles are specified with "-profile" at the command line
@@ -98,18 +100,19 @@ profiles {
}
test_singularity {
+ includeConfig "conf/test.config"
includeConfig "conf/singularity.config"
- includeConfig "conf/juno.config"
includeConfig "conf/containers.config"
includeConfig "conf/resources.config"
includeConfig "conf/references.config"
-
if(params.assayType == "exome") {
includeConfig "conf/exome.config"
}
if(params.assayType == "genome") {
includeConfig "conf/genome.config"
}
+ executor.name = "lsf"
+ params.scatterCount = 3
}
test {
@@ -124,6 +127,7 @@ profiles {
if(params.assayType == "genome") {
includeConfig "conf/genome.config"
}
+ params.scatterCount = 3
}
}
diff --git a/pipeline.nf b/pipeline.nf
old mode 100755
new mode 100644
index 8db68bb6..bcfe0ea1
--- a/pipeline.nf
+++ b/pipeline.nf
@@ -526,11 +526,15 @@ if (params.mapping) {
file("size.txt") into sizeOutput
script:
+
+ bamSize = 0
+ bam.each{ bamSize = bamSize + it.size()}
+
if (workflow.profile == "juno") {
- if(bam.size() > 100.GB) {
+ if(bamSize() > 100.GB) {
task.time = { params.maxWallTime }
}
- else if (bam.size() < 80.GB) {
+ else if (bamSize() < 80.GB) {
task.time = task.exitStatus.toString() in wallTimeExitCode ? { params.medWallTime } : { params.minWallTime }
}
else {
@@ -2660,7 +2664,7 @@ process QcAlfred {
options = ""
if (params.assayType == "exome") {
- if (target == "agilent") options = "--bed ${targets}"
+ options = "--bed ${targets}"
}
def ignore = ignore_rg ? "--ignore" : ""
def outfile = ignore_rg ? "${idSample}.alfred.tsv.gz" : "${idSample}.alfred.per_readgroup.tsv.gz"
diff --git a/test_inputs/local/full_test_mapping.tsv b/test_inputs/local/full_test_mapping.tsv
index 24c220b2..4614de9b 100644
--- a/test_inputs/local/full_test_mapping.tsv
+++ b/test_inputs/local/full_test_mapping.tsv
@@ -1,13 +1,13 @@
-SAMPLE ASSAY TARGET FASTQ_PE1 FASTQ_PE2
-DU874145-N exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC1-N_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC1-N_R2_xxx.fastq.gz
-DU874145-N exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants1-N_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants1-N_R2_xxx.fastq.gz
-DU874145-T exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC1-T_L001_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC1-T_L001_R2_xxx.fastq.gz
-DU874145-T exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants1-T_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants1-T_R2_xxx.fastq.gz
-DU874146-N exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC2-N_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC2-N_R2_xxx.fastq.gz
-DU874146-N exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants2-N_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants2-N_R2_xxx.fastq.gz
-DU874146-T exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC2-T_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_MHC2-T_R2_xxx.fastq.gz
-DU874146-T exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants2-T_R1_xxx.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/small_Variants2-T_R2_xxx.fastq.gz
-DU874145-N exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874145-N_IGO_00000_TEST_R1_001.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874145-N_IGO_00000_TEST_R2_001.fastq.gz
-DU874145-T exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874145-T_IGO_00000_TEST_R1_001.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874145-T_IGO_00000_TEST_R2_001.fastq.gz
-DU874146-N exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874146-N_IGO_00000_TEST_R1_001.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874146-N_IGO_00000_TEST_R2_001.fastq.gz
-DU874146-T exome agilent /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874146-T_IGO_00000_TEST_R1_001.fastq.gz /juno/work/taylorlab/cmopipeline/testdata/fastq/DU874146-T_IGO_00000_TEST_R2_001.fastq.gz
+SAMPLE ASSAY TARGET FASTQ_PE1 FASTQ_PE2 NUM_OF_PAIRS
+DU874145-N exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC1-N_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC1-N_R2_xxx.fastq.gz 3
+DU874145-N exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants1-N_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants1-N_R2_xxx.fastq.gz 3
+DU874145-T exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC1-T_L001_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC1-T_L001_R2_xxx.fastq.gz 3
+DU874145-T exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants1-T_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants1-T_R2_xxx.fastq.gz 3
+DU874146-N exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC2-N_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC2-N_R2_xxx.fastq.gz 3
+DU874146-N exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants2-N_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants2-N_R2_xxx.fastq.gz 3
+DU874146-T exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC2-T_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_MHC2-T_R2_xxx.fastq.gz 3
+DU874146-T exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants2-T_R1_xxx.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/small_Variants2-T_R2_xxx.fastq.gz 3
+DU874145-N exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/DU874145-N_IGO_00000_TEST_R1_001.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/DU874145-N_IGO_00000_TEST_R2_001.fastq.gz 3
+DU874145-T exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/DU874145-T_IGO_00000_TEST_R1_001.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/DU874145-T_IGO_00000_TEST_R2_001.fastq.gz 3
+DU874146-N exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/DU874146-N_IGO_00000_TEST_R1_001.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/DU874146-N_IGO_00000_TEST_R2_001.fastq.gz 3
+DU874146-T exome agilent /juno/work/tempo/cmopipeline/testdata/fastq/DU874146-T_IGO_00000_TEST_R1_001.fastq.gz /juno/work/tempo/cmopipeline/testdata/fastq/DU874146-T_IGO_00000_TEST_R2_001.fastq.gz 3
diff --git a/tests/test_somatic.tsv b/tests/bamMapping.tsv
similarity index 100%
rename from tests/test_somatic.tsv
rename to tests/bamMapping.tsv
diff --git a/tests/cohort.tsv b/tests/cohort.tsv
new file mode 100644
index 00000000..9c37bd7f
--- /dev/null
+++ b/tests/cohort.tsv
@@ -0,0 +1,2 @@
+NORMAL_ID TUMOR_ID COHORT PATH COHORT_SIZE
+1234N 1234T cohort1 test-data/test_result 1
diff --git a/tests/duplicate_samplelane_makebamqc.tsv b/tests/duplicate_samplelane_makebamqc.tsv
deleted file mode 100644
index d66e67df..00000000
--- a/tests/duplicate_samplelane_makebamqc.tsv
+++ /dev/null
@@ -1,14 +0,0 @@
-SAMPLE ASSAY TARGET FASTQ_PE1 FASTQ_PE2
-1234N exome agilent test-data/testdata/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz
-1234N exome agilent test-data/testdata/tiny/normal/tiny_n_L002_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L002_R2_xxx.fastq.gz
-1234N exome agilent test-data/testdata/tiny/normal/tiny_n_L004_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L004_R2_xxx.fastq.gz
-1234N exome agilent test-data/testdata/tiny/normal/tiny_n_L007_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L007_R2_xxx.fastq.gz
-1234N exome agilent test-data/testdata/tiny/normal/tiny_n_L008_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L008_R2_xxx.fastq.gz
-1234T exome agilent test-data/testdata/tiny/tumor/tiny_t_L001_R1_xxx.fastq.gz test-data/testdata/tiny/tumor/tiny_t_L001_R2_xxx.fastq.gz
-1234T exome agilent test-data/testdata/tiny/tumor/tiny_t_L002_R1_xxx.fastq.gz test-data/testdata/tiny/tumor/tiny_t_L002_R2_xxx.fastq.gz
-1234T exome agilent test-data/testdata/tiny/tumor/tiny_t_L003_R1_xxx.fastq.gz test-data/testdata/tiny/tumor/tiny_t_L003_R2_xxx.fastq.gz
-1234T exome agilent test-data/testdata/tiny/tumor/tiny_t_L005_R1_xxx.fastq.gz test-data/testdata/tiny/tumor/tiny_t_L005_R2_xxx.fastq.gz
-1234T exome agilent test-data/testdata/tiny/tumor/tiny_t_L006_R1_xxx.fastq.gz test-data/testdata/tiny/tumor/tiny_t_L006_R2_xxx.fastq.gz
-1234T exome agilent test-data/testdata/tiny/tumor/tiny_t_L007_R1_xxx.fastq.gz test-data/testdata/tiny/tumor/tiny_t_L007_R2_xxx.fastq.gz
-1235N exome idt test-data/testdata/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz
-1235T exome idt test-data/testdata/tiny/normal/tiny_n_L001_R1_xxx.fastq.gz test-data/testdata/tiny/normal/tiny_n_L001_R2_xxx.fastq.gz
\ No newline at end of file
diff --git a/tests/test_make_bam_and_qc.tsv b/tests/mapping.tsv
similarity index 100%
rename from tests/test_make_bam_and_qc.tsv
rename to tests/mapping.tsv
diff --git a/tests/test_make_bam_and_qc_pairing.tsv b/tests/pairing.tsv
similarity index 100%
rename from tests/test_make_bam_and_qc_pairing.tsv
rename to tests/pairing.tsv
diff --git a/tests/test_pairing_duplicate.tsv b/tests/test_pairing_duplicate.tsv
deleted file mode 100644
index 4665e900..00000000
--- a/tests/test_pairing_duplicate.tsv
+++ /dev/null
@@ -1,4 +0,0 @@
-NORMAL_ID TUMOR_ID
-1234N 1234T
-1235N 1235T
-1234N 1234T
diff --git a/tests/tests.json b/tests/tests.json
index 3e44a37f..4b1e35cf 100644
--- a/tests/tests.json
+++ b/tests/tests.json
@@ -1,26 +1,18 @@
{
- "test_make_bam_part": {
- "command": ["./nextflow", "run", "pipeline.nf", "--mapping", "tests/test_make_bam_and_qc.tsv", "-profile", "test"],
- "checks": [{"type": "checkNumberOfLines", "filename": "bamMapping.tsv", "num_lines": 3}, {"type": "checkExitCode", "expected": 0}]
+ "test_full": {
+ "command": ["nextflow", "run", "dsl2.nf", "--mapping", "tests/mapping.tsv", "--pairing", "tests/pairing.tsv", "-profile", "test", "--workflows", "snv,sv,mutsig,lohhla,facets,msisensor,germSNV,germSV", "-resume", "--aggregate"],
+ "checks": [{"type": "checkNumberOfLines", "filename": "test-data/test_result/cohort_level/default_cohort/sample_data.txt", "num_lines": 2}, {"type": "checkExitCode", "expected": 0}]
},
- "test_manta_strelka": {
- "command": ["./nextflow", "run", "pipeline.nf", "--bamMapping", "tests/test_somatic.tsv", "--pairing", "tests/test_make_bam_and_qc_pairing.tsv", "-profile", "test", "--tools", "manta,strelka2", "--somatic", "--germline"],
- "checks": [{"type": "checkExitCode", "expected": 0}]
+ "test_aggregate_different_cohort": {
+ "command": ["nextflow", "run", "dsl2.nf", "--mapping", "tests/mapping.tsv", "--pairing", "tests/pairing.tsv", "-profile", "test", "--workflows", "snv,sv,mutsig,lohhla,facets,msisensor, germSNV, germSV", "--aggregate", "tests/cohort.tsv", "-resume"],
+ "checks": [{"type": "checkNumberOfLines", "filename": "test-data/test_result/cohort_level/cohort1/sample_data.txt", "num_lines": 2}, {"type": "checkExitCode", "expected": 0}]
},
- "test_sv": {
- "command": ["./nextflow", "run", "pipeline.nf", "--bamMapping", "tests/test_somatic.tsv", "--pairing", "tests/test_make_bam_and_qc_pairing.tsv", "-profile", "test", "--tools", "delly,manta", "--somatic", "--germline", "--aggregate"],
- "checks": [{"type": "checkExitCode", "expected": 0}]
+ "test_bamMapping": {
+ "command": ["nextflow", "run", "dsl2.nf", "--bamMapping", "tests/bamMapping.tsv", "--pairing", "tests/pairing.tsv", "-profile", "test", "--workflows", "snv,sv,mutsig,lohhla,facets,msisensor,germSNV,germSV", "--aggregate"],
+ "checks": [{"type": "checkNumberOfLines", "filename": "test-data/test_result/cohort_level/default_cohort/sample_data.txt", "num_lines": 2}, {"type": "checkExitCode", "expected": 0}]
},
- "test_msisensor": {
- "command": ["./nextflow", "run", "pipeline.nf", "--bamMapping", "tests/test_somatic.tsv", "--pairing", "tests/test_make_bam_and_qc_pairing.tsv", "-profile", "test", "--tools", "msisensor", "--somatic"],
- "checks": [{"type": "checkExitCode", "expected": 0}]
- },
- "test_pairing_file_validation_pipeline": {
- "command": ["./nextflow", "run", "pipeline.nf", "--mapping", "tests/test_make_bam_and_qc.tsv", "--pairing", "tests/test_pairing_duplicate.tsv", "--somatic", "-profile", "test"],
- "checks": [{"type": "checkExitCode", "expected": 1}]
- },
- "test_nonUnique_sampleLane_validation_pipeline": {
- "command": ["./nextflow", "run", "pipeline.nf", "--mapping", "tests/duplicate_samplelane_makebamqc.tsv", "-profile", "test"],
- "checks": [{"type": "checkExitCode", "expected": 1}]
+ "test_aggregate_from_result": {
+ "command": ["nextflow", "run", "dsl2.nf", "-profile", "test", "--aggregate", "tests/cohort.tsv"],
+ "checks": [{"type": "checkNumberOfLines", "filename": "test-data/test_result/cohort_level/cohort1/sample_data.txt", "num_lines": 2}, {"type": "checkExitCode", "expected": 0}]
}
}
TEMPO
+# Time-Efficient Mutational Profiling in Oncology (Tempo)
-Tempo is a CMO Computational Sciences (CCS) research pipeline processing WES & WGS tumor-normal pairs using the [Nextflow framework](https://www.nextflow.io/). Currently the pipeline is composed of alignment and QC, and detection of both somatic alterations and germline variants. Users can begin with inputs of either paired-end FASTQs or BAMs, and process these via the command line.
+Tempo is a computational pipeline for processing data of paired-end whole-exome (WES) and whole-genome sequencing (WGS) of human cancer samples with matched normals. Its components are containerized and the pipeline runs on the [Juno high-performance computing cluster](http://mskcchpc.org/display/CLUS/Juno+Cluster+Guide) at Memorial Sloan Kettering Cancer Center and on [Amazon Web Services (AWS)](https://aws.amazon.com). The pipeline was written by members of the [Center for Molecular Oncology](https://www.mskcc.org/research-programs/molecular-oncology).
-For further details of how to begin processing data with Tempo, please view our [documentation](https://cmotempo.netlify.com/). For contributing to this project, please make a pull request as detailed [here](https://cmotempo.netlify.com/contributing-to-tempo.html).
+These pages contain instructions on how to run the Tempo pipeline. It also contains documentation on the bioinformatic components in the pipeline, some motivation for various parameter choices, plus an outline describing the reference resources used.
+
+If there are any questions or comments, you are welcome to [raise an issue](https://github.com/mskcc/tempo/issues/new?title=[User%20question]).
+
+Note: Tempo currently only supports human samples. The pipeline has only been tested for exome and genome sequencing experiments, and all reference files are in build GRCh37 of the human genome.
+
+---
+
+## Table of Contents
+
+### 1. Getting Started
+
+#### 1.1. Setup
+* [Installation](docs/installation.md)
+* [Setup on Juno](docs/juno-setup.md)
+* [Setup on AWS](docs/aws-setup.md)
+
+#### 1.2. Usage
+* [Running the Pipeline](docs/running-the-pipeline.md)
+ * [Overview](docs/running-the-pipeline.md#overview)
+ * [Input Files](docs/running-the-pipeline.md#input-files)
+ * [Execution Mode](docs/running-the-pipeline.md#execution-mode)
+ * [Modifying or Resuming Pipeline Run](docs/running-the-pipeline.md#modifying-or-resuming-pipeline-run)
+ * [After Successful Run](docs/running-the-pipeline.md#after-successful-run)
+* [Nextflow Basics](docs/nextflow-basics.md)
+* [Working With Containers](docs/working-with-containers.md)
+
+#### 1.3 Outputs
+* [BAM Files](docs/outputs.md#bam-files)
+* [QC Outputs](docs/outputs.md#qc-outputs)
+* [Somatic Data](docs/outputs.md#somatic-data)
+* [Germline Data](docs/outputs.md#germline-data)
+* [Cohort Level Outputs](docs/outputs.md#cohort-level-outputs)
+
+### 2. Pipeline contents
+
+#### 2.1. Bioinformatic Components
+* [Read Alignment](docs/bioinformatic-components.md#read-alignment)
+* [Somatic Analyses](docs/bioinformatic-components.md#somatic-analyses)
+* [Germline Analyses](docs/bioinformatic-components.md#germline-analyses)
+* [Quality Control](docs/bioinformatic-components.md#quality-control)
+
+#### 2.2. Reference Resources
+* [Genome Assembly](docs/reference-files.md#genome-assembly)
+* [Genomic Intervals](docs/reference-files.md#genomic-intervals)
+* [RepeatMasker and Mappability Blacklist](docs/reference-files.md#repeatmasker-and-mappability-blacklist)
+* [Preferred Transcript Isoforms](docs/reference-files.md#preferred-transcript-isoforms)
+* [Hotspot Annotation](docs/reference-files.md#hotspot-annotation.md)
+* [OncoKB Annotation](docs/reference-files.md#oncokb.md)
+* [gnomAD](docs/gnomad.md)
+* [Panel of Normals for Exomes](docs/wes-panel-of-normals.md)
+
+#### 2.3. Variant Annotation and Filtering
+* [Somatic SNVs and Indels](docs/variant-annotation-and-filtering.md#somatic-snvs-and-indels)
+* [Germline SNVs and Indels](docs/variant-annotation-and-filtering.md#germline-snvs-and-indels)
+* [Somatic and Germline SVs](docs/variant-annotation-and-filtering.md#somatic-and-germline-svs)
+
+### 3. Help and Other Resources
+* [Troubleshooting](docs/troubleshooting.md)
+* [AWS Glossary](docs/aws-glossary.md)
+
+### 4. Contributing
+* [Contributing to Tempo](docs/contributing-to-tempo.md)
+
+### 5. Acknowledgements
+* [Acknowledgements](docs/acknowledgements.md)
-The inspiration for this project derives from [Sarek](https://github.com/SciLifeLab/Sarek), developed at [SciLifeLab](https://github.com/SciLifeLab).
## Pipeline Flowchart
-
-
-