-
Notifications
You must be signed in to change notification settings - Fork 5
/
nextflow.config
152 lines (137 loc) · 6.51 KB
/
nextflow.config
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
params {
// Inputs
csv = null // the input file containing all input data
model = null // the model file in python, the model that will be tested by this pipeline
exp_conf = null // the json config file that specifies all the parameters relative to the data manipulation
tune_conf = null // the config file with all the hyperparameter directives (choiches) and all ray tune specs
// Optional inputs
initial_weights = null // the initial weights of the model. These files can be used to start the training instead of random initialization. One can provide several files, each of them will be used for a different run.
// Output options
outdir = "./results/" // the outdir has to be the one the user specify _ the unique name of the run _ the time so that multiple runs will not overlap
publish_dir_mode = "copy"
// Computational resources
max_cpus = 12 // this flasg and the following are for regulating resources, profiles can overwrite these.
max_gpus = 1 // requesting the gpus for the tuning steps.
max_memory = 32.GB
max_time = "72.h"
// Error options
max_retries = 0
err_start = 'finish'
// Optional flags
check_model = true // flag to tell whether to check or not if the model can be tuned and trained. It does one call of the batch function, (predicting), of the model importing and using everything needed for that. Default run such a check.
check_model_num_samples = null // optional flag to do a more extensive check during check_model. This will override user given num_sample value for the tune run. This will give the user control on how extensive it wants the check to be.
shuffle = true // flag to tell wether to shuffle or not the data and run a train on it. Sanity check always run on default. (If the way we think at shuffle change maybe is better to remove this flag and make it into a parameter of the user given json for noise nad split)
debug_mode = false // flag used to switch to debug mode for the pipeline.
// General
singularity_cache_dir = "singularity_cache"
help = false
validate_params = true // tells wether or not to validate input values using nf-schema.
// Config options
config_profile_name = null
config_profile_description = null
}
// Load modules.config for DSL2 module specific options
includeConfig 'conf/modules.config'
profiles {
docker {
docker.enabled = true
docker.runOptions = '-u $(id -u):$(id -g)'
conda.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
apptainer {
apptainer.enabled = true
apptainer.autoMounts = true
apptainer.cacheDir = "${params.singularity_cache_dir}"
conda.enabled = false
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
singularity {
singularity.enabled = true
singularity.autoMounts = true
singularity.cacheDir = "${params.singularity_cache_dir}"
conda.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
apptainer.enabled = false
}
debug {
dumpHashes = true
process.beforeScript = 'echo $HOSTNAME'
process.debug = true
cleanup = false
nextflow.enable.configProcessNamesValidation = true
}
crg { includeConfig "conf/crg.config" }
crg_slurm { includeConfig "conf/crg_slurm.config" }
test { includeConfig "conf/test.config" }
test_learn { includeConfig "conf/test_learn.config" }
test_stub { includeConfig "conf/test_stub.config" }
local { includeConfig "conf/local.config" }
}
// Nextflow plugins
plugins {
id '[email protected]' // Validation of pipeline parameters and creation of an input channel from a sample sheet
}
// trace/report options
// this will allow the pipeline to create tracing/report files with all the steps and the time/memory/cpu they took
def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
def trace_dir = "${params.outdir}/pipeline_info"
timeline {
enabled = true
file = "${trace_dir}/execution_timeline_${trace_timestamp}.html"
}
report {
enabled = true
file = "${trace_dir}/execution_report_${trace_timestamp}.html"
}
trace {
enabled = true
file = "${trace_dir}/execution_trace_${trace_timestamp}.txt"
}
dag {
enabled = true
file = "${trace_dir}/execution_dag_${trace_timestamp}.html"
}
// Function to ensure that resource requirements don't go beyond
// a maximum limit
def check_max(obj, type) {
if (type == 'memory') {
try {
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
return params.max_memory as nextflow.util.MemoryUnit
else
return obj
} catch (all) {
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'time') {
try {
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
return params.max_time as nextflow.util.Duration
else
return obj
} catch (all) {
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
return obj
}
} else if (type == 'cpus') {
try {
return Math.min( obj, params.max_cpus as int )
} catch (all) {
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
return obj
}
}
}