Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add deleteTasksOnCompletion to Azure Batch configuration #4114

Merged
merged 11 commits into from
Jul 25, 2023
Merged

Add deleteTasksOnCompletion to Azure Batch configuration #4114

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
778d0f0
Add deleteTasksOnCompletion to Azure Configuration
adamrtalbot Jul 19, 2023
32abc3c
Revert commited Const.groovy file
adamrtalbot Jul 20, 2023
51bf0d9
minor edits
bentsherman Jul 21, 2023
caa2366
Cleanup docs and comments
bentsherman Jul 21, 2023
64307dc
Change default Azure deleteJobsOnCompletion to false
adamrtalbot Jul 21, 2023
a86498a
Be more specific with azure.batch.terminateJobsOnCompletion docs
adamrtalbot Jul 21, 2023
48c59ff
Azure config tests corrected
adamrtalbot Jul 21, 2023
46a4301
Update docs/config.md
adamrtalbot Jul 21, 2023
1e0fa0b
Update docs/config.md
adamrtalbot Jul 21, 2023
c6c9527
Update tests and docs
bentsherman Jul 24, 2023
e363418
Update docs for deleteTasksOnCompletion
bentsherman Jul 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/config.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,9 @@ The following settings are available:
`azure.batch.terminateJobsOnCompletion`
: Enables the Batch Job to automatically terminate a job once all tasks have completed (default: `true`).

`azure.batch.deleteTasksOnCompletion`
: Delete tasks after successful completion. This deletes them on the Azure Batch service but files and resources may persist on the compute nodes (default: `true`).

`azure.batch.deleteJobsOnCompletion`
: Enable the automatic deletion of jobs created by the pipeline execution (default: `true`).

Expand Down
4 changes: 2 additions & 2 deletions plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchTaskHandler.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,15 +130,15 @@ class AzBatchTaskHandler extends TaskHandler implements FusionAwareTask {
}

private Boolean shouldDelete() {
executor.config.batch().deleteJobsOnCompletion
bentsherman marked this conversation as resolved.
Show resolved Hide resolved
executor.config.batch().deleteTasksOnCompletion
}

protected void deleteTask(AzTaskKey taskKey, TaskRun task) {
if( !taskKey || shouldDelete()==Boolean.FALSE )
return

if( !task.isSuccess() && shouldDelete()==null ) {
// do not delete successfully executed pods for debugging purpose
// do not delete successfully executed tasks for debugging purpose
return
}

Expand Down
2 changes: 2 additions & 0 deletions plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ class AzBatchOpts implements CloudTransferOptions {
Boolean autoPoolMode
Boolean allowPoolCreation
Boolean terminateJobsOnCompletion
Boolean deleteTasksOnCompletion
Boolean deleteJobsOnCompletion
Boolean deletePoolsOnCompletion
CopyToolInstallMode copyToolInstallMode
Expand All @@ -64,6 +65,7 @@ class AzBatchOpts implements CloudTransferOptions {
autoPoolMode = config.autoPoolMode
allowPoolCreation = config.allowPoolCreation
terminateJobsOnCompletion = config.terminateJobsOnCompletion
deleteTasksOnCompletion = config.deleteTasksOnCompletion
deleteJobsOnCompletion = config.deleteJobsOnCompletion
deletePoolsOnCompletion = config.deletePoolsOnCompletion
pools = parsePools(config.pools instanceof Map ? config.pools as Map<String,Map> : Collections.<String,Map>emptyMap())
Expand Down
1 change: 0 additions & 1 deletion plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,6 @@ class AzBatchServiceTest extends Specification {

}


def 'should cleanup jobs by default' () {
given:
def CONFIG = [:]
Expand Down
7 changes: 6 additions & 1 deletion plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzureConfigTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,10 @@ class AzureConfigTest extends Specification {
endpoint: ENDPOINT,
location: LOCATION,
autoPoolMode: true,
allowPoolCreation: true, deleteJobsOnCompletion: false,
allowPoolCreation: true,
terminateJobsOnCompletion: false,
deleteTasksOnCompletion: false,
deleteJobsOnCompletion: false,
deletePoolsOnCompletion: true,
pools: [ myPool: [
vmType: 'Foo_A1',
Expand All @@ -124,6 +127,8 @@ class AzureConfigTest extends Specification {
cfg.batch().location == LOCATION
cfg.batch().autoPoolMode == true
cfg.batch().allowPoolCreation == true
cfg.batch().terminateJobsOnCompletion == false
cfg.batch().deleteTasksOnCompletion == false
cfg.batch().deleteJobsOnCompletion == false
cfg.batch().deletePoolsOnCompletion == true
cfg.batch().canCreatePool()
Expand Down
Loading