Add deleteTasksOnCompletion to Azure Configuration

Deleting Azure Tasks was checking the configuration object deleteJobsOnCompletion which was incorrect since a task belongs to a job. This adds the equivalent configuration for tasks which is checked before deleting the tasks. Signed-off-by: Adam Talbot <[email protected]>
nextflow-io · Jul 19, 2023 · 778d0f0 · 778d0f0
1 parent 0933f47
commit 778d0f0
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 6 deletions.
diff --git a/docs/config.md b/docs/config.md
@@ -333,6 +333,9 @@ The following settings are available:
 `azure.batch.terminateJobsOnCompletion`
 : Enables the Batch Job to automatically terminate a job once all tasks have completed (default: `true`).
 
+`azure.batch.deleteTasksOnCompletion`
+: Delete tasks after successful completion. This deletes them on the Azure Batch service but files and resources may persist on the compute nodes (default: `true`).
+
 `azure.batch.deleteJobsOnCompletion`
 : Enable the automatic deletion of jobs created by the pipeline execution (default: `true`).
 

diff --git a/modules/nf-commons/src/main/nextflow/Const.groovy b/modules/nf-commons/src/main/nextflow/Const.groovy
@@ -57,12 +57,12 @@ class Const {
     /**
      * The app build time as linux/unix timestamp
      */
-    static public final long APP_TIMESTAMP = 1686776566745
+    static public final long APP_TIMESTAMP = 1689766233402
 
     /**
      * The app build number
      */
-    static public final int APP_BUILDNUM = 5864
+    static public final int APP_BUILDNUM = 5867
 
     /**
      * The app build time string relative to UTC timezone

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchTaskHandler.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchTaskHandler.groovy
@@ -130,15 +130,15 @@ class AzBatchTaskHandler extends TaskHandler implements FusionAwareTask {
     }
 
     private Boolean shouldDelete() {
-        executor.config.batch().deleteJobsOnCompletion
+        executor.config.batch().deleteTasksOnCompletion
     }
 
     protected void deleteTask(AzTaskKey taskKey, TaskRun task) {
         if( !taskKey || shouldDelete()==Boolean.FALSE )
             return
 
         if( !task.isSuccess() && shouldDelete()==null ) {
-            // do not delete successfully executed pods for debugging purpose
+            // do not delete successfully executed tasks for debugging purpose
             return
         }
 

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzBatchOpts.groovy
@@ -48,6 +48,7 @@ class AzBatchOpts implements CloudTransferOptions {
     Boolean autoPoolMode
     Boolean allowPoolCreation
     Boolean terminateJobsOnCompletion
+    Boolean deleteTasksOnCompletion
     Boolean deleteJobsOnCompletion
     Boolean deletePoolsOnCompletion
     CopyToolInstallMode copyToolInstallMode
@@ -64,6 +65,7 @@ class AzBatchOpts implements CloudTransferOptions {
         autoPoolMode = config.autoPoolMode
         allowPoolCreation = config.allowPoolCreation
         terminateJobsOnCompletion = config.terminateJobsOnCompletion
+        deleteTasksOnCompletion = config.deleteTasksOnCompletion
         deleteJobsOnCompletion = config.deleteJobsOnCompletion
         deletePoolsOnCompletion = config.deletePoolsOnCompletion
         pools = parsePools(config.pools instanceof Map ? config.pools as Map<String,Map> : Collections.<String,Map>emptyMap())

diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy
@@ -350,7 +350,6 @@ class AzBatchServiceTest extends Specification {
 
     }
 
-
     def 'should cleanup jobs by default' () {
         given:
         def CONFIG = [:]

diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzureConfigTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/config/AzureConfigTest.groovy
@@ -99,7 +99,10 @@ class AzureConfigTest extends Specification {
                                              endpoint: ENDPOINT,
                                              location: LOCATION,
                                              autoPoolMode: true,
-                                             allowPoolCreation: true,                                            deleteJobsOnCompletion: false,
+                                             allowPoolCreation: true,                                            
+                                             terminateJobsOnCompletion: false,
+                                             deleteTasksOnCompletion: false,
+                                             deleteJobsOnCompletion: false,
                                              deletePoolsOnCompletion: true,
                                              pools: [ myPool: [
                                                      vmType: 'Foo_A1',
@@ -124,6 +127,8 @@ class AzureConfigTest extends Specification {
         cfg.batch().location == LOCATION
         cfg.batch().autoPoolMode == true
         cfg.batch().allowPoolCreation == true
+        cfg.batch().terminateJobsOnCompletion == false
+        cfg.batch().deleteTasksOnCompletion == false
         cfg.batch().deleteJobsOnCompletion == false
         cfg.batch().deletePoolsOnCompletion == true
         cfg.batch().canCreatePool()