only copy big files if sucess

C2SM · Nov 27, 2024 · 49b4b1e · 49b4b1e
1 parent 2414dc2
commit 49b4b1e
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 5 deletions.
diff --git a/jenkins/RemoteExtraction b/jenkins/RemoteExtraction
@@ -62,7 +62,7 @@ pipeline {
     }
     post {
         success {
-            sh "python3 src/archive_artifacts.py --workspace ${WORKSPACE} --destination ${https_public_root} --hash-file ${WORKSPACE}/hash.txt"
+            sh "python3 src/archive_artifacts.py --workspace ${WORKSPACE} --destination ${https_public_root} --hash-file ${WORKSPACE}/hash.txt --sucess"
             withCredentials([string(credentialsId: 'd976fe24-cabf-479e-854f-587c152644bc', variable: 'GITHUB_AUTH_TOKEN')]) {
                 sh "python3 src/report.py --auth_token ${GITHUB_AUTH_TOKEN} --issue_id_file ${WORKSPACE}/issue.txt --hash-file ${WORKSPACE}/hash.txt"
             }

diff --git a/src/archive_artifacts.py b/src/archive_artifacts.py
@@ -10,13 +10,14 @@ def move_files(src_pattern, dest_dir, prefix=""):
         print(f"Moving {file} to {dest_file}")
         shutil.move(file, dest_file)
 
-def move_extpar(workspace, dest):
+def move_extpar(workspace, dest, is_success):
     i = 1
     for domain in sorted(glob.glob(os.path.join(workspace, 'extpar_*'))):
         # Move logfiles
         move_files(os.path.join(domain, "*.log"), os.path.join(dest, 'logs'), f"DOM_{i}_")
         # Move external parameter file
-        move_files(os.path.join(domain, "external_parameter*.nc"), dest, f"DOM_{i}_")
+        if is_success:
+            move_files(os.path.join(domain, "external_parameter.nc"), dest, f"DOM_{i}_")
         i += 1
 
 def move_icontools(workspace, dest):
@@ -51,6 +52,8 @@ def main():
     parser.add_argument('--destination', type=str, required=True, help='The destination folder to store the zip file')
     parser.add_argument('--hash-file', type=str, required=True, help='Hash file')
     parser.add_argument('--workspace', type=str, required=True, help='The workspace folder')
+    parser.add_argument('--sucess', action='store_true', help='If the simulation was successful, archive nc and html files')
+
 
     # Parse the arguments
     args = parser.parse_args()
@@ -64,10 +67,11 @@ def main():
     os.makedirs(output_dir, exist_ok=True)
 
     # Move extpar files
-    move_extpar(args.workspace, output_dir)
+    move_extpar(args.workspace, output_dir, args.sucess)
 
     # Move icontools files
-    move_icontools(args.workspace, output_dir)
+    if args.sucess:
+        move_icontools(args.workspace, output_dir)
 
     # Create a zip file
     zip_file_path = os.path.join(args.workspace, 'output.zip')