-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
c259e65
commit 23f56e8
Showing
6 changed files
with
179 additions
and
138 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| import os | ||
| import argparse | ||
| import shutil | ||
| import glob | ||
| import zipfile | ||
|
|
||
| def copy_files(src_pattern, dest_dir, prefix=""): | ||
| for file in glob.glob(src_pattern): | ||
| dest_file = os.path.join(dest_dir, prefix + os.path.basename(file)) | ||
| print(f"Copying {file} to {dest_file}") | ||
| shutil.copy(file, dest_file) | ||
|
|
||
| def copy_extpar(workspace, dest): | ||
| i = 1 | ||
| for domain in sorted(glob.glob(os.path.join(workspace, 'extpar_*'))): | ||
| # Copy logfiles | ||
| copy_files(os.path.join(domain, "*.log"), os.path.join(dest,'logs'), f"DOM_{i}_") | ||
| # Copy external parameter file | ||
| copy_files(os.path.join(domain, "external_parameter*.nc"), dest, f"DOM_{i}_") | ||
| i += 1 | ||
|
|
||
|
|
||
| def copy_icontools(workspace, dest): | ||
| # Copy .nc files | ||
| copy_files(os.path.join(workspace, 'icontools', '*.nc'), os.path.join(dest)) | ||
| # Copy .html files | ||
| copy_files(os.path.join(workspace, 'icontools', '*.html'), dest) | ||
|
|
||
| def copy_zip(destination, zip_file, hash): | ||
| folder = os.path.join(destination, hash) | ||
| # Create the directory | ||
| os.makedirs(folder, exist_ok=True) | ||
| print(f"Created directory {folder}") | ||
|
|
||
| # Copy the zip file to the directory | ||
| shutil.copy(zip_file, folder) | ||
| print(f"Copied {zip_file} to {folder}") | ||
|
|
||
| def create_zip(zip_file_path, source_dir): | ||
| with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: | ||
| for root, dirs, files in os.walk(source_dir): | ||
| for file in files: | ||
| file_path = os.path.join(root, file) | ||
| arcname = os.path.relpath(file_path, source_dir) | ||
| zipf.write(file_path, arcname) | ||
|
|
||
| def main(): | ||
| # Create the parser | ||
| parser = argparse.ArgumentParser(description="Archive artifacts to a unique folder.") | ||
|
|
||
| # Add the arguments | ||
| parser.add_argument('--destination', type=str, required=True, help='The destination folder to store the zip file') | ||
| parser.add_argument('--hash-file', type=str, required=True, help='Hash file') | ||
| parser.add_argument('--workspace', type=str, required=True, help='The workspace folder') | ||
|
|
||
| # Parse the arguments | ||
| args = parser.parse_args() | ||
|
|
||
| with open(args.hash_file, 'r') as f: | ||
| hash = f.read() | ||
|
|
||
| # Copy icontools and extpar files to the output directory | ||
| output_dir = os.path.join(args.workspace, 'output') | ||
| copy_icontools(args.workspace, output_dir) | ||
| copy_extpar(args.workspace, output_dir) | ||
|
|
||
| # Create a zip file | ||
| zip_file = os.path.join(args.workspace, 'output.zip') | ||
| create_zip(zip_file, output_dir) | ||
| copy_zip(args.destination, zip_file, hash) | ||
|
|
||
| if __name__ == '__main__': | ||
| main() |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,19 @@ | ||
| import os | ||
| import hashlib | ||
| import argparse | ||
| import shutil | ||
| from datetime import datetime | ||
|
|
||
| # Create the parser | ||
| parser = argparse.ArgumentParser(description="Hash Build ID and create a file with the hash value.") | ||
|
|
||
| # Add the arguments | ||
| parser.add_argument('--build-id', type=str, required=True, help='The build ID') | ||
| parser.add_argument('--hash-file', type=str, required=True, help='Hash file') | ||
|
|
||
| # Parse the arguments | ||
| args = parser.parse_args() | ||
|
|
||
| # Compute the SHA256 hash of BUILD_ID | ||
| hash = hashlib.sha256(args.build_id.encode()).hexdigest() | ||
| # Get the current time as a string | ||
| current_time = datetime.now().isoformat() | ||
|
|
||
| # Create a hash from the current time | ||
| hash = hashlib.sha256(current_time.encode()).hexdigest() | ||
|
|
||
| with open(args.hash_file, 'w') as f: | ||
| f.write(hash) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -e | ||
|
|
||
| # Loop over all folders with the pattern extpar_* | ||
| i=1 | ||
| for dir in ${WORKSPACE}/extpar_*; do | ||
| echo "Processing directory: $dir" | ||
| cd "$dir" | ||
| grid_file=$(cat ../icontools/grid_$i.txt) # Assuming grid.txt contains the grid file name | ||
| podman run \ | ||
| -v /c2sm-data/extpar-input-data:/data \ | ||
| -v ${WORKSPACE}/icontools:/grid \ | ||
| -v "$dir":/work \ | ||
| extpar-image \ | ||
| python3 -m extpar.WrapExtpar \ | ||
| --run-dir /work \ | ||
| --raw-data-path /data/linked_data \ | ||
| --account none \ | ||
| --no-batch-job \ | ||
| --host docker \ | ||
| --input-grid /grid/${grid_file} \ | ||
| --extpar-config /work/config.json | ||
| cd .. | ||
| ((i++)) | ||
| done |