From 0da7c1a7e925b8d92f9c23a97146c09f0b9b7e64 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Wed, 30 Oct 2024 16:33:36 -0600 Subject: [PATCH 1/3] update ami --- .../terraform/executions/execution_gp_example.json | 2 +- .../terraform/test/execution_gp_arm_docker_buildNtester.json | 2 +- research_datastream/terraform/test/execution_gp_test.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/research_datastream/terraform/executions/execution_gp_example.json b/research_datastream/terraform/executions/execution_gp_example.json index 205a0f2a..1db212df 100644 --- a/research_datastream/terraform/executions/execution_gp_example.json +++ b/research_datastream/terraform/executions/execution_gp_example.json @@ -8,7 +8,7 @@ }, "instance_parameters" : { - "ImageId" : "ami-062bdcbb454b8d833", + "ImageId" : "ami-03b72f226b125860d", "InstanceType" : "t4g.large", "KeyName" : "jlaser_west2", "SecurityGroupIds" : ["sg-04365a4248fe126bc"], diff --git a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json index 5a037265..b0c7d658 100644 --- a/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json +++ b/research_datastream/terraform/test/execution_gp_arm_docker_buildNtester.json @@ -14,7 +14,7 @@ }, "instance_parameters" : { - "ImageId" : "ami-062bdcbb454b8d833", + "ImageId" : "ami-03b72f226b125860d", "InstanceType" : "t4g.large", "KeyName" : "actions_key_arm", "SecurityGroupIds" : ["sg-0fcbe0c6d6faa0117"], diff --git a/research_datastream/terraform/test/execution_gp_test.json b/research_datastream/terraform/test/execution_gp_test.json index c8c62a22..c088dc03 100644 --- a/research_datastream/terraform/test/execution_gp_test.json +++ b/research_datastream/terraform/test/execution_gp_test.json @@ -8,7 +8,7 @@ }, "instance_parameters" : { - "ImageId" : "ami-062bdcbb454b8d833", + "ImageId" : "ami-03b72f226b125860d", "InstanceType" : "t4g.nano", "KeyName" : "actions_key", "SecurityGroupIds" : ["sg-06f57f883e902d7bc"], From 44d96dccb8a7647e776616440ac3ae0cf2783f41 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Fri, 1 Nov 2024 12:11:03 -0600 Subject: [PATCH 2/3] update forcing URL --- .../src/forcingprocessor/processor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/forcingprocessor/src/forcingprocessor/processor.py b/forcingprocessor/src/forcingprocessor/processor.py index 5f1206d5..33757339 100644 --- a/forcingprocessor/src/forcingprocessor/processor.py +++ b/forcingprocessor/src/forcingprocessor/processor.py @@ -571,7 +571,7 @@ def write_netcdf(data, vpu, t_ax, catchments): """ if storage_type == 's3': s3_client = boto3.session.Session().client("s3") - nc_filename = forcing_path + f'/{vpu}_forcings.nc' + nc_filename = forcing_path + f'/ngen.{FCST_CYCLE}z.{URLBASE}.forcing.f{LEAD_START}_{LEAD_END}.{vpu}.nc' else: nc_filename = Path(forcing_path,f'{vpu}_forcings.nc') @@ -761,7 +761,7 @@ def prep_ngen_data(conf): elif storage_type == "s3": bucket_path = output_path - forcing_path = bucket_path + '/forcings' + forcing_path = bucket_path meta_path = bucket_path + '/metadata' metaf_path = bucket_path + '/metadata/forcings_metadata' bucket, key = convert_url2key(metaf_path,storage_type) @@ -796,6 +796,19 @@ def prep_ngen_data(conf): nwm_forcing_files.append(jline.strip()) nfiles = len(nwm_forcing_files) + # s3://noaa-nwm-pds/nwm.20241029/forcing_short_range/nwm.t00z.short_range.forcing.f001.conus.nc + pattern = r"nwm\.(\d{8})/forcing_(\w+)/nwm\.(\w+)(\d{2})z\.\w+\.forcing\.f(\d{3})\.conus\.nc" + + global URLBASE, FCST_CYCLE, LEAD_START, LEAD_END + match = re.search(pattern, nwm_forcing_files[0]) + if match: + URLBASE = match.group(2) + FCST_CYCLE = match.group(3) + match.group(4) + LEAD_START = match.group(5) + match = re.search(pattern, nwm_forcing_files[-1]) + if match: + LEAD_END = match.group(5) + global fs_type if 's3://' in nwm_forcing_files[0] in nwm_forcing_files[0]: fs = s3fs.S3FileSystem( From 184bea8803471a788b4fede363185d5bcaa942e7 Mon Sep 17 00:00:00 2001 From: Jordan Laser Date: Fri, 1 Nov 2024 12:18:24 -0600 Subject: [PATCH 3/3] implement hack for analysis assim extend --- forcingprocessor/src/forcingprocessor/processor.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/forcingprocessor/src/forcingprocessor/processor.py b/forcingprocessor/src/forcingprocessor/processor.py index 33757339..f281b71b 100644 --- a/forcingprocessor/src/forcingprocessor/processor.py +++ b/forcingprocessor/src/forcingprocessor/processor.py @@ -571,7 +571,7 @@ def write_netcdf(data, vpu, t_ax, catchments): """ if storage_type == 's3': s3_client = boto3.session.Session().client("s3") - nc_filename = forcing_path + f'/ngen.{FCST_CYCLE}z.{URLBASE}.forcing.f{LEAD_START}_{LEAD_END}.{vpu}.nc' + nc_filename = forcing_path + f'/ngen.{FCST_CYCLE}z.{URLBASE}.forcing.{LEAD_START}_{LEAD_END}.{vpu}.nc' else: nc_filename = Path(forcing_path,f'{vpu}_forcings.nc') @@ -797,17 +797,17 @@ def prep_ngen_data(conf): nfiles = len(nwm_forcing_files) # s3://noaa-nwm-pds/nwm.20241029/forcing_short_range/nwm.t00z.short_range.forcing.f001.conus.nc - pattern = r"nwm\.(\d{8})/forcing_(\w+)/nwm\.(\w+)(\d{2})z\.\w+\.forcing\.f(\d{3})\.conus\.nc" + pattern = r"nwm\.(\d{8})/forcing_(\w+)/nwm\.(\w+)(\d{2})z\.\w+\.forcing\.(\w+)(\d{2})\.conus\.nc" global URLBASE, FCST_CYCLE, LEAD_START, LEAD_END match = re.search(pattern, nwm_forcing_files[0]) if match: URLBASE = match.group(2) FCST_CYCLE = match.group(3) + match.group(4) - LEAD_START = match.group(5) + LEAD_START = match.group(5) + match.group(6) match = re.search(pattern, nwm_forcing_files[-1]) if match: - LEAD_END = match.group(5) + LEAD_END = match.group(5) + match.group(6) global fs_type if 's3://' in nwm_forcing_files[0] in nwm_forcing_files[0]: @@ -851,6 +851,9 @@ def prep_ngen_data(conf): # Hack to ensure data is always written out with time moving forward. t_ax=list(reversed(t_ax)) data_array = np.flip(data_array,axis=0) + tmp = LEAD_START + LEAD_START = LEAD_END + LEAD_END = tmp t_extract = time.perf_counter() - t0 complexity = (nfiles_tot * ncatchments) / 10000