Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

regrid-xy scripting to use pathlib for filename parsing #32

Closed
wants to merge 3 commits into from
Closed

regrid-xy scripting to use pathlib for filename parsing #32

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1f1293b
#31 Use pathlib for filepath parsing, and cleanup
Sep 26, 2024
88a7d60
Merge remote-tracking branch 'origin/main' into 31.use-pathlib
Sep 27, 2024
332be9a
#31 Replaced most of regrid-xy string filepath parsing with pathlib
Sep 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 39 additions & 59 deletions app/regrid-xy/regrid_xy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
import metomi.rose.config as rose_cfg
from netCDF4 import Dataset

FREGRID_SHARED_FILES='/home/fms/shared_fregrid_remap_files'

# formerly in shared.sh
def truncate_date(date, freq):
format=freq_to_date_format(freq)
Expand Down Expand Up @@ -164,36 +162,29 @@ def regrid_xy( ):
'''

## rose config load check
config_name = os.getcwd()
config_name += '/rose-app-run.conf'
#config_name += '/rose-app.conf'
config_name = Path.cwd().joinpath('rose-app-run.conf')
print(f'config_name = {config_name}')
try:
rose_app_config = rose_cfg.load(config_name)
rose_app_config = rose_cfg.load(str(config_name))
except Exception as exc:
raise Exception(f'config_name = {config_name} not found.') \
from exc


# mandatory arguments- code exits if any of these are not present
input_dir = os.getenv( 'inputDir' )
output_dir = os.getenv( 'outputDir' )
input_dir = Path(os.getenv( 'inputDir' ))
output_dir = Path(os.getenv( 'outputDir' ))
begin = os.getenv( 'begin' )
tmp_dir = os.getenv( 'TMPDIR' )
remap_dir = os.getenv( 'fregridRemapDir' )
tmp_dir = Path(os.getenv( 'TMPDIR' ))
remap_dir = Path(os.getenv( 'fregridRemapDir' ))
source = os.getenv( 'source' )
grid_spec = os.getenv( 'gridSpec' )
grid_spec = Path(os.getenv( 'gridSpec' ))
def_xy_interp = os.getenv( 'defaultxyInterp' )
if None in [ input_dir , output_dir ,
begin , tmp_dir ,
remap_dir , source ,
grid_spec , def_xy_interp ]:
raise Exception(f'a mandatory input argument is not present in {config_name})')
# if any( [ input_dir is None, output_dir is None,
# begin is None, tmp_dir is None,
# remap_dir is None, source is None,
# grid_spec is None, def_xy_interp is None ] ):
# raise Exception(f'a mandatory input argument is not present in {config_name}')

def_xy_interp = def_xy_interp.split(',')
def_xy_interp[0] = def_xy_interp[0].replace('"', '')
Expand All @@ -213,39 +204,39 @@ def regrid_xy( ):
f'default xy interpolation has invalid format: \n def_xy_interp = {def_xy_interp}')

# input dir must exist
if not Path( input_dir ).exists():
if not input_dir.exists():
raise Exception(f'input_dir={input_dir} \n does not exist')

# tmp_dir check
if not Path( tmp_dir ).exists():
if not tmp_dir.exists():
raise Exception(f'tmp_dir={tmp_dir} \n does not exist.')

# output dir check
Path( output_dir ).mkdir( parents = True, exist_ok = True )
if not Path( output_dir ).exists() :
output_dir.mkdir( parents = True, exist_ok = True )
if not output_dir.exists() :
raise Exception('the following does not exist and/or could not be created:' +
f'output_dir=\n{output_dir}')

# work/ dir check
work_dir = tmp_dir + 'work/'
Path( work_dir ).mkdir( exist_ok = True )
if not Path( work_dir ).exists():
work_dir = tmp_dir.joinpath('work')
work_dir.mkdir( exist_ok = True )
if not work_dir.exists():
raise Exception('the following does not exist and/or could not be created:' +
f'work_dir=\n{work_dir}')


# fregrid remap dir check
Path(remap_dir).mkdir( exist_ok = True )
if not Path( remap_dir ).exists():
remap_dir.mkdir( exist_ok = True )
if not remap_dir.exists():
raise Exception(f'{remap_dir} could not be created')


# grid_spec file management
starting_dir = os.getcwd()
starting_dir = Path.cwd()
os.chdir(work_dir)
if '.tar' in grid_spec:
if '.tar' in str(grid_spec):
untar_sp = \
subprocess.run( ['tar', '-xvf', grid_spec], check = False , capture_output = True)
subprocess.run( ['tar', '-xvf', grid_spec], check = True , capture_output = True)
if untar_sp.returncode != 0:
raise Exception(
f'untarring of {grid_spec} file failed, ret_code={untar_sp.returncode}, stderr={untar_sp.stderr}')
Expand All @@ -257,7 +248,7 @@ def regrid_xy( ):
raise Exception(f'grid_spec_file cannot be determined from grid_spec={grid_spec}')
else:
try:
grid_spec_file=grid_spec.split('/').pop()
grid_spec_file = grid_spec.name()
shutil.copy(grid_spec, grid_spec_file )
except Exception as exc:
raise Exception(f'grid_spec={grid_spec} could not be copied.') \
Expand Down Expand Up @@ -299,10 +290,9 @@ def regrid_xy( ):

#target input variable resolution
is_tiled = 'cubedsphere' in input_grid
target_file = input_dir
target_file += f"/{truncate_date(begin,'P1D')}.{source}.tile1.nc" \
target_file = input_dir.joinpath(truncate_date(begin, 'P1D') + f".{source}.tile1.nc") \
if is_tiled \
else f"/{truncate_date(begin,'P1D')}.{source}.nc"
else input_dir.joinpath(truncate_date(begin,'P1D') + f".{source}.nc")
if not Path( target_file ).exists():
raise Exception(f'regrid_xy target does not exist. \ntarget_file={target_file}')
print(f'target_file={target_file}') #DELETE
Expand Down Expand Up @@ -359,7 +349,7 @@ def regrid_xy( ):
if remap_file is not None:
try:
shutil.copy( remap_file,
remap_file.split('/').pop() )
remap_file.name() )
except Exception as exc:
raise Exception('remap_file={remap_file} could not be copied to local dir') \
from exc
Expand All @@ -372,24 +362,16 @@ def regrid_xy( ):
else \
f'fregrid_remap_file_{def_xy_interp(0)}_by_{def_xy_interp(1)}.nc'
remap_cache_file = \
f'{remap_dir}/{input_grid}/{input_realm}/' + \
f'{source_nx}-by-{source_ny}/{interp_method}/{remap_file}'
central_remap_cache_file = \
f'{FREGRID_SHARED_FILES}/{input_grid}/' + \
f'{source_nx}_by_{source_ny}/{remap_file}'
remap_dir.joinpath(input_grid, input_realm, \
f'{source_nx}-by-{source_ny}', interp_method, remap_file)

print(f'remap_file = {remap_file }' + \
f'remap_cache_file = {remap_cache_file }' + \
f'central_remap_cache_file = {central_remap_cache_file}' )
f'remap_cache_file = {remap_cache_file }')

if Path( remap_cache_file ).exists():
print(f'NOTE: using cached remap file {remap_cache_file}')
shutil.copy(remap_cache_file,
remap_cache_file.split('/').pop())
elif Path( central_remap_cache_file ).exists():
print(f'NOTE: using centrally cached remap file {remap_cache_file}')
shutil.copy(central_remap_cache_file,
central_remap_cache_file.split('/').pop())
remap_cache_file.name)



Expand All @@ -407,58 +389,56 @@ def regrid_xy( ):


# massage input file argument to fregrid.
input_file = target_file.replace('.tile1.nc','') \
if '.tile1' in target_file \
input_file = Path(str(target_file).replace('.tile1.nc','')) \
if '.tile1' in str(target_file) \
else target_file
input_file=input_file.split('/').pop()

# create output file argument...
output_file = target_file.replace('.tile1','') \
if 'tile1' in target_file \
output_file = Path(str(target_file).replace('.tile1','')) \
if 'tile1' in str(target_file) \
else target_file
output_file = output_file.split('/').pop()

fregrid_command = [
'fregrid',
'--debug',
'--standard_dimension',
'--input_mosaic', f'{input_mosaic}',
'--input_dir', f'{input_dir}',
'--input_file', f'{input_file}',
'--input_file', input_file.name,
'--associated_file_dir', f'{input_dir}',
'--interp_method', f'{interp_method}',
'--remap_file', f'{remap_file}',
'--nlon', f'{str(output_grid_lon)}',
'--nlat', f'{str(output_grid_lat)}',
'--scalar_field', f'{regrid_vars_str}',
'--output_file', f'{output_file}']
'--output_file', output_file.name]
if more_options is not None:
fregrid_command.append(f'{more_options}')

print(f"\n\nabout to run the following command: \n{' '.join(fregrid_command)}\n")
fregrid_proc = subprocess.run( fregrid_command, check = False )#i hate it
fregrid_proc = subprocess.run( fregrid_command, check = True )
fregrid_rc =fregrid_proc.returncode
print(f'fregrid_result.returncode()={fregrid_rc}')


# output wrangling

# copy the remap file to the cache location
if not Path( remap_cache_file ).exists():
remap_cache_file_dir='/'.join(remap_cache_file.split('/')[0:-1])
Path( remap_cache_file_dir ).mkdir( parents = True , exist_ok = True)
if not remap_cache_file.exists():
remap_cache_file_dir=remap_cache_file.parent
remap_cache_file_dir.mkdir( parents = True , exist_ok = True)
print(f'copying \nremap_file={remap_file} to')
print(f'remap_cache_file_dir={remap_cache_file_dir}')
shutil.copy(remap_file, remap_cache_file_dir)

# more output wrangling
final_output_dir = output_dir \
if output_grid_type is None \
else output_dir + '/' + output_grid_type
else output_dir.joinpath(output_grid_type)
Path( final_output_dir ).mkdir( exist_ok = True)

print(f'TRYING TO COPY {output_file} TO {final_output_dir}')
shutil.copy(output_file, final_output_dir)
shutil.copy(output_file.name, final_output_dir)

continue # end of comp loop, exit or next one.

Expand Down